From 16108de71ef00634e70665542929cba8cd3317d2 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 24 Feb 2022 22:14:05 +0800
Subject: [PATCH 001/126] add voxceleb1 dataset prepare process

---
 examples/voxceleb/sv0/local/data.sh | 25 +++++++++++++++++++++++++
 examples/voxceleb/sv0/run.sh        | 14 ++++++++++++++
 examples/voxceleb/sv0/utils         |  1 +
 3 files changed, 40 insertions(+)
 create mode 100755 examples/voxceleb/sv0/local/data.sh
 create mode 100755 examples/voxceleb/sv0/run.sh
 create mode 120000 examples/voxceleb/sv0/utils

diff --git a/examples/voxceleb/sv0/local/data.sh b/examples/voxceleb/sv0/local/data.sh
new file mode 100755
index 00000000..0c2d008d
--- /dev/null
+++ b/examples/voxceleb/sv0/local/data.sh
@@ -0,0 +1,25 @@
+stage=-1
+stop_stage=100
+TARGET_DIR=${MAIN_ROOT}/dataset
+
+. utils/parse_options.sh || exit -1;
+
+src=$1
+mkdir -p data/{dev,test}
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    # download data, generate manifests
+    # create data/{dev,test} directory to store the manifest files
+    /home/users/xiongxinlei/.conda/envs/xxl_base/bin/python3 ${TARGET_DIR}/voxceleb/voxceleb1.py \
+    --manifest_prefix="data/manifest" \
+    --target_dir="${src}"
+
+    if [ $? -ne 0 ]; then
+        echo "Prepare Voxceleb failed. Terminated."
+        exit 1
+    fi
+    mv data/manifest.dev data/dev
+    mv data/voxceleb1.dev.meta data/dev
+
+    mv data/manifest.test data/test
+    mv data/voxceleb1.test.meta data/test
+fi
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
new file mode 100755
index 00000000..281f7b40
--- /dev/null
+++ b/examples/voxceleb/sv0/run.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+stage=0
+voxceleb1_root=/mnt/dataset_12/sv/voxCeleb1_v2/
+
+if [ $stage -le 0 ]; then
+    echo "======================================================================================================"
+    echo "=========================== Stage 0: Prepare the VoxCeleb1 dataset ==================================="
+    echo "======================================================================================================"
+    # prepare the data elapsed about 20s
+    # the script will create the data/{dev,test}
+    local/data.sh ${voxceleb1_root}|| exit 1;
+fi
diff --git a/examples/voxceleb/sv0/utils b/examples/voxceleb/sv0/utils
new file mode 120000
index 00000000..256f914a
--- /dev/null
+++ b/examples/voxceleb/sv0/utils
@@ -0,0 +1 @@
+../../../utils/
\ No newline at end of file

From 35b7968ed10525a42bb8b1f3d82d6b203801f982 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Fri, 25 Feb 2022 19:38:22 +0800
Subject: [PATCH 002/126] remove invalid directory

---
 examples/voxceleb/sv0/local/data.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/voxceleb/sv0/local/data.sh b/examples/voxceleb/sv0/local/data.sh
index 0c2d008d..6df9c3b8 100755
--- a/examples/voxceleb/sv0/local/data.sh
+++ b/examples/voxceleb/sv0/local/data.sh
@@ -9,7 +9,7 @@ mkdir -p data/{dev,test}
 if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
     # download data, generate manifests
     # create data/{dev,test} directory to store the manifest files
-    /home/users/xiongxinlei/.conda/envs/xxl_base/bin/python3 ${TARGET_DIR}/voxceleb/voxceleb1.py \
+    python3 ${TARGET_DIR}/voxceleb/voxceleb1.py \
     --manifest_prefix="data/manifest" \
     --target_dir="${src}"
 
@@ -22,4 +22,4 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
 
     mv data/manifest.test data/test
     mv data/voxceleb1.test.meta data/test
-fi
\ No newline at end of file
+fi

From 6f7e9656febf8e399ef09b749da7641bee438dc5 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Fri, 25 Feb 2022 20:05:25 +0800
Subject: [PATCH 003/126] add kaldi feats ark dataset

---
 paddlespeech/vector/datasets/dataset.py | 143 ++++++++++++++++++++++++
 paddlespeech/vector/utils/data_utils.py | 125 +++++++++++++++++++++
 paddlespeech/vector/utils/utils.py      | 132 ++++++++++++++++++++++
 3 files changed, 400 insertions(+)
 create mode 100644 paddlespeech/vector/datasets/dataset.py
 create mode 100644 paddlespeech/vector/utils/data_utils.py
 create mode 100644 paddlespeech/vector/utils/utils.py

diff --git a/paddlespeech/vector/datasets/dataset.py b/paddlespeech/vector/datasets/dataset.py
new file mode 100644
index 00000000..e7030053
--- /dev/null
+++ b/paddlespeech/vector/datasets/dataset.py
@@ -0,0 +1,143 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import random
+import numpy as np
+import kaldi_python_io as k_io
+from paddle.io import Dataset
+from paddlespeech.vector.utils.data_utils import batch_pad_right
+import paddlespeech.vector.utils as utils
+from paddlespeech.vector.utils.utils import read_map_file
+
+def ark_collate_fn(batch):
+    """
+    Custom collate function for kaldi feats dataset
+
+    Args:
+        min_chunk_size: min chunk size of a utterance
+        max_chunk_size: max chunk size of a utterance
+
+    Returns:
+        ark_collate_fn: collate funtion for dataloader
+    """
+
+    data = []
+    target = []
+    for items in batch:
+        for x, y in zip(items[0], items[1]):
+            data.append(np.array(x))
+            target.append(y)
+
+    data, lengths = batch_pad_right(data)
+    return np.array(data, dtype=np.float32), \
+           np.array(lengths, dtype=np.float32), \
+           np.array(target, dtype=np.long).reshape((len(target), 1))
+
+
+class KaldiArkDataset(Dataset):
+    """
+    Dataset used to load kaldi ark/scp files.
+    """
+    def __init__(self, scp_file, label2utt, min_item_size=1,
+                 max_item_size=1, repeat=50, min_chunk_size=200,
+                 max_chunk_size=400, select_by_speaker=True):
+        self.scp_file = scp_file
+        self.scp_reader = None
+        self.repeat = repeat
+        self.min_item_size = min_item_size
+        self.max_item_size = max_item_size
+        self.min_chunk_size = min_chunk_size
+        self.max_chunk_size = max_chunk_size
+        self._collate_fn = ark_collate_fn
+        self._is_select_by_speaker = select_by_speaker
+        if utils.is_exist(self.scp_file):
+            self.scp_reader = k_io.ScriptReader(self.scp_file)
+
+        label2utts, utt2label = read_map_file(label2utt, key_func=int)
+        self.utt_info = list(label2utts.items()) if self._is_select_by_speaker else list(utt2label.items())
+
+    @property
+    def collate_fn(self):
+        """
+        Return a collate funtion.
+        """
+        return self._collate_fn
+
+    def _random_chunk(self, length):
+        chunk_size = random.randint(self.min_chunk_size, self.max_chunk_size)
+        if chunk_size >= length:
+            return 0, length
+        start = random.randint(0, length - chunk_size)
+        end = start + chunk_size
+
+        return start, end
+
+    def _select_by_speaker(self, index):
+        if self.scp_reader is None or not self.utt_info:
+            return []
+        index = index % (len(self.utt_info))
+        inputs = []
+        labels = []
+        item_size = random.randint(self.min_item_size, self.max_item_size)
+        for loop_idx in range(item_size):
+            try:
+                utt_index = random.randint(0, len(self.utt_info[index][1])) \
+                        % len(self.utt_info[index][1])
+                key = self.utt_info[index][1][utt_index]
+            except:
+                print(index, utt_index, len(self.utt_info[index][1]))
+                sys.exit(-1)
+            x = self.scp_reader[key]
+            x = np.transpose(x)
+            bg, end = self._random_chunk(x.shape[-1])
+            inputs.append(x[:, bg: end])
+            labels.append(self.utt_info[index][0])
+        return inputs, labels
+
+    def _select_by_utt(self, index):
+        if self.scp_reader is None or len(self.utt_info) == 0:
+            return {}
+        index = index % (len(self.utt_info))
+        key = self.utt_info[index][0]
+        x = self.scp_reader[key]
+        x = np.transpose(x)
+        bg, end = self._random_chunk(x.shape[-1])
+
+        y = self.utt_info[index][1]
+
+        return [x[:, bg: end]], [y]
+
+    def __getitem__(self, index):
+        if self._is_select_by_speaker:
+            return self._select_by_speaker(index)
+        else:
+            return self._select_by_utt(index)
+
+    def __len__(self):
+        return len(self.utt_info) * self.repeat
+
+    def __iter__(self):
+        self._start = 0
+        return self
+
+    def __next__(self):
+        if self._start < len(self):
+            ret = self[self._start]
+            self._start += 1
+            return ret
+        else:
+            raise StopIteration
+
+return KaldiArkDataset
diff --git a/paddlespeech/vector/utils/data_utils.py b/paddlespeech/vector/utils/data_utils.py
new file mode 100644
index 00000000..4a33a795
--- /dev/null
+++ b/paddlespeech/vector/utils/data_utils.py
@@ -0,0 +1,125 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+data utilities
+"""
+import os
+import sys
+import numpy
+import paddle
+
+
+def pad_right_to(array, target_shape, mode="constant", value=0):
+    """
+    This function takes a numpy array of arbitrary shape and pads it to target
+    shape by appending values on the right.
+
+    Args:
+        array: input numpy array. Input array whose dimension we need to pad.
+    target_shape : (list, tuple). Target shape we want for the target array its len must be equal to array.ndim
+    mode : str. Pad mode, please refer to numpy.pad documentation.
+    value : float. Pad value, please refer to numpy.pad documentation.
+
+    Returns:
+        array: numpy.array. Padded array.
+        valid_vals : list. List containing proportion for each dimension of original, non-padded values.
+    """
+    assert len(target_shape) == array.ndim
+    pads = []  # this contains the abs length of the padding for each dimension.
+    valid_vals = []  # thic contains the relative lengths for each dimension.
+    i = 0 # iterating over target_shape ndims
+    while i < len(target_shape):
+        assert (
+            target_shape[i] >= array.shape[i]
+        ), "Target shape must be >= original shape for every dim"
+        pads.append([0, target_shape[i] - array.shape[i]])
+        valid_vals.append(array.shape[i] / target_shape[i])
+        i += 1
+
+    array = numpy.pad(array, pads, mode=mode, constant_values=value)
+
+    return array, valid_vals
+
+
+def batch_pad_right(arrays, mode="constant", value=0):
+    """Given a list of numpy arrays it batches them together by padding to the right
+    on each dimension in order to get same length for all.
+
+    Args:
+        arrays : list. List of array we wish to pad together.
+        mode : str. Padding mode see numpy.pad documentation.
+        value : float. Padding value see numpy.pad documentation.
+
+    Returns:
+        array : numpy.array. Padded array.
+        valid_vals : list. List containing proportion for each dimension of original, non-padded values.
+    """
+
+    if not len(arrays):
+        raise IndexError("arrays list must not be empty")
+
+    if len(arrays) == 1:
+        # if there is only one array in the batch we simply unsqueeze it.
+        return numpy.expand_dims(arrays[0], axis=0), numpy.array([1.0])
+
+    if not (
+        any(
+            [arrays[i].ndim == arrays[0].ndim for i in range(1, len(arrays))]
+        )
+    ):
+        raise IndexError("All arrays must have same number of dimensions")
+
+    # FIXME we limit the support here: we allow padding of only the last dimension
+    # need to remove this when feat extraction is updated to handle multichannel.
+    max_shape = []
+    for dim in range(arrays[0].ndim):
+        if dim != (arrays[0].ndim - 1):
+            if not all(
+                [x.shape[dim] == arrays[0].shape[dim] for x in arrays[1:]]
+            ):
+                raise EnvironmentError(
+                    "arrays should have same dimensions except for last one"
+                )
+        max_shape.append(max([x.shape[dim] for x in arrays]))
+
+    batched = []
+    valid = []
+    for t in arrays:
+        # for each array we apply pad_right_to
+        padded, valid_percent = pad_right_to(
+            t, max_shape, mode=mode, value=value
+        )
+        batched.append(padded)
+        valid.append(valid_percent[-1])
+
+    batched = numpy.stack(batched)
+
+    return batched, numpy.array(valid)
+
+
+def length_to_mask(length, max_len=None, dtype=None):
+    """Creates a binary mask for each sequence.
+    """
+    assert len(length.shape) == 1
+
+    if max_len is None:
+        max_len = paddle.cast(paddle.max(length), dtype="int64")  # using arange to generate mask
+    mask = paddle.arange(max_len, dtype=length.dtype).expand([paddle.shape(length)[0], max_len]) < length.unsqueeze(1)
+
+    if dtype is None:
+        dtype = length.dtype
+
+    mask = paddle.cast(mask, dtype=dtype)
+    return mask
diff --git a/paddlespeech/vector/utils/utils.py b/paddlespeech/vector/utils/utils.py
new file mode 100644
index 00000000..c46e42c2
--- /dev/null
+++ b/paddlespeech/vector/utils/utils.py
@@ -0,0 +1,132 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+utilities
+"""
+import os
+import sys
+import paddle
+import numpy as np
+
+from sidt import _logger as log
+
+
+def exit_if_not_exist(in_path):
+    """
+    Check the existence of a file or directory, if not exit, exit the program.
+
+    Args:
+        in_path: input dicrector
+    """
+    if not is_exist(in_path):
+        sys.exit(-1)
+
+
+def is_exist(in_path):
+    """
+    Check the existence of a file or directory
+
+    Args:
+        in_path: input dicrector
+
+    Returns:
+        True or False
+    """
+    if not os.path.exists(in_path):
+        log.error("No such file or directory: %s" % (in_path))
+        return False
+
+    return True
+
+
+def get_latest_file(target_dir):
+    """
+    Get the latest file in target directory
+
+    Args:
+        target_dir: target directory
+
+    Returns:
+        latest_file: a string or None
+    """
+    items = os.listdir(target_dir)
+    items.sort(key=lambda fn: os.path.getmtime(os.path.join(target_dir, fn)) \
+               if not os.path.isdir(os.path.join(target_dir, fn)) else 0)
+    latest_file = None if not items else os.path.join(target_dir, items[-1])
+    return latest_file
+
+
+def avg_models(models):
+    """
+    merge multiple models
+    """
+    checkpoint_dict = paddle.load(models[0])
+    final_state_dict = checkpoint_dict
+
+    if len(models) > 1:
+        for model in models[1:]:
+            checkpoint_dict = paddle.load(model)
+            for k, v in checkpoint_dict.items():
+                final_state_dict[k] += v
+        for k in final_state_dict.keys():
+            final_state_dict[k] /= float(len(models))
+            if np.any(np.isnan(final_state_dict[k])):
+                print("Nan in %s" % (k))
+
+    return final_state_dict
+
+def Q_from_tokens(token_num):
+    """
+    get prior model, data from uniform, would support others(guassian) in future
+    """
+    freq = [1] * token_num
+    Q = paddle.to_tensor(freq, dtype = 'float64')
+    return Q / Q.sum()
+
+
+def read_map_file(map_file, key_func=None, value_func=None, values_func=None):
+    """ Read map file. First colume is key, the rest columes are values.
+
+    Args:
+        map_file: map file
+        key_func: convert function for key
+        value_func: convert function for each value
+        values_func: convert function for values
+
+    Returns:
+        dict: key 2 value
+        dict: value 2 key
+    """
+    if not is_exist(map_file):
+        sys.exit(0)
+
+    key2val = {}
+    val2key = {}
+    with open(map_file, 'r') as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            items = line.split()
+            assert len(items) >= 2
+            key = items[0] if not key_func else key_func(items[0])
+            values = items[1:] if not value_func else [value_func(item) for item in items[1:]]
+            if values_func:
+                values = values_func(values)
+            key2val[key] = values
+            for value in values:
+                val2key[value] = key
+
+    return key2val, val2key

From d7da629302d40e7f8b1e2d488e40a369b556acfe Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sat, 26 Feb 2022 15:27:51 +0800
Subject: [PATCH 004/126] add kaldi feats egs dataset

---
 paddlespeech/vector/__init__.py             |  31 +++++
 paddlespeech/vector/datasets/ark_dataset.py | 142 ++++++++++++++++++++
 paddlespeech/vector/datasets/egs_dataset.py |  91 +++++++++++++
 paddlespeech/vector/utils/data_utils.py     |   0
 paddlespeech/vector/utils/utils.py          |   2 +-
 5 files changed, 265 insertions(+), 1 deletion(-)
 create mode 100755 paddlespeech/vector/datasets/ark_dataset.py
 create mode 100644 paddlespeech/vector/datasets/egs_dataset.py
 mode change 100644 => 100755 paddlespeech/vector/utils/data_utils.py
 mode change 100644 => 100755 paddlespeech/vector/utils/utils.py

diff --git a/paddlespeech/vector/__init__.py b/paddlespeech/vector/__init__.py
index 185a92b8..2a3588ec 100644
--- a/paddlespeech/vector/__init__.py
+++ b/paddlespeech/vector/__init__.py
@@ -11,3 +11,34 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+"""
+__init__ file for sidt package.
+"""
+
+import logging as sidt_logging
+import colorlog
+
+LOG_COLOR_CONFIG = {
+    'DEBUG': 'white',
+    'INFO': 'white',
+    'WARNING': 'yellow',
+    'ERROR': 'red',
+    'CRITICAL': 'purple',
+}
+
+# 设置全局的logger
+colored_formatter = colorlog.ColoredFormatter(
+    '%(log_color)s [%(levelname)s] [%(asctime)s] [%(filename)s:%(lineno)d] - %(message)s',
+    datefmt="%Y-%m-%d %H:%M:%S",
+    log_colors=LOG_COLOR_CONFIG)  # 日志输出格式
+_logger = sidt_logging.getLogger("sidt")
+handler = colorlog.StreamHandler()
+handler.setLevel(sidt_logging.INFO)
+handler.setFormatter(colored_formatter)
+_logger.addHandler(handler)
+_logger.setLevel(sidt_logging.INFO)
+
+from .trainer.trainer import Trainer
+from .dataset.ark_dataset import create_kaldi_ark_dataset
+from .dataset.egs_dataset import create_kaldi_egs_dataset
diff --git a/paddlespeech/vector/datasets/ark_dataset.py b/paddlespeech/vector/datasets/ark_dataset.py
new file mode 100755
index 00000000..7a00e7ba
--- /dev/null
+++ b/paddlespeech/vector/datasets/ark_dataset.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import random
+import numpy as np
+import kaldi_python_io as k_io
+from paddle.io import Dataset
+from paddlespeech.vector.utils.data_utils import batch_pad_right
+import paddlespeech.vector.utils as utils
+from paddlespeech.vector.utils.utils import read_map_file
+from paddlespeech.vector import _logger as log
+
+def ark_collate_fn(batch):
+    """
+    Custom collate function] for kaldi feats dataset
+
+    Args:
+        min_chunk_size: min chunk size of a utterance
+        max_chunk_size: max chunk size of a utterance
+
+    Returns:
+        ark_collate_fn: collate funtion for dataloader
+    """
+
+    data = []
+    target = []
+    for items in batch:
+        for x, y in zip(items[0], items[1]):
+            data.append(np.array(x))
+            target.append(y)
+
+    data, lengths = batch_pad_right(data)
+    return np.array(data, dtype=np.float32), \
+           np.array(lengths, dtype=np.float32), \
+           np.array(target, dtype=np.long).reshape((len(target), 1))
+
+
+class KaldiArkDataset(Dataset):
+    """
+    Dataset used to load kaldi ark/scp files.
+    """
+    def __init__(self, scp_file, label2utt, min_item_size=1,
+                 max_item_size=1, repeat=50, min_chunk_size=200,
+                 max_chunk_size=400, select_by_speaker=True):
+        self.scp_file = scp_file
+        self.scp_reader = None
+        self.repeat = repeat
+        self.min_item_size = min_item_size
+        self.max_item_size = max_item_size
+        self.min_chunk_size = min_chunk_size
+        self.max_chunk_size = max_chunk_size
+        self._collate_fn = ark_collate_fn
+        self._is_select_by_speaker = select_by_speaker
+        if utils.is_exist(self.scp_file):
+            self.scp_reader = k_io.ScriptReader(self.scp_file)
+
+        label2utts, utt2label = read_map_file(label2utt, key_func=int)
+        self.utt_info = list(label2utts.items()) if self._is_select_by_speaker else list(utt2label.items())
+
+    @property
+    def collate_fn(self):
+        """
+        Return a collate funtion.
+        """
+        return self._collate_fn
+
+    def _random_chunk(self, length):
+        chunk_size = random.randint(self.min_chunk_size, self.max_chunk_size)
+        if chunk_size >= length:
+            return 0, length
+        start = random.randint(0, length - chunk_size)
+        end = start + chunk_size
+
+        return start, end
+
+    def _select_by_speaker(self, index):
+        if self.scp_reader is None or not self.utt_info:
+            return []
+        index = index % (len(self.utt_info))
+        inputs = []
+        labels = []
+        item_size = random.randint(self.min_item_size, self.max_item_size)
+        for loop_idx in range(item_size):
+            try:
+                utt_index = random.randint(0, len(self.utt_info[index][1])) \
+                        % len(self.utt_info[index][1])
+                key = self.utt_info[index][1][utt_index]
+            except:
+                print(index, utt_index, len(self.utt_info[index][1]))
+                sys.exit(-1)
+            x = self.scp_reader[key]
+            x = np.transpose(x)
+            bg, end = self._random_chunk(x.shape[-1])
+            inputs.append(x[:, bg: end])
+            labels.append(self.utt_info[index][0])
+        return inputs, labels
+
+    def _select_by_utt(self, index):
+        if self.scp_reader is None or len(self.utt_info) == 0:
+            return {}
+        index = index % (len(self.utt_info))
+        key = self.utt_info[index][0]
+        x = self.scp_reader[key]
+        x = np.transpose(x)
+        bg, end = self._random_chunk(x.shape[-1])
+
+        y = self.utt_info[index][1]
+
+        return [x[:, bg: end]], [y]
+
+    def __getitem__(self, index):
+        if self._is_select_by_speaker:
+            return self._select_by_speaker(index)
+        else:
+            return self._select_by_utt(index)
+
+    def __len__(self):
+        return len(self.utt_info) * self.repeat
+
+    def __iter__(self):
+        self._start = 0
+        return self
+
+    def __next__(self):
+        if self._start < len(self):
+            ret = self[self._start]
+            self._start += 1
+            return ret
+        else:
+            raise StopIteration
diff --git a/paddlespeech/vector/datasets/egs_dataset.py b/paddlespeech/vector/datasets/egs_dataset.py
new file mode 100644
index 00000000..53130d5f
--- /dev/null
+++ b/paddlespeech/vector/datasets/egs_dataset.py
@@ -0,0 +1,91 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Load nnet3 training egs which generated by kaldi
+"""
+
+import random
+import numpy as np
+import kaldi_python_io as k_io
+from paddle.io import Dataset
+import paddlespeech.vector.utils.utils as utils
+from paddlespeech.vector import _logger as log
+class KaldiEgsDataset(Dataset):
+    """
+    Dataset used to load kaldi nnet3 egs files.
+    """
+    def __init__(self, egs_list_file, egs_idx, transforms=None):
+        self.scp_reader = None
+        self.subset_idx = egs_idx - 1
+        self.transforms = transforms
+        if not utils.is_exist(egs_list_file):
+            return
+
+        self.egs_files = []
+        with open(egs_list_file, 'r') as in_fh:
+            for line in in_fh:
+                if line.strip():
+                    self.egs_files.append(line.strip())
+
+        self.next_subset()
+
+    def next_subset(self, target_index=None, delta_index=None):
+        """
+        Use next specific subset
+
+        Args:
+            target_index: target egs index
+            delta_index: incremental value of egs index
+        """
+        if self.egs_files:
+            if target_index:
+                self.subset_idx = target_index
+            else:
+                delta_index = delta_index if delta_index else 1
+                self.subset_idx += delta_index
+            log.info("egs dataset subset index: %d" % (self.subset_idx))
+            egs_file = self.egs_files[self.subset_idx % len(self.egs_files)]
+            if utils.is_exist(egs_file):
+                self.scp_reader = k_io.Nnet3EgsScriptReader(egs_file)
+            else:
+                log.warning("No such file or directory: %s" % (egs_file))
+
+    def __getitem__(self, index):
+        if self.scp_reader is None:
+            return {}
+        index %= len(self)
+        in_dict, out_dict = self.scp_reader[index]
+        x = np.array(in_dict['matrix'])
+        x = np.transpose(x)
+        y = np.array(out_dict['matrix'][0][0][0], dtype=np.int).reshape((1,))
+        if self.transforms is not None:
+            idx = random.randint(0, len(self.transforms) - 1)
+            x = self.transforms[idx](x)
+        return x, y
+
+    def __len__(self):
+        return len(self.scp_reader)
+
+    def __iter__(self):
+        self._start = 0
+        return self
+
+    def __next__(self):
+        if self._start < len(self):
+            ret = self[self._start]
+            self._start += 1
+            return ret
+        else:
+            raise StopIteration
\ No newline at end of file
diff --git a/paddlespeech/vector/utils/data_utils.py b/paddlespeech/vector/utils/data_utils.py
old mode 100644
new mode 100755
diff --git a/paddlespeech/vector/utils/utils.py b/paddlespeech/vector/utils/utils.py
old mode 100644
new mode 100755
index c46e42c2..a28cb526
--- a/paddlespeech/vector/utils/utils.py
+++ b/paddlespeech/vector/utils/utils.py
@@ -20,7 +20,7 @@ import sys
 import paddle
 import numpy as np
 
-from sidt import _logger as log
+from paddlespeech.vector import _logger as log
 
 
 def exit_if_not_exist(in_path):

From 70d3b01c0dc76a70dfe2a93ab6184fb1a69757d9 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sat, 26 Feb 2022 17:35:03 +0800
Subject: [PATCH 005/126] remove invalid code

---
 paddlespeech/vector/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/paddlespeech/vector/__init__.py b/paddlespeech/vector/__init__.py
index 2a3588ec..5c846193 100644
--- a/paddlespeech/vector/__init__.py
+++ b/paddlespeech/vector/__init__.py
@@ -39,6 +39,3 @@ handler.setFormatter(colored_formatter)
 _logger.addHandler(handler)
 _logger.setLevel(sidt_logging.INFO)
 
-from .trainer.trainer import Trainer
-from .dataset.ark_dataset import create_kaldi_ark_dataset
-from .dataset.egs_dataset import create_kaldi_egs_dataset

From 7ef60ebae2e4c14c9f2bd1954c420987a5d8369a Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 2 Mar 2022 20:36:39 +0800
Subject: [PATCH 006/126] add voxceleb1 data prepare

---
 dataset/voxceleb/voxceleb1.py        | 463 +++++++++++++++++----------
 examples/voxceleb/sv0/local/train.py |  31 ++
 examples/voxceleb/sv0/path.sh        |  11 +
 examples/voxceleb/sv0/run.sh         |  10 +
 4 files changed, 342 insertions(+), 173 deletions(-)
 create mode 100644 examples/voxceleb/sv0/local/train.py
 create mode 100755 examples/voxceleb/sv0/path.sh
 create mode 100755 examples/voxceleb/sv0/run.sh

diff --git a/dataset/voxceleb/voxceleb1.py b/dataset/voxceleb/voxceleb1.py
index e50c91bc..0c9c68dc 100644
--- a/dataset/voxceleb/voxceleb1.py
+++ b/dataset/voxceleb/voxceleb1.py
@@ -11,182 +11,299 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Prepare VoxCeleb1 dataset
-
-create manifest files.
-Manifest file is a json-format file with each line containing the
-meta data (i.e. audio filepath, transcript and audio duration)
-of each audio file in the data set.
-
-researchers should download the voxceleb1 dataset yourselves
-through google form to get the username & password and unpack the data
-"""
-import argparse
-import codecs
+
+import collections
+import csv
 import glob
-import json
 import os
-import subprocess
-from pathlib import Path
+import random
+from typing import Dict, List, Tuple
 
-import soundfile
+from paddle.io import Dataset
+from tqdm import tqdm
+from pathos.multiprocessing import Pool
 
-from utils.utility import check_md5sum
+from paddleaudio.backends import load as load_audio
+from paddleaudio.utils import DATA_HOME, decompress, download_and_decompress
+from paddleaudio.datasets.dataset import feat_funcs
+from utils.utility import unpack
 from utils.utility import download
-from utils.utility import unzip
-
-# all the data will be download in the current data/voxceleb directory default
-DATA_HOME = os.path.expanduser('.')
-
-# if you use the http://www.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a/ as the download base url
-# you need to get the username & password via the google form
-
-# if you use the https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a as the download base url,
-# you need use --no-check-certificate to connect the target download url 
-
-BASE_URL = "https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a"
-
-# dev data
-DEV_LIST = {
-    "vox1_dev_wav_partaa": "e395d020928bc15670b570a21695ed96",
-    "vox1_dev_wav_partab": "bbfaaccefab65d82b21903e81a8a8020",
-    "vox1_dev_wav_partac": "017d579a2a96a077f40042ec33e51512",
-    "vox1_dev_wav_partad": "7bb1e9f70fddc7a678fa998ea8b3ba19",
-}
-DEV_TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f532ba230b"
-
-# test data
-TEST_LIST = {"vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102"}
-TEST_TARGET_DATA = "vox1_test_wav.zip vox1_test_wav.zip 185fdc63c3c739954633d50379a3d102"
-
-# kaldi trial
-# this trial file is organized by kaldi according the official file,
-# which is a little different with the official trial veri_test2.txt
-KALDI_BASE_URL = "http://www.openslr.org/resources/49/"
-TRIAL_LIST = {"voxceleb1_test_v2.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7"}
-TRIAL_TARGET_DATA = "voxceleb1_test_v2.txt voxceleb1_test_v2.txt 29fc7cc1c5d59f0816dc15d6e8be60f7"
-
-parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument(
-    "--target_dir",
-    default=DATA_HOME + "/voxceleb1/",
-    type=str,
-    help="Directory to save the voxceleb1 dataset. (default: %(default)s)")
-parser.add_argument(
-    "--manifest_prefix",
-    default="manifest",
-    type=str,
-    help="Filepath prefix for output manifests. (default: %(default)s)")
-
-args = parser.parse_args()
-
-
-def create_manifest(data_dir, manifest_path_prefix):
-    print("Creating manifest %s ..." % manifest_path_prefix)
-    json_lines = []
-    data_path = os.path.join(data_dir, "wav", "**", "*.wav")
-    total_sec = 0.0
-    total_text = 0.0
-    total_num = 0
-    speakers = set()
-    for audio_path in glob.glob(data_path, recursive=True):
-        audio_id = "-".join(audio_path.split("/")[-3:])
-        utt2spk = audio_path.split("/")[-3]
-        duration = soundfile.info(audio_path).duration
-        text = ""
-        json_lines.append(
-            json.dumps(
-                {
-                    "utt": audio_id,
-                    "utt2spk": str(utt2spk),
-                    "feat": audio_path,
-                    "feat_shape": (duration, ),
-                    "text": text  # compatible with asr data format
-                },
-                ensure_ascii=False))
-
-        total_sec += duration
-        total_text += len(text)
-        total_num += 1
-        speakers.add(utt2spk)
-
-    # data_dir_name refer to dev or test
-    # voxceleb1 is given explicit in the path
-    data_dir_name = Path(data_dir).name
-    manifest_path_prefix = manifest_path_prefix + "." + data_dir_name
-    with codecs.open(manifest_path_prefix, 'w', encoding='utf-8') as f:
-        for line in json_lines:
-            f.write(line + "\n")
-
-    manifest_dir = os.path.dirname(manifest_path_prefix)
-    meta_path = os.path.join(manifest_dir, "voxceleb1." +
-                             data_dir_name) + ".meta"
-    with codecs.open(meta_path, 'w', encoding='utf-8') as f:
-        print(f"{total_num} utts", file=f)
-        print(f"{len(speakers)} speakers", file=f)
-        print(f"{total_sec / (60 * 60)} h", file=f)
-        print(f"{total_text} text", file=f)
-        print(f"{total_text / total_sec} text/sec", file=f)
-        print(f"{total_sec / total_num} sec/utt", file=f)
-
-
-def prepare_dataset(base_url, data_list, target_dir, manifest_path,
-                    target_data):
-    if not os.path.exists(target_dir):
-        os.mkdir(target_dir)
-
-    # wav directory already exists, it need do nothing
-    if not os.path.exists(os.path.join(target_dir, "wav")):
-        # download all dataset part
-        for zip_part in data_list.keys():
-            download_url = " --no-check-certificate " + base_url + "/" + zip_part
-            download(
-                url=download_url,
-                md5sum=data_list[zip_part],
-                target_dir=target_dir)
-
-        # pack the all part to target zip file
-        all_target_part, target_name, target_md5sum = target_data.split()
-        target_name = os.path.join(target_dir, target_name)
-        if not os.path.exists(target_name):
-            pack_part_cmd = "cat {}/{} > {}".format(target_dir, all_target_part,
-                                                    target_name)
-            subprocess.call(pack_part_cmd, shell=True)
-
-        # check the target zip file md5sum
-        if not check_md5sum(target_name, target_md5sum):
-            raise RuntimeError("{} MD5 checkssum failed".format(target_name))
-        else:
-            print("Check {} md5sum successfully".format(target_name))
-
-        # unzip the all zip file
-        if target_name.endswith(".zip"):
-            unzip(target_name, target_dir)
-
-    # create the manifest file
-    create_manifest(data_dir=target_dir, manifest_path_prefix=manifest_path)
-
-
-def main():
-    if args.target_dir.startswith('~'):
-        args.target_dir = os.path.expanduser(args.target_dir)
 
-    prepare_dataset(
-        base_url=BASE_URL,
-        data_list=DEV_LIST,
-        target_dir=os.path.join(args.target_dir, "dev"),
-        manifest_path=args.manifest_prefix,
-        target_data=DEV_TARGET_DATA)
-
-    prepare_dataset(
-        base_url=BASE_URL,
-        data_list=TEST_LIST,
-        target_dir=os.path.join(args.target_dir, "test"),
-        manifest_path=args.manifest_prefix,
-        target_data=TEST_TARGET_DATA)
-
-    print("Manifest prepare done!")
-
-
-if __name__ == '__main__':
-    main()
+__all__ = ['VoxCeleb1']
+
+
+class VoxCeleb1(Dataset):
+    source_url = 'https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a/'
+    archieves_audio_dev = [
+        {
+            'url': source_url + 'vox1_dev_wav_partaa',
+            'md5': 'e395d020928bc15670b570a21695ed96',
+        },
+        {
+            'url': source_url + 'vox1_dev_wav_partab',
+            'md5': 'bbfaaccefab65d82b21903e81a8a8020',
+        },
+        {
+            'url': source_url + 'vox1_dev_wav_partac',
+            'md5': '017d579a2a96a077f40042ec33e51512',
+        },
+        {
+            'url': source_url + 'vox1_dev_wav_partad',
+            'md5': '7bb1e9f70fddc7a678fa998ea8b3ba19',
+        },
+    ]
+    archieves_audio_test = [
+        {
+            'url': source_url + 'vox1_test_wav.zip',
+            'md5': '185fdc63c3c739954633d50379a3d102',
+        },
+    ]
+    archieves_meta = [
+        {
+            'url': 'https://www.robots.ox.ac.uk/~vgg/data/voxceleb/meta/veri_test2.txt',
+            'md5': 'b73110731c9223c1461fe49cb48dddfc',
+        },
+    ]
+
+
+    num_speakers = 1211  # 1211 vox1, 5994 vox2, 7205 vox1+2, test speakers: 41
+    sample_rate = 16000
+    meta_info = collections.namedtuple(
+        'META_INFO', ('id', 'duration', 'wav', 'start', 'stop', 'spk_id'))
+    base_path = os.path.join(DATA_HOME, 'vox1')
+    wav_path = os.path.join(base_path, 'wav')
+    subsets = ['train', 'dev', 'enrol', 'test']
+
+    def __init__(self,
+                 subset: str = 'train',
+                 feat_type: str = 'raw',
+                 random_chunk: bool = True,
+                 chunk_duration: float = 3.0,       # seconds
+                 split_ratio: float = 0.9,          # train split ratio
+                 seed: int = 0,
+                 target_dir: str = None,
+                 **kwargs):
+
+        assert subset in self.subsets, \
+            'Dataset subset must be one in {}, but got {}'.format(self.subsets, subset)
+
+        self.subset = subset
+        self.spk_id2label = {}
+        self.feat_type = feat_type
+        self.feat_config = kwargs
+        self.random_chunk = random_chunk
+        self.chunk_duration = chunk_duration
+        self.split_ratio = split_ratio
+        self.target_dir = target_dir if target_dir else self.base_path
+        self.csv_path = os.path.join(target_dir, 'csv') if target_dir else os.path.join(self.base_path, 'csv')
+        self.meta_path = os.path.join(target_dir, 'meta') if target_dir else os.path.join(base_path, 'meta')
+        self.veri_test_file = os.path.join(self.meta_path, 'veri_test2.txt')
+        # self._data = self._get_data()[:1000]  # KP: Small dataset test.
+        self._data = self._get_data()
+        super(VoxCeleb1, self).__init__()
+
+        # Set up a seed to reproduce training or predicting result.
+        # random.seed(seed)
+
+    def _get_data(self):
+        # Download audio files.
+        # We need the users to decompress all vox1/dev/wav and vox1/test/wav/ to vox1/wav/ dir
+        # so, we check the vox1/wav dir status
+        print("wav base path: {}".format(self.wav_path))
+        if not os.path.isdir(self.wav_path):
+            print("start to download the voxceleb1 dataset")
+            download_and_decompress(    # multi-zip parts concatenate to vox1_dev_wav.zip
+                self.archieves_audio_dev, self.base_path, decompress=False)
+            download_and_decompress(    # download the vox1_test_wav.zip and unzip
+                self.archieves_audio_test, self.base_path, decompress=True)
+
+            # Download all parts and concatenate the files into one zip file.
+            dev_zipfile = os.path.join(self.base_path, 'vox1_dev_wav.zip')
+            print(f'Concatenating all parts to: {dev_zipfile}')
+            os.system(
+                f'cat {os.path.join(self.base_path, "vox1_dev_wav_parta*")} > {dev_zipfile}'
+            )
+
+            # Extract all audio files of dev and test set.
+            decompress(dev_zipfile, self.base_path)
+
+        # Download meta files.
+        if not os.path.isdir(self.meta_path):
+            download_and_decompress(
+                self.archieves_meta, self.meta_path, decompress=False)
+        
+        # Data preparation.
+        if not os.path.isdir(self.csv_path):
+            os.makedirs(self.csv_path)
+            self.prepare_data()
+
+        data = []
+        with open(os.path.join(self.csv_path, f'{self.subset}.csv'), 'r') as rf:
+            for line in rf.readlines()[1:]:
+                audio_id, duration, wav, start, stop, spk_id = line.strip(
+                ).split(',')
+                data.append(
+                    self.meta_info(audio_id, float(duration), wav, int(start),
+                                   int(stop), spk_id))
+
+        with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'r') as f:
+            for line in f.readlines():
+                spk_id, label = line.strip().split(' ')
+                self.spk_id2label[spk_id] = int(label)
+
+        return data
+
+    def _convert_to_record(self, idx: int):
+        sample = self._data[idx]
+
+        record = {}
+        # To show all fields in a namedtuple: `type(sample)._fields`
+        for field in type(sample)._fields:
+            record[field] = getattr(sample, field)
+
+        waveform, sr = load_audio(record['wav'])
+
+        # random select a chunk audio samples from the audio
+        if self.random_chunk:
+            num_wav_samples = waveform.shape[0]
+            num_chunk_samples = int(self.chunk_duration * sr)
+            start = random.randint(0, num_wav_samples - num_chunk_samples - 1)
+            stop = start + num_chunk_samples
+        else:
+            start = record['start']
+            stop = record['stop']
+
+        waveform = waveform[start:stop]
+
+        assert self.feat_type in feat_funcs.keys(), \
+            f"Unknown feat_type: {self.feat_type}, it must be one in {list(feat_funcs.keys())}"
+        feat_func = feat_funcs[self.feat_type]
+        feat = feat_func(
+            waveform, sr=sr, **self.feat_config) if feat_func else waveform
+
+        record.update({'feat': feat})
+        if self.subset in ['train',
+                           'dev']:  # Labels are available in train and dev.
+            record.update({'label': self.spk_id2label[record['spk_id']]})
+
+        return record
+
+    @staticmethod
+    def _get_chunks(seg_dur, audio_id, audio_duration):
+        num_chunks = int(audio_duration / seg_dur)  # all in milliseconds
+
+        chunk_lst = [
+            audio_id + "_" + str(i * seg_dur) + "_" + str(i * seg_dur + seg_dur)
+            for i in range(num_chunks)
+        ]
+        return chunk_lst
+
+    def _get_audio_info(self, wav_file: str,
+                        split_chunks: bool) -> List[List[str]]:
+        waveform, sr = load_audio(wav_file)
+        spk_id, sess_id, utt_id = wav_file.split("/")[-3:]
+        audio_id = '-'.join([spk_id, sess_id, utt_id.split(".")[0]])
+        audio_duration = waveform.shape[0] / sr
+
+        ret = []
+        if split_chunks:  # Split into pieces of self.chunk_duration seconds.
+            uniq_chunks_list = self._get_chunks(self.chunk_duration, audio_id,
+                                                audio_duration)
+
+            for chunk in uniq_chunks_list:
+                s, e = chunk.split("_")[-2:]  # Timestamps of start and end
+                start_sample = int(float(s) * sr)
+                end_sample = int(float(e) * sr)
+                # id, duration, wav, start, stop, spk_id
+                ret.append([
+                    chunk, audio_duration, wav_file, start_sample, end_sample,
+                    spk_id
+                ])
+        else:  # Keep whole audio.
+            ret.append([
+                audio_id, audio_duration, wav_file, 0, waveform.shape[0], spk_id
+            ])
+        return ret
+
+    def generate_csv(self,
+                     wav_files: List[str],
+                     output_file: str,
+                     split_chunks: bool = True):
+        print(f'Generating csv: {output_file}')
+        header = ["id", "duration", "wav", "start", "stop", "spk_id"]
+
+        with Pool(64) as p:
+            infos = list(
+                tqdm(
+                    p.imap(lambda x: self._get_audio_info(x, split_chunks), wav_files), total=len(wav_files)))
+
+        csv_lines = []
+        for info in infos:
+            csv_lines.extend(info)
+
+        with open(output_file, mode="w") as csv_f:
+            csv_writer = csv.writer(
+                csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
+            csv_writer.writerow(header)
+            for line in csv_lines:
+                csv_writer.writerow(line)
+
+    def prepare_data(self):
+        # Audio of speakers in veri_test_file should not be included in training set.
+        print("start to prepare the data csv file")
+        enrol_files = set()
+        test_files = set()
+        # get the enroll and test audio file path
+        with open(self.veri_test_file, 'r') as f:
+            for line in f.readlines():
+                _, enrol_file, test_file = line.strip().split(' ')
+                enrol_files.add(os.path.join(self.wav_path, enrol_file))
+                test_files.add(os.path.join(self.wav_path, test_file))
+            enrol_files = sorted(enrol_files)
+            test_files = sorted(test_files)
+
+        # get the enroll and test speakers
+        test_spks = set()
+        for file in (enrol_files + test_files):
+            spk = file.split('/wav/')[1].split('/')[0]
+            test_spks.add(spk)
+
+        # get all the train and dev audios file path
+        audio_files = []
+        speakers = set()
+        for path in [self.wav_path]:
+            for file in glob.glob(os.path.join(path, "**", "*.wav"), recursive=True):
+                spk = file.split('/wav/')[1].split('/')[0]
+                if spk in test_spks:
+                    continue
+                speakers.add(spk)
+                audio_files.append(file)
+
+        print("start to generate the {}".format(os.path.join(self.meta_path, 'spk_id2label.txt')))
+        # encode the train and dev speakers label to spk_id2label.txt
+        with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'w') as f:
+            for label, spk_id in enumerate(sorted(speakers)):  # 1211 vox1, 5994 vox2, 7205 vox1+2
+                f.write(f'{spk_id} {label}\n')
+
+        audio_files = sorted(audio_files)
+        random.shuffle(audio_files)
+        split_idx = int(self.split_ratio * len(audio_files))
+        # split_ratio to train
+        train_files, dev_files = audio_files[:split_idx], audio_files[split_idx:]
+
+        self.generate_csv(train_files, 
+                          os.path.join(self.csv_path, 'train.csv'))
+        self.generate_csv(dev_files, 
+                          os.path.join(self.csv_path, 'dev.csv'))
+        self.generate_csv(enrol_files,
+                          os.path.join(self.csv_path, 'enrol.csv'),
+                          split_chunks=False)
+        self.generate_csv(test_files,
+                          os.path.join(self.csv_path, 'test.csv'),
+                          split_chunks=False)
+
+    def __getitem__(self, idx):
+        return self._convert_to_record(idx)
+
+    def __len__(self):
+        return len(self._data)
diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
new file mode 100644
index 00000000..e8619cca
--- /dev/null
+++ b/examples/voxceleb/sv0/local/train.py
@@ -0,0 +1,31 @@
+import argparse
+import paddle
+from dataset.voxceleb.voxceleb1 import VoxCeleb1
+
+
+def main(args):
+    paddle.set_device(args.device)
+
+    # stage1: we must call the paddle.distributed.init_parallel_env() api at the begining
+    paddle.distributed.init_parallel_env()
+    nranks = paddle.distributed.get_world_size()
+    local_rank = paddle.distributed.get_rank()
+
+    # stage2: data prepare
+    train_ds = VoxCeleb1('train', target_dir=args.data_dir)
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--device', 
+                        choices=['cpu', 'gpu'], 
+                        default="cpu", 
+                        help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--data-dir",
+                        default="./data/",
+                        type=str,
+                        help="data directory")
+    args = parser.parse_args()
+    # yapf: enable
+
+    main(args)
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/path.sh b/examples/voxceleb/sv0/path.sh
new file mode 100755
index 00000000..38a242a4
--- /dev/null
+++ b/examples/voxceleb/sv0/path.sh
@@ -0,0 +1,11 @@
+export MAIN_ROOT=`realpath ${PWD}/../../../`
+
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
+export LC_ALL=C
+
+export PYTHONDONTWRITEBYTECODE=1
+# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
new file mode 100755
index 00000000..c24cbff4
--- /dev/null
+++ b/examples/voxceleb/sv0/run.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+. ./path.sh
+set -e
+export PPAUDIO_HOME=/home/users/xiongxinlei/exprts/v3
+dir=./data/
+mkdir -p ${dir}
+# you can set the variable DATA_HOME to specifiy the downloaded the vox1 and vox2 dataset
+/home/users/xiongxinlei/.conda/envs/xxl_base/bin/python3 \
+     local/train.py \
+     --data-dir ${dir}
\ No newline at end of file

From 0780d181d29d8470ba3579aa0d0ef9465c5ad264 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 2 Mar 2022 20:55:31 +0800
Subject: [PATCH 007/126] remove personal code test=doc

---
 dataset/voxceleb/voxceleb1.py        | 106 ++++++++++++++++-----------
 examples/voxceleb/sv0/local/train.py |  24 +++++-
 examples/voxceleb/sv0/run.sh         |   4 +-
 3 files changed, 85 insertions(+), 49 deletions(-)

diff --git a/dataset/voxceleb/voxceleb1.py b/dataset/voxceleb/voxceleb1.py
index 0c9c68dc..b2d5f5c3 100644
--- a/dataset/voxceleb/voxceleb1.py
+++ b/dataset/voxceleb/voxceleb1.py
@@ -11,23 +11,26 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import collections
 import csv
 import glob
 import os
 import random
-from typing import Dict, List, Tuple
+from typing import Dict
+from typing import List
+from typing import Tuple
 
 from paddle.io import Dataset
-from tqdm import tqdm
 from pathos.multiprocessing import Pool
+from tqdm import tqdm
 
 from paddleaudio.backends import load as load_audio
-from paddleaudio.utils import DATA_HOME, decompress, download_and_decompress
 from paddleaudio.datasets.dataset import feat_funcs
-from utils.utility import unpack
+from paddleaudio.utils import DATA_HOME
+from paddleaudio.utils import decompress
+from paddleaudio.utils import download_and_decompress
 from utils.utility import download
+from utils.utility import unpack
 
 __all__ = ['VoxCeleb1']
 
@@ -60,12 +63,13 @@ class VoxCeleb1(Dataset):
     ]
     archieves_meta = [
         {
-            'url': 'https://www.robots.ox.ac.uk/~vgg/data/voxceleb/meta/veri_test2.txt',
-            'md5': 'b73110731c9223c1461fe49cb48dddfc',
+            'url':
+            'https://www.robots.ox.ac.uk/~vgg/data/voxceleb/meta/veri_test2.txt',
+            'md5':
+            'b73110731c9223c1461fe49cb48dddfc',
         },
     ]
 
-
     num_speakers = 1211  # 1211 vox1, 5994 vox2, 7205 vox1+2, test speakers: 41
     sample_rate = 16000
     meta_info = collections.namedtuple(
@@ -74,15 +78,16 @@ class VoxCeleb1(Dataset):
     wav_path = os.path.join(base_path, 'wav')
     subsets = ['train', 'dev', 'enrol', 'test']
 
-    def __init__(self,
-                 subset: str = 'train',
-                 feat_type: str = 'raw',
-                 random_chunk: bool = True,
-                 chunk_duration: float = 3.0,       # seconds
-                 split_ratio: float = 0.9,          # train split ratio
-                 seed: int = 0,
-                 target_dir: str = None,
-                 **kwargs):
+    def __init__(
+            self,
+            subset: str='train',
+            feat_type: str='raw',
+            random_chunk: bool=True,
+            chunk_duration: float=3.0,  # seconds
+            split_ratio: float=0.9,  # train split ratio
+            seed: int=0,
+            target_dir: str=None,
+            **kwargs):
 
         assert subset in self.subsets, \
             'Dataset subset must be one in {}, but got {}'.format(self.subsets, subset)
@@ -95,8 +100,12 @@ class VoxCeleb1(Dataset):
         self.chunk_duration = chunk_duration
         self.split_ratio = split_ratio
         self.target_dir = target_dir if target_dir else self.base_path
-        self.csv_path = os.path.join(target_dir, 'csv') if target_dir else os.path.join(self.base_path, 'csv')
-        self.meta_path = os.path.join(target_dir, 'meta') if target_dir else os.path.join(base_path, 'meta')
+        self.csv_path = os.path.join(
+            target_dir, 'csv') if target_dir else os.path.join(self.base_path,
+                                                               'csv')
+        self.meta_path = os.path.join(
+            target_dir, 'meta') if target_dir else os.path.join(base_path,
+                                                                'meta')
         self.veri_test_file = os.path.join(self.meta_path, 'veri_test2.txt')
         # self._data = self._get_data()[:1000]  # KP: Small dataset test.
         self._data = self._get_data()
@@ -112,10 +121,14 @@ class VoxCeleb1(Dataset):
         print("wav base path: {}".format(self.wav_path))
         if not os.path.isdir(self.wav_path):
             print("start to download the voxceleb1 dataset")
-            download_and_decompress(    # multi-zip parts concatenate to vox1_dev_wav.zip
-                self.archieves_audio_dev, self.base_path, decompress=False)
-            download_and_decompress(    # download the vox1_test_wav.zip and unzip
-                self.archieves_audio_test, self.base_path, decompress=True)
+            download_and_decompress(  # multi-zip parts concatenate to vox1_dev_wav.zip
+                self.archieves_audio_dev,
+                self.base_path,
+                decompress=False)
+            download_and_decompress(  # download the vox1_test_wav.zip and unzip
+                self.archieves_audio_test,
+                self.base_path,
+                decompress=True)
 
             # Download all parts and concatenate the files into one zip file.
             dev_zipfile = os.path.join(self.base_path, 'vox1_dev_wav.zip')
@@ -131,7 +144,7 @@ class VoxCeleb1(Dataset):
         if not os.path.isdir(self.meta_path):
             download_and_decompress(
                 self.archieves_meta, self.meta_path, decompress=False)
-        
+
         # Data preparation.
         if not os.path.isdir(self.csv_path):
             os.makedirs(self.csv_path)
@@ -143,8 +156,9 @@ class VoxCeleb1(Dataset):
                 audio_id, duration, wav, start, stop, spk_id = line.strip(
                 ).split(',')
                 data.append(
-                    self.meta_info(audio_id, float(duration), wav, int(start),
-                                   int(stop), spk_id))
+                    self.meta_info(audio_id,
+                                   float(duration), wav,
+                                   int(start), int(stop), spk_id))
 
         with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'r') as f:
             for line in f.readlines():
@@ -228,14 +242,16 @@ class VoxCeleb1(Dataset):
     def generate_csv(self,
                      wav_files: List[str],
                      output_file: str,
-                     split_chunks: bool = True):
+                     split_chunks: bool=True):
         print(f'Generating csv: {output_file}')
         header = ["id", "duration", "wav", "start", "stop", "spk_id"]
 
         with Pool(64) as p:
             infos = list(
                 tqdm(
-                    p.imap(lambda x: self._get_audio_info(x, split_chunks), wav_files), total=len(wav_files)))
+                    p.imap(lambda x: self._get_audio_info(x, split_chunks),
+                           wav_files),
+                    total=len(wav_files)))
 
         csv_lines = []
         for info in infos:
@@ -272,35 +288,39 @@ class VoxCeleb1(Dataset):
         audio_files = []
         speakers = set()
         for path in [self.wav_path]:
-            for file in glob.glob(os.path.join(path, "**", "*.wav"), recursive=True):
+            for file in glob.glob(
+                    os.path.join(path, "**", "*.wav"), recursive=True):
                 spk = file.split('/wav/')[1].split('/')[0]
                 if spk in test_spks:
                     continue
                 speakers.add(spk)
                 audio_files.append(file)
 
-        print("start to generate the {}".format(os.path.join(self.meta_path, 'spk_id2label.txt')))
+        print("start to generate the {}".format(
+            os.path.join(self.meta_path, 'spk_id2label.txt')))
         # encode the train and dev speakers label to spk_id2label.txt
         with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'w') as f:
-            for label, spk_id in enumerate(sorted(speakers)):  # 1211 vox1, 5994 vox2, 7205 vox1+2
+            for label, spk_id in enumerate(
+                    sorted(speakers)):  # 1211 vox1, 5994 vox2, 7205 vox1+2
                 f.write(f'{spk_id} {label}\n')
 
         audio_files = sorted(audio_files)
         random.shuffle(audio_files)
         split_idx = int(self.split_ratio * len(audio_files))
         # split_ratio to train
-        train_files, dev_files = audio_files[:split_idx], audio_files[split_idx:]
-
-        self.generate_csv(train_files, 
-                          os.path.join(self.csv_path, 'train.csv'))
-        self.generate_csv(dev_files, 
-                          os.path.join(self.csv_path, 'dev.csv'))
-        self.generate_csv(enrol_files,
-                          os.path.join(self.csv_path, 'enrol.csv'),
-                          split_chunks=False)
-        self.generate_csv(test_files,
-                          os.path.join(self.csv_path, 'test.csv'),
-                          split_chunks=False)
+        train_files, dev_files = audio_files[:split_idx], audio_files[
+            split_idx:]
+
+        self.generate_csv(train_files, os.path.join(self.csv_path, 'train.csv'))
+        self.generate_csv(dev_files, os.path.join(self.csv_path, 'dev.csv'))
+        self.generate_csv(
+            enrol_files,
+            os.path.join(self.csv_path, 'enrol.csv'),
+            split_chunks=False)
+        self.generate_csv(
+            test_files,
+            os.path.join(self.csv_path, 'test.csv'),
+            split_chunks=False)
 
     def __getitem__(self, idx):
         return self._convert_to_record(idx)
diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index e8619cca..c0cb1e17 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -1,5 +1,20 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 import argparse
+
 import paddle
+
 from dataset.voxceleb.voxceleb1 import VoxCeleb1
 
 
@@ -14,12 +29,13 @@ def main(args):
     # stage2: data prepare
     train_ds = VoxCeleb1('train', target_dir=args.data_dir)
 
+
 if __name__ == "__main__":
     # yapf: disable
     parser = argparse.ArgumentParser(__doc__)
-    parser.add_argument('--device', 
-                        choices=['cpu', 'gpu'], 
-                        default="cpu", 
+    parser.add_argument('--device',
+                        choices=['cpu', 'gpu'],
+                        default="cpu",
                         help="Select which device to train model, defaults to gpu.")
     parser.add_argument("--data-dir",
                         default="./data/",
@@ -28,4 +44,4 @@ if __name__ == "__main__":
     args = parser.parse_args()
     # yapf: enable
 
-    main(args)
\ No newline at end of file
+    main(args)
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index c24cbff4..a8debfc6 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -5,6 +5,6 @@ export PPAUDIO_HOME=/home/users/xiongxinlei/exprts/v3
 dir=./data/
 mkdir -p ${dir}
 # you can set the variable DATA_HOME to specifiy the downloaded the vox1 and vox2 dataset
-/home/users/xiongxinlei/.conda/envs/xxl_base/bin/python3 \
+python3 \
      local/train.py \
-     --data-dir ${dir}
\ No newline at end of file
+     --data-dir ${dir}

From 3a943ca95b13818409efa6253b05fb2831ab2419 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 2 Mar 2022 20:59:45 +0800
Subject: [PATCH 008/126] repair the variable name bug

---
 examples/voxceleb/sv0/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index a8debfc6..a96c3827 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
 . ./path.sh
 set -e
-export PPAUDIO_HOME=/home/users/xiongxinlei/exprts/v3
+
 dir=./data/
 mkdir -p ${dir}
-# you can set the variable DATA_HOME to specifiy the downloaded the vox1 and vox2 dataset
+# you can set the variable PPAUDIO_HOME to specifiy the downloaded the vox1 and vox2 dataset
 python3 \
      local/train.py \
      --data-dir ${dir}

From dc28ebe4eecc7b0cd72b3f4991830fcf5f907b52 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 3 Mar 2022 14:55:37 +0800
Subject: [PATCH 009/126] move the csv vox format to paddleaudio, test=doc

---
 dataset/voxceleb/voxceleb1.py    | 487 +++++++++++--------------------
 paddleaudio/datasets/voxceleb.py | 329 +++++++++++++++++++++
 2 files changed, 504 insertions(+), 312 deletions(-)
 create mode 100644 paddleaudio/datasets/voxceleb.py

diff --git a/dataset/voxceleb/voxceleb1.py b/dataset/voxceleb/voxceleb1.py
index b2d5f5c3..c6fc0695 100644
--- a/dataset/voxceleb/voxceleb1.py
+++ b/dataset/voxceleb/voxceleb1.py
@@ -11,319 +11,182 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import collections
-import csv
+"""Prepare VoxCeleb1 dataset
+
+create manifest files.
+Manifest file is a json-format file with each line containing the
+meta data (i.e. audio filepath, transcript and audio duration)
+of each audio file in the data set.
+
+researchers should download the voxceleb1 dataset yourselves
+through google form to get the username & password and unpack the data
+"""
+import argparse
+import codecs
 import glob
+import json
 import os
-import random
-from typing import Dict
-from typing import List
-from typing import Tuple
-
-from paddle.io import Dataset
-from pathos.multiprocessing import Pool
-from tqdm import tqdm
-
-from paddleaudio.backends import load as load_audio
-from paddleaudio.datasets.dataset import feat_funcs
-from paddleaudio.utils import DATA_HOME
-from paddleaudio.utils import decompress
-from paddleaudio.utils import download_and_decompress
+import subprocess
+from pathlib import Path
+
+import soundfile
+
+from utils.utility import check_md5sum
 from utils.utility import download
-from utils.utility import unpack
-
-__all__ = ['VoxCeleb1']
-
-
-class VoxCeleb1(Dataset):
-    source_url = 'https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a/'
-    archieves_audio_dev = [
-        {
-            'url': source_url + 'vox1_dev_wav_partaa',
-            'md5': 'e395d020928bc15670b570a21695ed96',
-        },
-        {
-            'url': source_url + 'vox1_dev_wav_partab',
-            'md5': 'bbfaaccefab65d82b21903e81a8a8020',
-        },
-        {
-            'url': source_url + 'vox1_dev_wav_partac',
-            'md5': '017d579a2a96a077f40042ec33e51512',
-        },
-        {
-            'url': source_url + 'vox1_dev_wav_partad',
-            'md5': '7bb1e9f70fddc7a678fa998ea8b3ba19',
-        },
-    ]
-    archieves_audio_test = [
-        {
-            'url': source_url + 'vox1_test_wav.zip',
-            'md5': '185fdc63c3c739954633d50379a3d102',
-        },
-    ]
-    archieves_meta = [
-        {
-            'url':
-            'https://www.robots.ox.ac.uk/~vgg/data/voxceleb/meta/veri_test2.txt',
-            'md5':
-            'b73110731c9223c1461fe49cb48dddfc',
-        },
-    ]
-
-    num_speakers = 1211  # 1211 vox1, 5994 vox2, 7205 vox1+2, test speakers: 41
-    sample_rate = 16000
-    meta_info = collections.namedtuple(
-        'META_INFO', ('id', 'duration', 'wav', 'start', 'stop', 'spk_id'))
-    base_path = os.path.join(DATA_HOME, 'vox1')
-    wav_path = os.path.join(base_path, 'wav')
-    subsets = ['train', 'dev', 'enrol', 'test']
-
-    def __init__(
-            self,
-            subset: str='train',
-            feat_type: str='raw',
-            random_chunk: bool=True,
-            chunk_duration: float=3.0,  # seconds
-            split_ratio: float=0.9,  # train split ratio
-            seed: int=0,
-            target_dir: str=None,
-            **kwargs):
-
-        assert subset in self.subsets, \
-            'Dataset subset must be one in {}, but got {}'.format(self.subsets, subset)
-
-        self.subset = subset
-        self.spk_id2label = {}
-        self.feat_type = feat_type
-        self.feat_config = kwargs
-        self.random_chunk = random_chunk
-        self.chunk_duration = chunk_duration
-        self.split_ratio = split_ratio
-        self.target_dir = target_dir if target_dir else self.base_path
-        self.csv_path = os.path.join(
-            target_dir, 'csv') if target_dir else os.path.join(self.base_path,
-                                                               'csv')
-        self.meta_path = os.path.join(
-            target_dir, 'meta') if target_dir else os.path.join(base_path,
-                                                                'meta')
-        self.veri_test_file = os.path.join(self.meta_path, 'veri_test2.txt')
-        # self._data = self._get_data()[:1000]  # KP: Small dataset test.
-        self._data = self._get_data()
-        super(VoxCeleb1, self).__init__()
-
-        # Set up a seed to reproduce training or predicting result.
-        # random.seed(seed)
-
-    def _get_data(self):
-        # Download audio files.
-        # We need the users to decompress all vox1/dev/wav and vox1/test/wav/ to vox1/wav/ dir
-        # so, we check the vox1/wav dir status
-        print("wav base path: {}".format(self.wav_path))
-        if not os.path.isdir(self.wav_path):
-            print("start to download the voxceleb1 dataset")
-            download_and_decompress(  # multi-zip parts concatenate to vox1_dev_wav.zip
-                self.archieves_audio_dev,
-                self.base_path,
-                decompress=False)
-            download_and_decompress(  # download the vox1_test_wav.zip and unzip
-                self.archieves_audio_test,
-                self.base_path,
-                decompress=True)
-
-            # Download all parts and concatenate the files into one zip file.
-            dev_zipfile = os.path.join(self.base_path, 'vox1_dev_wav.zip')
-            print(f'Concatenating all parts to: {dev_zipfile}')
-            os.system(
-                f'cat {os.path.join(self.base_path, "vox1_dev_wav_parta*")} > {dev_zipfile}'
-            )
-
-            # Extract all audio files of dev and test set.
-            decompress(dev_zipfile, self.base_path)
-
-        # Download meta files.
-        if not os.path.isdir(self.meta_path):
-            download_and_decompress(
-                self.archieves_meta, self.meta_path, decompress=False)
-
-        # Data preparation.
-        if not os.path.isdir(self.csv_path):
-            os.makedirs(self.csv_path)
-            self.prepare_data()
-
-        data = []
-        with open(os.path.join(self.csv_path, f'{self.subset}.csv'), 'r') as rf:
-            for line in rf.readlines()[1:]:
-                audio_id, duration, wav, start, stop, spk_id = line.strip(
-                ).split(',')
-                data.append(
-                    self.meta_info(audio_id,
-                                   float(duration), wav,
-                                   int(start), int(stop), spk_id))
-
-        with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'r') as f:
-            for line in f.readlines():
-                spk_id, label = line.strip().split(' ')
-                self.spk_id2label[spk_id] = int(label)
-
-        return data
-
-    def _convert_to_record(self, idx: int):
-        sample = self._data[idx]
-
-        record = {}
-        # To show all fields in a namedtuple: `type(sample)._fields`
-        for field in type(sample)._fields:
-            record[field] = getattr(sample, field)
-
-        waveform, sr = load_audio(record['wav'])
-
-        # random select a chunk audio samples from the audio
-        if self.random_chunk:
-            num_wav_samples = waveform.shape[0]
-            num_chunk_samples = int(self.chunk_duration * sr)
-            start = random.randint(0, num_wav_samples - num_chunk_samples - 1)
-            stop = start + num_chunk_samples
+from utils.utility import unzip
+
+# all the data will be download in the current data/voxceleb directory default
+DATA_HOME = os.path.expanduser('.')
+
+# if you use the http://www.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a/ as the download base url
+# you need to get the username & password via the google form
+
+# if you use the https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a as the download base url,
+# you need use --no-check-certificate to connect the target download url 
+
+BASE_URL = "https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a"
+
+# dev data
+DEV_LIST = {
+    "vox1_dev_wav_partaa": "e395d020928bc15670b570a21695ed96",
+    "vox1_dev_wav_partab": "bbfaaccefab65d82b21903e81a8a8020",
+    "vox1_dev_wav_partac": "017d579a2a96a077f40042ec33e51512",
+    "vox1_dev_wav_partad": "7bb1e9f70fddc7a678fa998ea8b3ba19",
+}
+DEV_TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f532ba230b"
+
+# test data
+TEST_LIST = {"vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102"}
+TEST_TARGET_DATA = "vox1_test_wav.zip vox1_test_wav.zip 185fdc63c3c739954633d50379a3d102"
+
+# kaldi trial
+# this trial file is organized by kaldi according the official file,
+# which is a little different with the official trial veri_test2.txt
+KALDI_BASE_URL = "http://www.openslr.org/resources/49/"
+TRIAL_LIST = {"voxceleb1_test_v2.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7"}
+TRIAL_TARGET_DATA = "voxceleb1_test_v2.txt voxceleb1_test_v2.txt 29fc7cc1c5d59f0816dc15d6e8be60f7"
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument(
+    "--target_dir",
+    default=DATA_HOME + "/voxceleb1/",
+    type=str,
+    help="Directory to save the voxceleb1 dataset. (default: %(default)s)")
+parser.add_argument(
+    "--manifest_prefix",
+    default="manifest",
+    type=str,
+    help="Filepath prefix for output manifests. (default: %(default)s)")
+
+args = parser.parse_args()
+
+
+def create_manifest(data_dir, manifest_path_prefix):
+    print("Creating manifest %s ..." % manifest_path_prefix)
+    json_lines = []
+    data_path = os.path.join(data_dir, "wav", "**", "*.wav")
+    total_sec = 0.0
+    total_text = 0.0
+    total_num = 0
+    speakers = set()
+    for audio_path in glob.glob(data_path, recursive=True):
+        audio_id = "-".join(audio_path.split("/")[-3:])
+        utt2spk = audio_path.split("/")[-3]
+        duration = soundfile.info(audio_path).duration
+        text = ""
+        json_lines.append(
+            json.dumps(
+                {
+                    "utt": audio_id,
+                    "utt2spk": str(utt2spk),
+                    "feat": audio_path,
+                    "feat_shape": (duration, ),
+                    "text": text  # compatible with asr data format
+                },
+                ensure_ascii=False))
+
+        total_sec += duration
+        total_text += len(text)
+        total_num += 1
+        speakers.add(utt2spk)
+
+    # data_dir_name refer to dev or test
+    # voxceleb1 is given explicit in the path
+    data_dir_name = Path(data_dir).name
+    manifest_path_prefix = manifest_path_prefix + "." + data_dir_name
+    with codecs.open(manifest_path_prefix, 'w', encoding='utf-8') as f:
+        for line in json_lines:
+            f.write(line + "\n")
+
+    manifest_dir = os.path.dirname(manifest_path_prefix)
+    meta_path = os.path.join(manifest_dir, "voxceleb1." +
+                             data_dir_name) + ".meta"
+    with codecs.open(meta_path, 'w', encoding='utf-8') as f:
+        print(f"{total_num} utts", file=f)
+        print(f"{len(speakers)} speakers", file=f)
+        print(f"{total_sec / (60 * 60)} h", file=f)
+        print(f"{total_text} text", file=f)
+        print(f"{total_text / total_sec} text/sec", file=f)
+        print(f"{total_sec / total_num} sec/utt", file=f)
+
+
+def prepare_dataset(base_url, data_list, target_dir, manifest_path,
+                    target_data):
+    if not os.path.exists(target_dir):
+        os.mkdir(target_dir)
+
+    # wav directory already exists, it need do nothing
+    if not os.path.exists(os.path.join(target_dir, "wav")):
+        # download all dataset part
+        for zip_part in data_list.keys():
+            download_url = " --no-check-certificate " + base_url + "/" + zip_part
+            download(
+                url=download_url,
+                md5sum=data_list[zip_part],
+                target_dir=target_dir)
+
+        # pack the all part to target zip file
+        all_target_part, target_name, target_md5sum = target_data.split()
+        target_name = os.path.join(target_dir, target_name)
+        if not os.path.exists(target_name):
+            pack_part_cmd = "cat {}/{} > {}".format(target_dir, all_target_part,
+                                                    target_name)
+            subprocess.call(pack_part_cmd, shell=True)
+
+        # check the target zip file md5sum
+        if not check_md5sum(target_name, target_md5sum):
+            raise RuntimeError("{} MD5 checkssum failed".format(target_name))
         else:
-            start = record['start']
-            stop = record['stop']
-
-        waveform = waveform[start:stop]
-
-        assert self.feat_type in feat_funcs.keys(), \
-            f"Unknown feat_type: {self.feat_type}, it must be one in {list(feat_funcs.keys())}"
-        feat_func = feat_funcs[self.feat_type]
-        feat = feat_func(
-            waveform, sr=sr, **self.feat_config) if feat_func else waveform
-
-        record.update({'feat': feat})
-        if self.subset in ['train',
-                           'dev']:  # Labels are available in train and dev.
-            record.update({'label': self.spk_id2label[record['spk_id']]})
-
-        return record
-
-    @staticmethod
-    def _get_chunks(seg_dur, audio_id, audio_duration):
-        num_chunks = int(audio_duration / seg_dur)  # all in milliseconds
-
-        chunk_lst = [
-            audio_id + "_" + str(i * seg_dur) + "_" + str(i * seg_dur + seg_dur)
-            for i in range(num_chunks)
-        ]
-        return chunk_lst
-
-    def _get_audio_info(self, wav_file: str,
-                        split_chunks: bool) -> List[List[str]]:
-        waveform, sr = load_audio(wav_file)
-        spk_id, sess_id, utt_id = wav_file.split("/")[-3:]
-        audio_id = '-'.join([spk_id, sess_id, utt_id.split(".")[0]])
-        audio_duration = waveform.shape[0] / sr
-
-        ret = []
-        if split_chunks:  # Split into pieces of self.chunk_duration seconds.
-            uniq_chunks_list = self._get_chunks(self.chunk_duration, audio_id,
-                                                audio_duration)
-
-            for chunk in uniq_chunks_list:
-                s, e = chunk.split("_")[-2:]  # Timestamps of start and end
-                start_sample = int(float(s) * sr)
-                end_sample = int(float(e) * sr)
-                # id, duration, wav, start, stop, spk_id
-                ret.append([
-                    chunk, audio_duration, wav_file, start_sample, end_sample,
-                    spk_id
-                ])
-        else:  # Keep whole audio.
-            ret.append([
-                audio_id, audio_duration, wav_file, 0, waveform.shape[0], spk_id
-            ])
-        return ret
-
-    def generate_csv(self,
-                     wav_files: List[str],
-                     output_file: str,
-                     split_chunks: bool=True):
-        print(f'Generating csv: {output_file}')
-        header = ["id", "duration", "wav", "start", "stop", "spk_id"]
-
-        with Pool(64) as p:
-            infos = list(
-                tqdm(
-                    p.imap(lambda x: self._get_audio_info(x, split_chunks),
-                           wav_files),
-                    total=len(wav_files)))
-
-        csv_lines = []
-        for info in infos:
-            csv_lines.extend(info)
-
-        with open(output_file, mode="w") as csv_f:
-            csv_writer = csv.writer(
-                csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
-            csv_writer.writerow(header)
-            for line in csv_lines:
-                csv_writer.writerow(line)
-
-    def prepare_data(self):
-        # Audio of speakers in veri_test_file should not be included in training set.
-        print("start to prepare the data csv file")
-        enrol_files = set()
-        test_files = set()
-        # get the enroll and test audio file path
-        with open(self.veri_test_file, 'r') as f:
-            for line in f.readlines():
-                _, enrol_file, test_file = line.strip().split(' ')
-                enrol_files.add(os.path.join(self.wav_path, enrol_file))
-                test_files.add(os.path.join(self.wav_path, test_file))
-            enrol_files = sorted(enrol_files)
-            test_files = sorted(test_files)
-
-        # get the enroll and test speakers
-        test_spks = set()
-        for file in (enrol_files + test_files):
-            spk = file.split('/wav/')[1].split('/')[0]
-            test_spks.add(spk)
-
-        # get all the train and dev audios file path
-        audio_files = []
-        speakers = set()
-        for path in [self.wav_path]:
-            for file in glob.glob(
-                    os.path.join(path, "**", "*.wav"), recursive=True):
-                spk = file.split('/wav/')[1].split('/')[0]
-                if spk in test_spks:
-                    continue
-                speakers.add(spk)
-                audio_files.append(file)
-
-        print("start to generate the {}".format(
-            os.path.join(self.meta_path, 'spk_id2label.txt')))
-        # encode the train and dev speakers label to spk_id2label.txt
-        with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'w') as f:
-            for label, spk_id in enumerate(
-                    sorted(speakers)):  # 1211 vox1, 5994 vox2, 7205 vox1+2
-                f.write(f'{spk_id} {label}\n')
-
-        audio_files = sorted(audio_files)
-        random.shuffle(audio_files)
-        split_idx = int(self.split_ratio * len(audio_files))
-        # split_ratio to train
-        train_files, dev_files = audio_files[:split_idx], audio_files[
-            split_idx:]
-
-        self.generate_csv(train_files, os.path.join(self.csv_path, 'train.csv'))
-        self.generate_csv(dev_files, os.path.join(self.csv_path, 'dev.csv'))
-        self.generate_csv(
-            enrol_files,
-            os.path.join(self.csv_path, 'enrol.csv'),
-            split_chunks=False)
-        self.generate_csv(
-            test_files,
-            os.path.join(self.csv_path, 'test.csv'),
-            split_chunks=False)
-
-    def __getitem__(self, idx):
-        return self._convert_to_record(idx)
-
-    def __len__(self):
-        return len(self._data)
+            print("Check {} md5sum successfully".format(target_name))
+
+        # unzip the all zip file
+        if target_name.endswith(".zip"):
+            unzip(target_name, target_dir)
+
+    # create the manifest file
+    create_manifest(data_dir=target_dir, manifest_path_prefix=manifest_path)
+
+
+def main():
+    if args.target_dir.startswith('~'):
+        args.target_dir = os.path.expanduser(args.target_dir)
+
+    prepare_dataset(
+        base_url=BASE_URL,
+        data_list=DEV_LIST,
+        target_dir=os.path.join(args.target_dir, "dev"),
+        manifest_path=args.manifest_prefix,
+        target_data=DEV_TARGET_DATA)
+
+    prepare_dataset(
+        base_url=BASE_URL,
+        data_list=TEST_LIST,
+        target_dir=os.path.join(args.target_dir, "test"),
+        manifest_path=args.manifest_prefix,
+        target_data=TEST_TARGET_DATA)
+
+    print("Manifest prepare done!")
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/paddleaudio/datasets/voxceleb.py b/paddleaudio/datasets/voxceleb.py
new file mode 100644
index 00000000..70cf3e7a
--- /dev/null
+++ b/paddleaudio/datasets/voxceleb.py
@@ -0,0 +1,329 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import collections
+import csv
+import glob
+import os
+import random
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+from paddle.io import Dataset
+from pathos.multiprocessing import Pool
+from tqdm import tqdm
+
+from paddleaudio.backends import load as load_audio
+from paddleaudio.datasets.dataset import feat_funcs
+from paddleaudio.utils import DATA_HOME
+from paddleaudio.utils import decompress
+from paddleaudio.utils import download_and_decompress
+from utils.utility import download
+from utils.utility import unpack
+
+__all__ = ['VoxCeleb1']
+
+
+class VoxCeleb1(Dataset):
+    source_url = 'https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a/'
+    archieves_audio_dev = [
+        {
+            'url': source_url + 'vox1_dev_wav_partaa',
+            'md5': 'e395d020928bc15670b570a21695ed96',
+        },
+        {
+            'url': source_url + 'vox1_dev_wav_partab',
+            'md5': 'bbfaaccefab65d82b21903e81a8a8020',
+        },
+        {
+            'url': source_url + 'vox1_dev_wav_partac',
+            'md5': '017d579a2a96a077f40042ec33e51512',
+        },
+        {
+            'url': source_url + 'vox1_dev_wav_partad',
+            'md5': '7bb1e9f70fddc7a678fa998ea8b3ba19',
+        },
+    ]
+    archieves_audio_test = [
+        {
+            'url': source_url + 'vox1_test_wav.zip',
+            'md5': '185fdc63c3c739954633d50379a3d102',
+        },
+    ]
+    archieves_meta = [
+        {
+            'url':
+            'https://www.robots.ox.ac.uk/~vgg/data/voxceleb/meta/veri_test2.txt',
+            'md5':
+            'b73110731c9223c1461fe49cb48dddfc',
+        },
+    ]
+
+    num_speakers = 1211  # 1211 vox1, 5994 vox2, 7205 vox1+2, test speakers: 41
+    sample_rate = 16000
+    meta_info = collections.namedtuple(
+        'META_INFO', ('id', 'duration', 'wav', 'start', 'stop', 'spk_id'))
+    base_path = os.path.join(DATA_HOME, 'vox1')
+    wav_path = os.path.join(base_path, 'wav')
+    subsets = ['train', 'dev', 'enrol', 'test']
+
+    def __init__(
+            self,
+            subset: str='train',
+            feat_type: str='raw',
+            random_chunk: bool=True,
+            chunk_duration: float=3.0,  # seconds
+            split_ratio: float=0.9,  # train split ratio
+            seed: int=0,
+            target_dir: str=None,
+            **kwargs):
+
+        assert subset in self.subsets, \
+            'Dataset subset must be one in {}, but got {}'.format(self.subsets, subset)
+
+        self.subset = subset
+        self.spk_id2label = {}
+        self.feat_type = feat_type
+        self.feat_config = kwargs
+        self.random_chunk = random_chunk
+        self.chunk_duration = chunk_duration
+        self.split_ratio = split_ratio
+        self.target_dir = target_dir if target_dir else self.base_path
+        self.csv_path = os.path.join(
+            target_dir, 'csv') if target_dir else os.path.join(self.base_path,
+                                                               'csv')
+        self.meta_path = os.path.join(
+            target_dir, 'meta') if target_dir else os.path.join(self.base_path,
+                                                                'meta')
+        self.veri_test_file = os.path.join(self.meta_path, 'veri_test2.txt')
+        # self._data = self._get_data()[:1000]  # KP: Small dataset test.
+        self._data = self._get_data()
+        super(VoxCeleb1, self).__init__()
+
+        # Set up a seed to reproduce training or predicting result.
+        # random.seed(seed)
+
+    def _get_data(self):
+        # Download audio files.
+        # We need the users to decompress all vox1/dev/wav and vox1/test/wav/ to vox1/wav/ dir
+        # so, we check the vox1/wav dir status
+        print("wav base path: {}".format(self.wav_path))
+        if not os.path.isdir(self.wav_path):
+            print("start to download the voxceleb1 dataset")
+            download_and_decompress(  # multi-zip parts concatenate to vox1_dev_wav.zip
+                self.archieves_audio_dev,
+                self.base_path,
+                decompress=False)
+            download_and_decompress(  # download the vox1_test_wav.zip and unzip
+                self.archieves_audio_test,
+                self.base_path,
+                decompress=True)
+
+            # Download all parts and concatenate the files into one zip file.
+            dev_zipfile = os.path.join(self.base_path, 'vox1_dev_wav.zip')
+            print(f'Concatenating all parts to: {dev_zipfile}')
+            os.system(
+                f'cat {os.path.join(self.base_path, "vox1_dev_wav_parta*")} > {dev_zipfile}'
+            )
+
+            # Extract all audio files of dev and test set.
+            decompress(dev_zipfile, self.base_path)
+
+        # Download meta files.
+        if not os.path.isdir(self.meta_path):
+            download_and_decompress(
+                self.archieves_meta, self.meta_path, decompress=False)
+
+        # Data preparation.
+        if not os.path.isdir(self.csv_path):
+            os.makedirs(self.csv_path)
+            self.prepare_data()
+
+        data = []
+        with open(os.path.join(self.csv_path, f'{self.subset}.csv'), 'r') as rf:
+            for line in rf.readlines()[1:]:
+                audio_id, duration, wav, start, stop, spk_id = line.strip(
+                ).split(',')
+                data.append(
+                    self.meta_info(audio_id,
+                                   float(duration), wav,
+                                   int(start), int(stop), spk_id))
+
+        with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'r') as f:
+            for line in f.readlines():
+                spk_id, label = line.strip().split(' ')
+                self.spk_id2label[spk_id] = int(label)
+
+        return data
+
+    def _convert_to_record(self, idx: int):
+        sample = self._data[idx]
+
+        record = {}
+        # To show all fields in a namedtuple: `type(sample)._fields`
+        for field in type(sample)._fields:
+            record[field] = getattr(sample, field)
+
+        waveform, sr = load_audio(record['wav'])
+
+        # random select a chunk audio samples from the audio
+        if self.random_chunk:
+            num_wav_samples = waveform.shape[0]
+            num_chunk_samples = int(self.chunk_duration * sr)
+            start = random.randint(0, num_wav_samples - num_chunk_samples - 1)
+            stop = start + num_chunk_samples
+        else:
+            start = record['start']
+            stop = record['stop']
+
+        waveform = waveform[start:stop]
+
+        assert self.feat_type in feat_funcs.keys(), \
+            f"Unknown feat_type: {self.feat_type}, it must be one in {list(feat_funcs.keys())}"
+        feat_func = feat_funcs[self.feat_type]
+        feat = feat_func(
+            waveform, sr=sr, **self.feat_config) if feat_func else waveform
+
+        record.update({'feat': feat})
+        if self.subset in ['train',
+                           'dev']:  # Labels are available in train and dev.
+            record.update({'label': self.spk_id2label[record['spk_id']]})
+
+        return record
+
+    @staticmethod
+    def _get_chunks(seg_dur, audio_id, audio_duration):
+        num_chunks = int(audio_duration / seg_dur)  # all in milliseconds
+
+        chunk_lst = [
+            audio_id + "_" + str(i * seg_dur) + "_" + str(i * seg_dur + seg_dur)
+            for i in range(num_chunks)
+        ]
+        return chunk_lst
+
+    def _get_audio_info(self, wav_file: str,
+                        split_chunks: bool) -> List[List[str]]:
+        waveform, sr = load_audio(wav_file)
+        spk_id, sess_id, utt_id = wav_file.split("/")[-3:]
+        audio_id = '-'.join([spk_id, sess_id, utt_id.split(".")[0]])
+        audio_duration = waveform.shape[0] / sr
+
+        ret = []
+        if split_chunks:  # Split into pieces of self.chunk_duration seconds.
+            uniq_chunks_list = self._get_chunks(self.chunk_duration, audio_id,
+                                                audio_duration)
+
+            for chunk in uniq_chunks_list:
+                s, e = chunk.split("_")[-2:]  # Timestamps of start and end
+                start_sample = int(float(s) * sr)
+                end_sample = int(float(e) * sr)
+                # id, duration, wav, start, stop, spk_id
+                ret.append([
+                    chunk, audio_duration, wav_file, start_sample, end_sample,
+                    spk_id
+                ])
+        else:  # Keep whole audio.
+            ret.append([
+                audio_id, audio_duration, wav_file, 0, waveform.shape[0], spk_id
+            ])
+        return ret
+
+    def generate_csv(self,
+                     wav_files: List[str],
+                     output_file: str,
+                     split_chunks: bool=True):
+        print(f'Generating csv: {output_file}')
+        header = ["id", "duration", "wav", "start", "stop", "spk_id"]
+
+        with Pool(64) as p:
+            infos = list(
+                tqdm(
+                    p.imap(lambda x: self._get_audio_info(x, split_chunks),
+                           wav_files),
+                    total=len(wav_files)))
+
+        csv_lines = []
+        for info in infos:
+            csv_lines.extend(info)
+
+        with open(output_file, mode="w") as csv_f:
+            csv_writer = csv.writer(
+                csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
+            csv_writer.writerow(header)
+            for line in csv_lines:
+                csv_writer.writerow(line)
+
+    def prepare_data(self):
+        # Audio of speakers in veri_test_file should not be included in training set.
+        print("start to prepare the data csv file")
+        enrol_files = set()
+        test_files = set()
+        # get the enroll and test audio file path
+        with open(self.veri_test_file, 'r') as f:
+            for line in f.readlines():
+                _, enrol_file, test_file = line.strip().split(' ')
+                enrol_files.add(os.path.join(self.wav_path, enrol_file))
+                test_files.add(os.path.join(self.wav_path, test_file))
+            enrol_files = sorted(enrol_files)
+            test_files = sorted(test_files)
+
+        # get the enroll and test speakers
+        test_spks = set()
+        for file in (enrol_files + test_files):
+            spk = file.split('/wav/')[1].split('/')[0]
+            test_spks.add(spk)
+
+        # get all the train and dev audios file path
+        audio_files = []
+        speakers = set()
+        for path in [self.wav_path]:
+            for file in glob.glob(
+                    os.path.join(path, "**", "*.wav"), recursive=True):
+                spk = file.split('/wav/')[1].split('/')[0]
+                if spk in test_spks:
+                    continue
+                speakers.add(spk)
+                audio_files.append(file)
+
+        print("start to generate the {}".format(
+            os.path.join(self.meta_path, 'spk_id2label.txt')))
+        # encode the train and dev speakers label to spk_id2label.txt
+        with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'w') as f:
+            for label, spk_id in enumerate(
+                    sorted(speakers)):  # 1211 vox1, 5994 vox2, 7205 vox1+2
+                f.write(f'{spk_id} {label}\n')
+
+        audio_files = sorted(audio_files)
+        random.shuffle(audio_files)
+        split_idx = int(self.split_ratio * len(audio_files))
+        # split_ratio to train
+        train_files, dev_files = audio_files[:split_idx], audio_files[
+            split_idx:]
+
+        self.generate_csv(train_files, os.path.join(self.csv_path, 'train.csv'))
+        self.generate_csv(dev_files, os.path.join(self.csv_path, 'dev.csv'))
+        self.generate_csv(
+            enrol_files,
+            os.path.join(self.csv_path, 'enrol.csv'),
+            split_chunks=False)
+        self.generate_csv(
+            test_files,
+            os.path.join(self.csv_path, 'test.csv'),
+            split_chunks=False)
+
+    def __getitem__(self, idx):
+        return self._convert_to_record(idx)
+
+    def __len__(self):
+        return len(self._data)

From 57c4f4a68cf5b722bfaf6ee0f90c9f1768e7dded Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 3 Mar 2022 16:37:26 +0800
Subject: [PATCH 010/126] add sid learning rate and training model

---
 examples/voxceleb/sv0/local/train.py      | 34 ++++++++++++-
 paddlespeech/vector/layers/lr.py          | 45 +++++++++++++++++
 paddlespeech/vector/training/sid_model.py | 60 +++++++++++++++++++++++
 3 files changed, 138 insertions(+), 1 deletion(-)
 create mode 100644 paddlespeech/vector/layers/lr.py
 create mode 100644 paddlespeech/vector/training/sid_model.py

diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index c0cb1e17..8dea5fff 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -15,10 +15,14 @@ import argparse
 
 import paddle
 
-from dataset.voxceleb.voxceleb1 import VoxCeleb1
+from paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddlespeech.vector.layers.lr import CyclicLRScheduler
+from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
+from paddlespeech.vector.training.sid_model import SpeakerIdetification
 
 
 def main(args):
+    # stage0: set the training device, cpu or gpu
     paddle.set_device(args.device)
 
     # stage1: we must call the paddle.distributed.init_parallel_env() api at the begining
@@ -27,8 +31,32 @@ def main(args):
     local_rank = paddle.distributed.get_rank()
 
     # stage2: data prepare
+    # note: some cmd must do in rank==0
     train_ds = VoxCeleb1('train', target_dir=args.data_dir)
 
+    # stage3: build the dnn backbone model network
+    model_conf = {
+        "input_size": 80,
+        "channels": [1024, 1024, 1024, 1024, 3072],
+        "kernel_sizes": [5, 3, 3, 3, 1],
+        "dilations": [1, 2, 3, 4, 1],
+        "attention_channels": 128,
+        "lin_neurons": 192,
+    }
+    ecapa_tdnn = EcapaTdnn(**model_conf)
+
+    # stage4: build the speaker verification train instance with backbone model
+    model = SpeakerIdetification(
+        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
+
+    # stage5: build the optimizer, we now only construct the AdamW optimizer
+    lr_schedule = CyclicLRScheduler(
+        base_lr=args.learning_rate, max_lr=1e-3, step_size=140000 // nranks)
+    optimizer = paddle.optimizer.AdamW(
+        learning_rate=lr_schedule, parameters=model.parameters())
+
+    # stage6: build the loss function, we now only support LogSoftmaxWrapper
+
 
 if __name__ == "__main__":
     # yapf: disable
@@ -41,6 +69,10 @@ if __name__ == "__main__":
                         default="./data/",
                         type=str,
                         help="data directory")
+    parser.add_argument("--learning_rate",
+                        type=float,
+                        default=1e-8,
+                        help="Learning rate used to train with warmup.")
     args = parser.parse_args()
     # yapf: enable
 
diff --git a/paddlespeech/vector/layers/lr.py b/paddlespeech/vector/layers/lr.py
new file mode 100644
index 00000000..3dcac057
--- /dev/null
+++ b/paddlespeech/vector/layers/lr.py
@@ -0,0 +1,45 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from paddle.optimizer.lr import LRScheduler
+
+
+class CyclicLRScheduler(LRScheduler):
+    def __init__(self,
+                 base_lr: float=1e-8,
+                 max_lr: float=1e-3,
+                 step_size: int=10000):
+
+        super(CyclicLRScheduler, self).__init__()
+
+        self.current_step = -1
+        self.base_lr = base_lr
+        self.max_lr = max_lr
+        self.step_size = step_size
+
+    def step(self):
+        if not hasattr(self, 'current_step'):
+            return
+
+        self.current_step += 1
+        if self.current_step >= 2 * self.step_size:
+            self.current_step %= 2 * self.step_size
+
+        self.last_lr = self.get_lr()
+
+    def get_lr(self):
+        p = self.current_step / (2 * self.step_size)  # Proportion in one cycle.
+        if p < 0.5:  # Increase
+            return self.base_lr + p / 0.5 * (self.max_lr - self.base_lr)
+        else:  # Decrease
+            return self.max_lr - (p / 0.5 - 1) * (self.max_lr - self.base_lr)
diff --git a/paddlespeech/vector/training/sid_model.py b/paddlespeech/vector/training/sid_model.py
new file mode 100644
index 00000000..8a46c3cd
--- /dev/null
+++ b/paddlespeech/vector/training/sid_model.py
@@ -0,0 +1,60 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+
+class SpeakerIdetification(nn.Layer):
+    def __init__(
+            self,
+            backbone,
+            num_class,
+            lin_blocks=0,
+            lin_neurons=192,
+            dropout=0.1, ):
+
+        super(SpeakerIdetification, self).__init__()
+        self.backbone = backbone
+        if dropout > 0:
+            self.dropout = nn.Dropout(dropout)
+        else:
+            self.dropout = None
+        input_size = self.backbone.emb_size
+        self.blocks = nn.LayerList()
+        for i in range(lin_blocks):
+            self.blocks.extend([
+                nn.BatchNorm1D(input_size),
+                nn.Linear(in_features=input_size, out_features=lin_neurons),
+            ])
+            input_size = lin_neurons
+
+        self.weight = paddle.create_parameter(
+            shape=(input_size, num_class),
+            dtype='float32',
+            attr=paddle.ParamAttr(initializer=nn.initializer.XavierUniform()), )
+
+    def forward(self, x, lengths=None):
+        # x.shape: (N, C, L)
+        x = self.backbone(x, lengths).squeeze(
+            -1)  # (N, emb_size, 1) -> (N, emb_size)
+        if self.dropout is not None:
+            x = self.dropout(x)
+
+        for fc in self.blocks:
+            x = fc(x)
+
+        logits = F.linear(F.normalize(x), F.normalize(self.weight, axis=0))
+
+        return logits

From 6af2bc3d5badbaa865db9d2e5b371ea2eecb9a0d Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 3 Mar 2022 16:49:46 +0800
Subject: [PATCH 011/126] add sid loss wraper for voxceleb, test=doc

---
 examples/voxceleb/sv0/local/train.py | 39 +++++++++++++++-
 paddleaudio/utils/download.py        | 26 ++++++++---
 paddlespeech/vector/layers/loss.py   | 70 ++++++++++++++++++++++++++++
 3 files changed, 128 insertions(+), 7 deletions(-)
 create mode 100644 paddlespeech/vector/layers/loss.py

diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index 8dea5fff..1d9a78f9 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import argparse
 
 import paddle
@@ -19,7 +20,7 @@ from paddleaudio.datasets.voxceleb import VoxCeleb1
 from paddlespeech.vector.layers.lr import CyclicLRScheduler
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
 from paddlespeech.vector.training.sid_model import SpeakerIdetification
-
+from paddlespeech.vector.layers.loss import AdditiveAngularMargin, LogSoftmaxWrapper
 
 def main(args):
     # stage0: set the training device, cpu or gpu
@@ -33,6 +34,7 @@ def main(args):
     # stage2: data prepare
     # note: some cmd must do in rank==0
     train_ds = VoxCeleb1('train', target_dir=args.data_dir)
+    dev_ds = VoxCeleb1('dev', target_dir=args.data_dir)
 
     # stage3: build the dnn backbone model network
     model_conf = {
@@ -56,8 +58,38 @@ def main(args):
         learning_rate=lr_schedule, parameters=model.parameters())
 
     # stage6: build the loss function, we now only support LogSoftmaxWrapper
+    criterion = LogSoftmaxWrapper(
+        loss_fn=AdditiveAngularMargin(margin=0.2, scale=30))
+
+    
+    # stage7: confirm training start epoch
+    #         if pre-trained model exists, start epoch confirmed by the pre-trained model
+    start_epoch = 0
+    if args.load_checkpoint:
+        args.load_checkpoint = os.path.abspath(
+            os.path.expanduser(args.load_checkpoint))
+        try:
+            # load model checkpoint
+            state_dict = paddle.load(
+                os.path.join(args.load_checkpoint, 'model.pdparams'))
+            model.set_state_dict(state_dict)
 
+            # load optimizer checkpoint
+            state_dict = paddle.load(
+                os.path.join(args.load_checkpoint, 'model.pdopt'))
+            optimizer.set_state_dict(state_dict)
+            if local_rank == 0:
+                print(f'Checkpoint loaded from {args.load_checkpoint}')
+        except FileExistsError:
+            if local_rank == 0:
+                print('Train from scratch.')
 
+        try:
+            start_epoch = int(args.load_checkpoint[-1])
+            print(f'Restore training from epoch {start_epoch}.')
+        except ValueError:
+            pass
+    
 if __name__ == "__main__":
     # yapf: disable
     parser = argparse.ArgumentParser(__doc__)
@@ -73,6 +105,11 @@ if __name__ == "__main__":
                         type=float,
                         default=1e-8,
                         help="Learning rate used to train with warmup.")
+    parser.add_argument("--load_checkpoint", 
+                        type=str, 
+                        default=None, 
+                        help="Directory to load model checkpoint to contiune trainning.")
+
     args = parser.parse_args()
     # yapf: enable
 
diff --git a/paddleaudio/utils/download.py b/paddleaudio/utils/download.py
index 45a8e57b..a0c02ee1 100644
--- a/paddleaudio/utils/download.py
+++ b/paddleaudio/utils/download.py
@@ -23,15 +23,29 @@ from .log import logger
 download.logger = logger
 
 
-def decompress(file: str):
+def decompress(file: str, path: str=os.PathLike):
     """
-    Extracts all files from a compressed file.
+    Extracts all files from a compressed file to specific path.
     """
     assert os.path.isfile(file), "File: {} not exists.".format(file)
-    download._decompress(file)
 
+    if path is None:
+        print("decompress the data: {}".format(file))
+        download._decompress(file)
+    else:
+        print("decompress the data: {} to {}".format(file, path))
+        if not os.path.isdir(path):
+            os.makedirs(path)
 
-def download_and_decompress(archives: List[Dict[str, str]], path: str):
+        tmp_file = os.path.join(path, os.path.basename(file))
+        os.rename(file, tmp_file)
+        download._decompress(tmp_file)
+        os.rename(tmp_file, file)
+
+
+def download_and_decompress(archives: List[Dict[str, str]],
+                            path: str,
+                            decompress: bool=True):
     """
     Download archieves and decompress to specific path.
     """
@@ -41,8 +55,8 @@ def download_and_decompress(archives: List[Dict[str, str]], path: str):
     for archive in archives:
         assert 'url' in archive and 'md5' in archive, \
             'Dictionary keys of "url" and "md5" are required in the archive, but got: {list(archieve.keys())}'
-
-        download.get_path_from_url(archive['url'], path, archive['md5'])
+        download.get_path_from_url(
+            archive['url'], path, archive['md5'], decompress=decompress)
 
 
 def load_state_dict_from_url(url: str, path: str, md5: str=None):
diff --git a/paddlespeech/vector/layers/loss.py b/paddlespeech/vector/layers/loss.py
new file mode 100644
index 00000000..bf632b13
--- /dev/null
+++ b/paddlespeech/vector/layers/loss.py
@@ -0,0 +1,70 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+
+class AngularMargin(nn.Layer):
+    def __init__(self, margin=0.0, scale=1.0):
+        super(AngularMargin, self).__init__()
+        self.margin = margin
+        self.scale = scale
+
+    def forward(self, outputs, targets):
+        outputs = outputs - self.margin * targets
+        return self.scale * outputs
+
+
+class AdditiveAngularMargin(AngularMargin):
+    def __init__(self, margin=0.0, scale=1.0, easy_margin=False):
+        super(AdditiveAngularMargin, self).__init__(margin, scale)
+        self.easy_margin = easy_margin
+
+        self.cos_m = math.cos(self.margin)
+        self.sin_m = math.sin(self.margin)
+        self.th = math.cos(math.pi - self.margin)
+        self.mm = math.sin(math.pi - self.margin) * self.margin
+
+    def forward(self, outputs, targets):
+        cosine = outputs.astype('float32')
+        sine = paddle.sqrt(1.0 - paddle.pow(cosine, 2))
+        phi = cosine * self.cos_m - sine * self.sin_m  # cos(theta + m)
+        if self.easy_margin:
+            phi = paddle.where(cosine > 0, phi, cosine)
+        else:
+            phi = paddle.where(cosine > self.th, phi, cosine - self.mm)
+        outputs = (targets * phi) + ((1.0 - targets) * cosine)
+        return self.scale * outputs
+
+
+class LogSoftmaxWrapper(nn.Layer):
+    def __init__(self, loss_fn):
+        super(LogSoftmaxWrapper, self).__init__()
+        self.loss_fn = loss_fn
+        self.criterion = paddle.nn.KLDivLoss(reduction="sum")
+
+    def forward(self, outputs, targets, length=None):
+        targets = F.one_hot(targets, outputs.shape[1])
+        try:
+            predictions = self.loss_fn(outputs, targets)
+        except TypeError:
+            predictions = self.loss_fn(outputs)
+
+        predictions = F.log_softmax(predictions, axis=1)
+        loss = self.criterion(predictions, targets) / targets.sum()
+        return loss
\ No newline at end of file

From 7668f61422df4895706663ee7f563c0de83149d9 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 3 Mar 2022 17:05:20 +0800
Subject: [PATCH 012/126] add sid dataloader for training, test=doc

---
 examples/voxceleb/sv0/local/train.py  | 37 ++++++++++++++++++++++-----
 paddlespeech/vector/datasets/batch.py | 20 +++++++++++++++
 2 files changed, 50 insertions(+), 7 deletions(-)
 create mode 100644 paddlespeech/vector/datasets/batch.py

diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index 1d9a78f9..bddb94bb 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -11,16 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 import argparse
+import os
 
 import paddle
+from paddle.io import DataLoader
+from paddle.io import DistributedBatchSampler
 
 from paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddlespeech.vector.datasets.batch import waveform_collate_fn
+from paddlespeech.vector.layers.loss import AdditiveAngularMargin
+from paddlespeech.vector.layers.loss import LogSoftmaxWrapper
 from paddlespeech.vector.layers.lr import CyclicLRScheduler
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
 from paddlespeech.vector.training.sid_model import SpeakerIdetification
-from paddlespeech.vector.layers.loss import AdditiveAngularMargin, LogSoftmaxWrapper
+
 
 def main(args):
     # stage0: set the training device, cpu or gpu
@@ -61,7 +66,6 @@ def main(args):
     criterion = LogSoftmaxWrapper(
         loss_fn=AdditiveAngularMargin(margin=0.2, scale=30))
 
-    
     # stage7: confirm training start epoch
     #         if pre-trained model exists, start epoch confirmed by the pre-trained model
     start_epoch = 0
@@ -89,7 +93,19 @@ def main(args):
             print(f'Restore training from epoch {start_epoch}.')
         except ValueError:
             pass
-    
+
+    # stage8: we build the batch sampler for paddle.DataLoader
+    train_sampler = DistributedBatchSampler(
+        train_ds, batch_size=args.batch_size, shuffle=True, drop_last=False)
+    train_loader = DataLoader(
+        train_ds,
+        batch_sampler=train_sampler,
+        num_workers=args.num_workers,
+        collate_fn=waveform_collate_fn,
+        return_list=True,
+        use_buffer_reader=True, )
+
+
 if __name__ == "__main__":
     # yapf: disable
     parser = argparse.ArgumentParser(__doc__)
@@ -105,10 +121,17 @@ if __name__ == "__main__":
                         type=float,
                         default=1e-8,
                         help="Learning rate used to train with warmup.")
-    parser.add_argument("--load_checkpoint", 
-                        type=str, 
-                        default=None, 
+    parser.add_argument("--load_checkpoint",
+                        type=str,
+                        default=None,
                         help="Directory to load model checkpoint to contiune trainning.")
+    parser.add_argument("--batch_size",
+                        type=int, default=64,
+                        help="Total examples' number in batch for training.")
+    parser.add_argument("--num_workers",
+                        type=int,
+                        default=0,
+                        help="Number of workers in dataloader.")
 
     args = parser.parse_args()
     # yapf: enable
diff --git a/paddlespeech/vector/datasets/batch.py b/paddlespeech/vector/datasets/batch.py
new file mode 100644
index 00000000..a9e5d6ee
--- /dev/null
+++ b/paddlespeech/vector/datasets/batch.py
@@ -0,0 +1,20 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def waveform_collate_fn(batch):
+    waveforms = np.stack([item['feat'] for item in batch])
+    labels = np.stack([item['label'] for item in batch])
+
+    return {'waveforms': waveforms, 'labels': labels}

From 4648059b5f4c271458d85351b2442be3f61b0e60 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 3 Mar 2022 18:20:44 +0800
Subject: [PATCH 013/126] add training process for sid, test=doc

---
 examples/voxceleb/sv0/local/train.py  | 146 +++++++++++++++++++++++++-
 paddlespeech/vector/datasets/batch.py |  13 +++
 2 files changed, 154 insertions(+), 5 deletions(-)

diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index bddb94bb..f68f7373 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -14,11 +14,15 @@
 import argparse
 import os
 
+import numpy as np
 import paddle
 from paddle.io import DataLoader
 from paddle.io import DistributedBatchSampler
 
 from paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddleaudio.features.core import melspectrogram
+from paddleaudio.utils.time import Timer
+from paddlespeech.vector.datasets.batch import feature_normalize
 from paddlespeech.vector.datasets.batch import waveform_collate_fn
 from paddlespeech.vector.layers.loss import AdditiveAngularMargin
 from paddlespeech.vector.layers.loss import LogSoftmaxWrapper
@@ -26,6 +30,13 @@ from paddlespeech.vector.layers.lr import CyclicLRScheduler
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
 from paddlespeech.vector.training.sid_model import SpeakerIdetification
 
+# feat configuration
+cpu_feat_conf = {
+    'n_mels': 80,
+    'window_size': 400,
+    'hop_length': 160,
+}
+
 
 def main(args):
     # stage0: set the training device, cpu or gpu
@@ -42,9 +53,10 @@ def main(args):
     dev_ds = VoxCeleb1('dev', target_dir=args.data_dir)
 
     # stage3: build the dnn backbone model network
+    #"channels": [1024, 1024, 1024, 1024, 3072],
     model_conf = {
         "input_size": 80,
-        "channels": [1024, 1024, 1024, 1024, 3072],
+        "channels": [512, 512, 512, 512, 1536],
         "kernel_sizes": [5, 3, 3, 3, 1],
         "dilations": [1, 2, 3, 4, 1],
         "attention_channels": 128,
@@ -105,6 +117,122 @@ def main(args):
         return_list=True,
         use_buffer_reader=True, )
 
+    # stage9: start to train
+    #         we will comment the training process
+    steps_per_epoch = len(train_sampler)
+    timer = Timer(steps_per_epoch * args.epochs)
+    timer.start()
+
+    for epoch in range(start_epoch + 1, args.epochs + 1):
+        # at the begining, model must set to train mode
+        model.train()
+
+        avg_loss = 0
+        num_corrects = 0
+        num_samples = 0
+        for batch_idx, batch in enumerate(train_loader):
+            waveforms, labels = batch['waveforms'], batch['labels']
+
+            feats = []
+            for waveform in waveforms.numpy():
+                feat = melspectrogram(x=waveform, **cpu_feat_conf)
+                feats.append(feat)
+            feats = paddle.to_tensor(np.asarray(feats))
+            feats = feature_normalize(
+                feats, mean_norm=True, std_norm=False)  # Features normalization
+            logits = model(feats)
+
+            loss = criterion(logits, labels)
+            loss.backward()
+            optimizer.step()
+            if isinstance(optimizer._learning_rate,
+                          paddle.optimizer.lr.LRScheduler):
+                optimizer._learning_rate.step()
+            optimizer.clear_grad()
+
+            # Calculate loss
+            avg_loss += loss.numpy()[0]
+
+            # Calculate metrics
+            preds = paddle.argmax(logits, axis=1)
+            num_corrects += (preds == labels).numpy().sum()
+            num_samples += feats.shape[0]
+
+            timer.count()
+
+            if (batch_idx + 1) % args.log_freq == 0 and local_rank == 0:
+                lr = optimizer.get_lr()
+                avg_loss /= args.log_freq
+                avg_acc = num_corrects / num_samples
+
+                print_msg = 'Epoch={}/{}, Step={}/{}'.format(
+                    epoch, args.epochs, batch_idx + 1, steps_per_epoch)
+                print_msg += ' loss={:.4f}'.format(avg_loss)
+                print_msg += ' acc={:.4f}'.format(avg_acc)
+                print_msg += ' lr={:.4E} step/sec={:.2f} | ETA {}'.format(
+                    lr, timer.timing, timer.eta)
+                print(print_msg)
+
+                avg_loss = 0
+                num_corrects = 0
+                num_samples = 0
+
+        if epoch % args.save_freq == 0 and batch_idx + 1 == steps_per_epoch:
+            if local_rank != 0:
+                paddle.distributed.barrier(
+                )  # Wait for valid step in main process
+                continue  # Resume trainning on other process
+
+            dev_sampler = paddle.io.BatchSampler(
+                dev_ds,
+                batch_size=args.batch_size // 4,
+                shuffle=False,
+                drop_last=False)
+            dev_loader = paddle.io.DataLoader(
+                dev_ds,
+                batch_sampler=dev_sampler,
+                collate_fn=waveform_collate_fn,
+                num_workers=args.num_workers,
+                return_list=True, )
+
+            model.eval()
+            num_corrects = 0
+            num_samples = 0
+            print('Evaluate on validation dataset')
+            with paddle.no_grad():
+                for batch_idx, batch in enumerate(dev_loader):
+                    waveforms, labels = batch['waveforms'], batch['labels']
+                    # feats = feature_extractor(waveforms)
+                    feats = []
+                    for waveform in waveforms.numpy():
+                        feat = melspectrogram(x=waveform, **cpu_feat_conf)
+                        feats.append(feat)
+                    feats = paddle.to_tensor(np.asarray(feats))
+                    feats = feature_normalize(
+                        feats, mean_norm=True, std_norm=False)
+                    logits = model(feats)
+
+                    preds = paddle.argmax(logits, axis=1)
+                    num_corrects += (preds == labels).numpy().sum()
+                    num_samples += feats.shape[0]
+
+            print_msg = '[Evaluation result]'
+            print_msg += ' dev_acc={:.4f}'.format(num_corrects / num_samples)
+
+            print(print_msg)
+
+            # Save model
+            save_dir = os.path.join(args.checkpoint_dir,
+                                    'epoch_{}'.format(epoch))
+            print('Saving model checkpoint to {}'.format(save_dir))
+            paddle.save(model.state_dict(),
+                        os.path.join(save_dir, 'model.pdparams'))
+            paddle.save(optimizer.state_dict(),
+                        os.path.join(save_dir, 'model.pdopt'))
+
+            if nranks > 1:
+                paddle.distributed.barrier()  # Main process
+
 
 if __name__ == "__main__":
     # yapf: disable
@@ -117,21 +245,29 @@ if __name__ == "__main__":
                         default="./data/",
                         type=str,
                         help="data directory")
-    parser.add_argument("--learning_rate",
+    parser.add_argument("--learning-rate",
                         type=float,
                         default=1e-8,
                         help="Learning rate used to train with warmup.")
-    parser.add_argument("--load_checkpoint",
+    parser.add_argument("--load-checkpoint",
                         type=str,
                         default=None,
                         help="Directory to load model checkpoint to contiune trainning.")
-    parser.add_argument("--batch_size",
+    parser.add_argument("--batch-size",
                         type=int, default=64,
                         help="Total examples' number in batch for training.")
-    parser.add_argument("--num_workers",
+    parser.add_argument("--num-workers",
                         type=int,
                         default=0,
                         help="Number of workers in dataloader.")
+    parser.add_argument("--epochs",
+                        type=int,
+                        default=50,
+                        help="Number of epoches for fine-tuning.")
+    parser.add_argument("--log_freq",
+                        type=int,
+                        default=10,
+                        help="Log the training infomation every n steps.")
 
     args = parser.parse_args()
     # yapf: enable
diff --git a/paddlespeech/vector/datasets/batch.py b/paddlespeech/vector/datasets/batch.py
index a9e5d6ee..9db615f6 100644
--- a/paddlespeech/vector/datasets/batch.py
+++ b/paddlespeech/vector/datasets/batch.py
@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import numpy as np
+import paddle
 
 
 def waveform_collate_fn(batch):
@@ -18,3 +20,14 @@ def waveform_collate_fn(batch):
     labels = np.stack([item['label'] for item in batch])
 
     return {'waveforms': waveforms, 'labels': labels}
+
+
+def feature_normalize(feats: paddle.Tensor,
+                      mean_norm: bool=True,
+                      std_norm: bool=True):
+    # Features normalization if needed
+    mean = feats.mean(axis=-1, keepdim=True) if mean_norm else 0
+    std = feats.std(axis=-1, keepdim=True) if std_norm else 1
+    feats = (feats - mean) / std
+
+    return feats

From 1f74af110b54127bd7b2b76a3c0664c909dbea98 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 3 Mar 2022 22:17:14 +0800
Subject: [PATCH 014/126] add training log info and comment, test=doc

---
 examples/voxceleb/sv0/local/train.py | 52 +++++++++++++++------
 paddlespeech/vector/training/time.py | 67 ++++++++++++++++++++++++++++
 2 files changed, 106 insertions(+), 13 deletions(-)
 create mode 100644 paddlespeech/vector/training/time.py

diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index f68f7373..f86b0a86 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -16,12 +16,13 @@ import os
 
 import numpy as np
 import paddle
+from paddle.io import BatchSampler
 from paddle.io import DataLoader
 from paddle.io import DistributedBatchSampler
 
 from paddleaudio.datasets.voxceleb import VoxCeleb1
 from paddleaudio.features.core import melspectrogram
-from paddleaudio.utils.time import Timer
+from paddlespeech.vector.training.time import Timer
 from paddlespeech.vector.datasets.batch import feature_normalize
 from paddlespeech.vector.datasets.batch import waveform_collate_fn
 from paddlespeech.vector.layers.loss import AdditiveAngularMargin
@@ -37,7 +38,6 @@ cpu_feat_conf = {
     'hop_length': 160,
 }
 
-
 def main(args):
     # stage0: set the training device, cpu or gpu
     paddle.set_device(args.device)
@@ -82,6 +82,7 @@ def main(args):
     #         if pre-trained model exists, start epoch confirmed by the pre-trained model
     start_epoch = 0
     if args.load_checkpoint:
+        print("load the check point")
         args.load_checkpoint = os.path.abspath(
             os.path.expanduser(args.load_checkpoint))
         try:
@@ -131,18 +132,30 @@ def main(args):
         num_corrects = 0
         num_samples = 0
         for batch_idx, batch in enumerate(train_loader):
+            # stage 9-1: batch data is audio sample points and speaker id label
             waveforms, labels = batch['waveforms'], batch['labels']
 
+            # stage 9-2: audio sample augment method, which is done on the audio sample point
+            # todo
+
+            # stage 9-3: extract the audio feats,such fbank, mfcc, spectrogram
             feats = []
             for waveform in waveforms.numpy():
                 feat = melspectrogram(x=waveform, **cpu_feat_conf)
                 feats.append(feat)
             feats = paddle.to_tensor(np.asarray(feats))
+
+            # stage 9-4: feature normalize, which help converge and imporve the performance
             feats = feature_normalize(
                 feats, mean_norm=True, std_norm=False)  # Features normalization
+
+            # stage 9-5: model forward, such ecapa-tdnn, x-vector
             logits = model(feats)
 
+            # stage 9-6: loss function criterion, such AngularMargin, AdditiveAngularMargin
             loss = criterion(logits, labels)
+
+            # stage 9-7: update the gradient and clear the gradient cache
             loss.backward()
             optimizer.step()
             if isinstance(optimizer._learning_rate,
@@ -150,22 +163,22 @@ def main(args):
                 optimizer._learning_rate.step()
             optimizer.clear_grad()
 
-            # Calculate loss
+            # stage 9-8: Calculate average loss per batch
             avg_loss += loss.numpy()[0]
 
-            # Calculate metrics
+            # stage 9-9: Calculate metrics, which is one-best accuracy
             preds = paddle.argmax(logits, axis=1)
             num_corrects += (preds == labels).numpy().sum()
             num_samples += feats.shape[0]
+            timer.count()  # step plus one in timer
 
-            timer.count()
-
+            # stage 9-10: print the log information only on 0-rank per log-freq batchs
             if (batch_idx + 1) % args.log_freq == 0 and local_rank == 0:
                 lr = optimizer.get_lr()
                 avg_loss /= args.log_freq
                 avg_acc = num_corrects / num_samples
 
-                print_msg = 'Epoch={}/{}, Step={}/{}'.format(
+                print_msg = 'Train Epoch={}/{}, Step={}/{}'.format(
                     epoch, args.epochs, batch_idx + 1, steps_per_epoch)
                 print_msg += ' loss={:.4f}'.format(avg_loss)
                 print_msg += ' acc={:.4f}'.format(avg_acc)
@@ -177,36 +190,42 @@ def main(args):
                 num_corrects = 0
                 num_samples = 0
 
+        # stage 9-11: save the model parameters only on 0-rank per save-freq batchs
         if epoch % args.save_freq == 0 and batch_idx + 1 == steps_per_epoch:
             if local_rank != 0:
                 paddle.distributed.barrier(
                 )  # Wait for valid step in main process
                 continue  # Resume trainning on other process
 
-            dev_sampler = paddle.io.BatchSampler(
+            # stage 9-12: construct the valid dataset dataloader
+            dev_sampler = BatchSampler(
                 dev_ds,
                 batch_size=args.batch_size // 4,
                 shuffle=False,
                 drop_last=False)
-            dev_loader = paddle.io.DataLoader(
+            dev_loader = DataLoader(
                 dev_ds,
                 batch_sampler=dev_sampler,
                 collate_fn=waveform_collate_fn,
                 num_workers=args.num_workers,
                 return_list=True, )
 
+            # set the model to eval mode
             model.eval()
             num_corrects = 0
             num_samples = 0
+
+            # stage 9-13: evaluation the valid dataset batch data
             print('Evaluate on validation dataset')
             with paddle.no_grad():
                 for batch_idx, batch in enumerate(dev_loader):
                     waveforms, labels = batch['waveforms'], batch['labels']
-                    # feats = feature_extractor(waveforms)
+
                     feats = []
                     for waveform in waveforms.numpy():
                         feat = melspectrogram(x=waveform, **cpu_feat_conf)
                         feats.append(feat)
+
                     feats = paddle.to_tensor(np.asarray(feats))
                     feats = feature_normalize(
                         feats, mean_norm=True, std_norm=False)
@@ -218,10 +237,9 @@ def main(args):
 
             print_msg = '[Evaluation result]'
             print_msg += ' dev_acc={:.4f}'.format(num_corrects / num_samples)
-
             print(print_msg)
 
-            # Save model
+            # stage 9-14: Save model parameters
             save_dir = os.path.join(args.checkpoint_dir,
                                     'epoch_{}'.format(epoch))
             print('Saving model checkpoint to {}'.format(save_dir))
@@ -264,10 +282,18 @@ if __name__ == "__main__":
                         type=int,
                         default=50,
                         help="Number of epoches for fine-tuning.")
-    parser.add_argument("--log_freq",
+    parser.add_argument("--log-freq",
                         type=int,
                         default=10,
                         help="Log the training infomation every n steps.")
+    parser.add_argument("--save-freq",
+                        type=int,
+                        default=1,
+                        help="Save checkpoint every n epoch.")
+    parser.add_argument("--checkpoint-dir",
+                        type=str,
+                        default='./checkpoint',
+                        help="Directory to save model checkpoints.")
 
     args = parser.parse_args()
     # yapf: enable
diff --git a/paddlespeech/vector/training/time.py b/paddlespeech/vector/training/time.py
new file mode 100644
index 00000000..3a4e183d
--- /dev/null
+++ b/paddlespeech/vector/training/time.py
@@ -0,0 +1,67 @@
+# Copyright (c) 2021  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import time
+
+
+class Timer(object):
+    '''Calculate runing speed and estimated time of arrival(ETA)'''
+
+    def __init__(self, total_step: int):
+        self.total_step = total_step
+        self.last_start_step = 0
+        self.current_step = 0
+        self._is_running = True
+
+    def start(self):
+        self.last_time = time.time()
+        self.start_time = time.time()
+
+    def stop(self):
+        self._is_running = False
+        self.end_time = time.time()
+
+    def count(self) -> int:
+        if not self.current_step >= self.total_step:
+            self.current_step += 1
+        return self.current_step
+
+    @property
+    def timing(self) -> float:
+        run_steps = self.current_step - self.last_start_step
+        self.last_start_step = self.current_step
+        time_used = time.time() - self.last_time
+        self.last_time = time.time()
+        return time_used / run_steps
+
+    @property
+    def is_running(self) -> bool:
+        return self._is_running
+
+    @property
+    def eta(self) -> str:
+        if not self.is_running:
+            return '00:00:00'
+        scale = self.total_step / self.current_step
+        remaining_time = (time.time() - self.start_time) * scale
+        return seconds_to_hms(remaining_time)
+
+
+def seconds_to_hms(seconds: int) -> str:
+    '''Convert the number of seconds to hh:mm:ss'''
+    h = math.floor(seconds / 3600)
+    m = math.floor((seconds - h * 3600) / 60)
+    s = int(seconds - h * 3600 - m * 60)
+    hms_str = '{:0>2}:{:0>2}:{:0>2}'.format(h, m, s)
+    return hms_str

From 97ec01260b4188778256ef0a509ff77e8048f88a Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Fri, 4 Mar 2022 16:25:55 +0800
Subject: [PATCH 015/126] add speaker verification using cosine score, test=doc

---
 .../sv0/local/speaker_verification_cosine.py  | 238 ++++++++++++++++++
 examples/voxceleb/sv0/run.sh                  |  34 ++-
 2 files changed, 267 insertions(+), 5 deletions(-)
 create mode 100644 examples/voxceleb/sv0/local/speaker_verification_cosine.py

diff --git a/examples/voxceleb/sv0/local/speaker_verification_cosine.py b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
new file mode 100644
index 00000000..5665b5ee
--- /dev/null
+++ b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
@@ -0,0 +1,238 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import ast
+import os
+
+import numpy as np
+import paddle
+from paddle.io import BatchSampler
+from paddle.io import DataLoader
+import paddle.nn.functional as F
+from paddlespeech.vector.training.metrics import compute_eer
+from paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
+from paddlespeech.vector.training.sid_model import SpeakerIdetification
+from tqdm import tqdm
+
+
+def pad_right_2d(x, target_length, axis=-1, mode='constant', **kwargs):
+    x = np.asarray(x)
+    assert len(
+        x.shape) == 2, f'Only 2D arrays supported, but got shape: {x.shape}'
+
+    w = target_length - x.shape[axis]
+    assert w >= 0, f'Target length {target_length} is less than origin length {x.shape[axis]}'
+
+    if axis == 0:
+        pad_width = [[0, w], [0, 0]]
+    else:
+        pad_width = [[0, 0], [0, w]]
+
+    return np.pad(x, pad_width, mode=mode, **kwargs)
+
+
+def feature_normalize(batch, mean_norm: bool = True, std_norm: bool = True):
+    ids = [item['id'] for item in batch]
+    lengths = np.asarray([item['feat'].shape[1] for item in batch])
+    feats = list(
+        map(lambda x: pad_right_2d(x, lengths.max()),
+            [item['feat'] for item in batch]))
+    feats = np.stack(feats)
+
+    # Features normalization if needed
+    for i in range(len(feats)):
+        feat = feats[i][:, :lengths[i]]  # Excluding pad values.
+        mean = feat.mean(axis=-1, keepdims=True) if mean_norm else 0
+        std = feat.std(axis=-1, keepdims=True) if std_norm else 1
+        feats[i][:, :lengths[i]] = (feat - mean) / std
+        assert feats[i][:, lengths[i]:].sum(
+        ) == 0  # Padding valus should all be 0.
+
+    # Converts into ratios.
+    lengths = (lengths / lengths.max()).astype(np.float32)
+
+    return {'ids': ids, 'feats': feats, 'lengths': lengths}
+
+
+def main(args):
+    # stage0: set the training device, cpu or gpu
+    paddle.set_device(args.device)
+
+    # stage1: build the dnn backbone model network
+    ##"channels": [1024, 1024, 1024, 1024, 3072],
+    model_conf = {
+        "input_size": 80,
+        "channels": [512, 512, 512, 512, 1536],
+        "kernel_sizes": [5, 3, 3, 3, 1],
+        "dilations": [1, 2, 3, 4, 1],
+        "attention_channels": 128,
+        "lin_neurons": 192,
+    }
+    ecapa_tdnn = EcapaTdnn(**model_conf)
+
+    # stage2: build the speaker verification eval instance with backbone model
+    model = SpeakerIdetification(
+        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
+
+    # stage3: load the pre-trained model
+    args.load_checkpoint = os.path.abspath(
+        os.path.expanduser(args.load_checkpoint))
+
+    # load model checkpoint to sid model
+    state_dict = paddle.load(
+        os.path.join(args.load_checkpoint, 'model.pdparams'))
+    model.set_state_dict(state_dict)
+    print(f'Checkpoint loaded from {args.load_checkpoint}')
+
+    # stage4: construct the enroll and test dataloader
+    enrol_ds = VoxCeleb1(subset='enrol',
+                        feat_type='melspectrogram',
+                        random_chunk=False,
+                        n_mels=80,
+                        window_size=400,
+                        hop_length=160)
+    enrol_sampler = BatchSampler(
+                    enrol_ds, 
+                    batch_size=args.batch_size,
+                    shuffle=True)  # Shuffle to make embedding normalization more robust.
+    enrol_loader = DataLoader(enrol_ds,
+                    batch_sampler=enrol_sampler,
+                    collate_fn=lambda x: feature_normalize(
+                            x, mean_norm=True, std_norm=False),
+                    num_workers=args.num_workers,
+                    return_list=True,)
+
+    test_ds = VoxCeleb1(subset='test',
+                        feat_type='melspectrogram',
+                        random_chunk=False,
+                        n_mels=80,
+                        window_size=400,
+                        hop_length=160)
+
+    test_sampler = BatchSampler(test_ds, 
+                                batch_size=args.batch_size, 
+                                shuffle=True)
+    test_loader = DataLoader(test_ds,
+                            batch_sampler=test_sampler,
+                            collate_fn=lambda x: feature_normalize(
+                                x, mean_norm=True, std_norm=False),
+                            num_workers=args.num_workers,
+                            return_list=True,)
+    # stage6: we must set the model to eval mode
+    model.eval()
+
+    # stage7: global embedding norm to imporve the performance
+    if args.global_embedding_norm:
+        embedding_mean = None
+        embedding_std = None
+        mean_norm = args.embedding_mean_norm
+        std_norm = args.embedding_std_norm
+        batch_count = 0
+
+    # stage8: Compute embeddings of audios in enrol and test dataset from model.
+    id2embedding = {}
+    # Run multi times to make embedding normalization more stable.
+    for i in range(2):
+        for dl in [enrol_loader, test_loader]:
+            print(
+                f'Loop {[i+1]}: Computing embeddings on {dl.dataset.subset} dataset'
+            )
+            with paddle.no_grad():
+                for batch_idx, batch in enumerate(tqdm(dl)):
+
+                    # stage 8-1: extrac the audio embedding
+                    ids, feats, lengths = batch['ids'], batch['feats'], batch[
+                        'lengths']
+                    embeddings = model.backbone(feats, lengths).squeeze(
+                        -1).numpy()  # (N, emb_size, 1) -> (N, emb_size)
+
+                    # Global embedding normalization.
+                    if args.global_embedding_norm:
+                        batch_count += 1
+                        mean = embeddings.mean(axis=0) if mean_norm else 0
+                        std = embeddings.std(axis=0) if std_norm else 1
+                        # Update global mean and std.
+                        if embedding_mean is None and embedding_std is None:
+                            embedding_mean, embedding_std = mean, std
+                        else:
+                            weight = 1 / batch_count  # Weight decay by batches.
+                            embedding_mean = (
+                                1 - weight) * embedding_mean + weight * mean
+                            embedding_std = (
+                                1 - weight) * embedding_std + weight * std
+                        # Apply global embedding normalization.
+                        embeddings = (embeddings - embedding_mean) / embedding_std
+
+                    # Update embedding dict.
+                    id2embedding.update(dict(zip(ids, embeddings)))
+
+    # stage 9: Compute cosine scores.
+    labels = []
+    enrol_ids = []
+    test_ids = []
+    with open(VoxCeleb1.veri_test_file, 'r') as f:
+        for line in f.readlines():
+            label, enrol_id, test_id = line.strip().split(' ')
+            labels.append(int(label))
+            enrol_ids.append(enrol_id.split('.')[0].replace('/', '-'))
+            test_ids.append(test_id.split('.')[0].replace('/', '-'))
+
+    cos_sim_func = paddle.nn.CosineSimilarity(axis=1)
+    enrol_embeddings, test_embeddings = map(lambda ids: paddle.to_tensor(
+        np.asarray([id2embedding[id] for id in ids], dtype='float32')),
+                                            [enrol_ids, test_ids
+                                             ])  # (N, emb_size)
+    scores = cos_sim_func(enrol_embeddings, test_embeddings)
+    EER, threshold = compute_eer(np.asarray(labels), scores.numpy())
+    print(
+        f'EER of verification test: {EER*100:.4f}%, score threshold: {threshold:.5f}'
+    )
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--device', 
+                        choices=['cpu', 'gpu'], 
+                        default="gpu", 
+                        help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--batch-size", 
+                        type=int, 
+                        default=16, 
+                        help="Total examples' number in batch for training.")
+    parser.add_argument("--num-workers", 
+                        type=int, 
+                        default=0, 
+                        help="Number of workers in dataloader.")
+    parser.add_argument("--load-checkpoint", 
+                        type=str, 
+                        default='', 
+                        help="Directory to load model checkpoint to contiune trainning.")
+    parser.add_argument("--global-embedding-norm", 
+                        type=bool, 
+                        default=True, 
+                        help="Apply global normalization on speaker embeddings.")
+    parser.add_argument("--embedding-mean-norm", 
+                        type=bool, 
+                        default=True, 
+                        help="Apply mean normalization on speaker embeddings.")
+    parser.add_argument("--embedding-std-norm", 
+                        type=bool, 
+                        default=False, 
+                        help="Apply std normalization on speaker embeddings.")
+    args = parser.parse_args()
+    # yapf: enable
+
+    main(args)
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index a96c3827..c3b31ce5 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -2,9 +2,33 @@
 . ./path.sh
 set -e
 
-dir=./data/
-mkdir -p ${dir}
+#######################################################################
+# stage 1: train the speaker identification model
+# stage 2: test speaker identification 
+# stage 3: extract the training embeding to train the LDA and PLDA
+######################################################################
+
 # you can set the variable PPAUDIO_HOME to specifiy the downloaded the vox1 and vox2 dataset
-python3 \
-     local/train.py \
-     --data-dir ${dir}
+# default the dataset is the ~/.paddleaudio/
+# export PPAUDIO_HOME=
+
+stage=2
+dir=data/                     # data directory
+exp_dir=exp/ecapa-tdnn/       # experiment directory
+mkdir -p ${dir}
+
+if [ $stage -le 1 ]; then
+     # stage 1: train the speaker identification model
+     python3 \
+          -m paddle.distributed.launch --gpus=0,1,2,3 \
+          local/train.py --device "gpu" --checkpoint-dir ${exp_dir} \
+          --save-freq 10 --data-dir ${dir} --batch-size 256 --epochs 60
+fi
+
+if [ $stage -le 2 ]; then
+     # stage 1: train the speaker identification model
+     python3 \
+          local/speaker_verification_cosine.py \
+          --load-checkpoint ${exp_dir}/epoch_40/
+fi
+

From 016ed6d69cedff84c1913cc9650ea88c47b52dcc Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Fri, 4 Mar 2022 17:20:37 +0800
Subject: [PATCH 016/126] repair the code according to the part comment,
 test=doc

---
 .../sv0/local/speaker_verification_cosine.py  | 108 +++++++++---------
 examples/voxceleb/sv0/local/train.py          |  19 +--
 paddleaudio/datasets/voxceleb.py              |   3 +
 paddleaudio/utils/download.py                 |  34 +++---
 paddlespeech/vector/{datasets => io}/batch.py |   0
 .../vector/{layers => modules}/loss.py        |   3 +-
 paddlespeech/vector/{layers => modules}/lr.py |   0
 .../vector/{training => modules}/sid_model.py |   0
 paddlespeech/vector/training/metrics.py       |  28 +++++
 paddlespeech/vector/utils/download.py         |  72 ++++++++++++
 .../vector/{training => utils}/time.py        |   0
 11 files changed, 182 insertions(+), 85 deletions(-)
 rename paddlespeech/vector/{datasets => io}/batch.py (100%)
 rename paddlespeech/vector/{layers => modules}/loss.py (99%)
 rename paddlespeech/vector/{layers => modules}/lr.py (100%)
 rename paddlespeech/vector/{training => modules}/sid_model.py (100%)
 create mode 100644 paddlespeech/vector/training/metrics.py
 create mode 100644 paddlespeech/vector/utils/download.py
 rename paddlespeech/vector/{training => utils}/time.py (100%)

diff --git a/examples/voxceleb/sv0/local/speaker_verification_cosine.py b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
index 5665b5ee..1959e85c 100644
--- a/examples/voxceleb/sv0/local/speaker_verification_cosine.py
+++ b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
@@ -11,21 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import argparse
 import ast
 import os
 
 import numpy as np
 import paddle
+import paddle.nn.functional as F
 from paddle.io import BatchSampler
 from paddle.io import DataLoader
-import paddle.nn.functional as F
-from paddlespeech.vector.training.metrics import compute_eer
+from tqdm import tqdm
+
 from paddleaudio.datasets.voxceleb import VoxCeleb1
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
-from paddlespeech.vector.training.sid_model import SpeakerIdetification
-from tqdm import tqdm
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.metrics import compute_eer
 
 
 def pad_right_2d(x, target_length, axis=-1, mode='constant', **kwargs):
@@ -44,7 +44,7 @@ def pad_right_2d(x, target_length, axis=-1, mode='constant', **kwargs):
     return np.pad(x, pad_width, mode=mode, **kwargs)
 
 
-def feature_normalize(batch, mean_norm: bool = True, std_norm: bool = True):
+def feature_normalize(batch, mean_norm: bool=True, std_norm: bool=True):
     ids = [item['id'] for item in batch]
     lengths = np.asarray([item['feat'].shape[1] for item in batch])
     feats = list(
@@ -58,8 +58,8 @@ def feature_normalize(batch, mean_norm: bool = True, std_norm: bool = True):
         mean = feat.mean(axis=-1, keepdims=True) if mean_norm else 0
         std = feat.std(axis=-1, keepdims=True) if std_norm else 1
         feats[i][:, :lengths[i]] = (feat - mean) / std
-        assert feats[i][:, lengths[i]:].sum(
-        ) == 0  # Padding valus should all be 0.
+        assert feats[i][:, lengths[
+            i]:].sum() == 0  # Padding valus should all be 0.
 
     # Converts into ratios.
     lengths = (lengths / lengths.max()).astype(np.float32)
@@ -98,16 +98,16 @@ def main(args):
     print(f'Checkpoint loaded from {args.load_checkpoint}')
 
     # stage4: construct the enroll and test dataloader
-    enrol_ds = VoxCeleb1(subset='enrol',
-                        feat_type='melspectrogram',
-                        random_chunk=False,
-                        n_mels=80,
-                        window_size=400,
-                        hop_length=160)
+    enrol_ds = VoxCeleb1(
+        subset='enrol',
+        feat_type='melspectrogram',
+        random_chunk=False,
+        n_mels=80,
+        window_size=400,
+        hop_length=160)
     enrol_sampler = BatchSampler(
-                    enrol_ds, 
-                    batch_size=args.batch_size,
-                    shuffle=True)  # Shuffle to make embedding normalization more robust.
+        enrol_ds, batch_size=args.batch_size,
+        shuffle=True)  # Shuffle to make embedding normalization more robust.
     enrol_loader = DataLoader(enrol_ds,
                     batch_sampler=enrol_sampler,
                     collate_fn=lambda x: feature_normalize(
@@ -115,16 +115,16 @@ def main(args):
                     num_workers=args.num_workers,
                     return_list=True,)
 
-    test_ds = VoxCeleb1(subset='test',
-                        feat_type='melspectrogram',
-                        random_chunk=False,
-                        n_mels=80,
-                        window_size=400,
-                        hop_length=160)
+    test_ds = VoxCeleb1(
+        subset='test',
+        feat_type='melspectrogram',
+        random_chunk=False,
+        n_mels=80,
+        window_size=400,
+        hop_length=160)
 
-    test_sampler = BatchSampler(test_ds, 
-                                batch_size=args.batch_size, 
-                                shuffle=True)
+    test_sampler = BatchSampler(
+        test_ds, batch_size=args.batch_size, shuffle=True)
     test_loader = DataLoader(test_ds,
                             batch_sampler=test_sampler,
                             collate_fn=lambda x: feature_normalize(
@@ -169,12 +169,13 @@ def main(args):
                             embedding_mean, embedding_std = mean, std
                         else:
                             weight = 1 / batch_count  # Weight decay by batches.
-                            embedding_mean = (
-                                1 - weight) * embedding_mean + weight * mean
-                            embedding_std = (
-                                1 - weight) * embedding_std + weight * std
+                            embedding_mean = (1 - weight
+                                              ) * embedding_mean + weight * mean
+                            embedding_std = (1 - weight
+                                             ) * embedding_std + weight * std
                         # Apply global embedding normalization.
-                        embeddings = (embeddings - embedding_mean) / embedding_std
+                        embeddings = (
+                            embeddings - embedding_mean) / embedding_std
 
                     # Update embedding dict.
                     id2embedding.update(dict(zip(ids, embeddings)))
@@ -201,38 +202,39 @@ def main(args):
         f'EER of verification test: {EER*100:.4f}%, score threshold: {threshold:.5f}'
     )
 
+
 if __name__ == "__main__":
     # yapf: disable
     parser = argparse.ArgumentParser(__doc__)
-    parser.add_argument('--device', 
-                        choices=['cpu', 'gpu'], 
-                        default="gpu", 
+    parser.add_argument('--device',
+                        choices=['cpu', 'gpu'],
+                        default="gpu",
                         help="Select which device to train model, defaults to gpu.")
-    parser.add_argument("--batch-size", 
-                        type=int, 
-                        default=16, 
+    parser.add_argument("--batch-size",
+                        type=int,
+                        default=16,
                         help="Total examples' number in batch for training.")
-    parser.add_argument("--num-workers", 
-                        type=int, 
-                        default=0, 
+    parser.add_argument("--num-workers",
+                        type=int,
+                        default=0,
                         help="Number of workers in dataloader.")
-    parser.add_argument("--load-checkpoint", 
-                        type=str, 
-                        default='', 
+    parser.add_argument("--load-checkpoint",
+                        type=str,
+                        default='',
                         help="Directory to load model checkpoint to contiune trainning.")
-    parser.add_argument("--global-embedding-norm", 
-                        type=bool, 
-                        default=True, 
+    parser.add_argument("--global-embedding-norm",
+                        type=bool,
+                        default=True,
                         help="Apply global normalization on speaker embeddings.")
-    parser.add_argument("--embedding-mean-norm", 
-                        type=bool, 
-                        default=True, 
+    parser.add_argument("--embedding-mean-norm",
+                        type=bool,
+                        default=True,
                         help="Apply mean normalization on speaker embeddings.")
-    parser.add_argument("--embedding-std-norm", 
-                        type=bool, 
-                        default=False, 
+    parser.add_argument("--embedding-std-norm",
+                        type=bool,
+                        default=False,
                         help="Apply std normalization on speaker embeddings.")
     args = parser.parse_args()
     # yapf: enable
 
-    main(args)
\ No newline at end of file
+    main(args)
diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index f86b0a86..4eabf94c 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -22,22 +22,23 @@ from paddle.io import DistributedBatchSampler
 
 from paddleaudio.datasets.voxceleb import VoxCeleb1
 from paddleaudio.features.core import melspectrogram
-from paddlespeech.vector.training.time import Timer
-from paddlespeech.vector.datasets.batch import feature_normalize
-from paddlespeech.vector.datasets.batch import waveform_collate_fn
-from paddlespeech.vector.layers.loss import AdditiveAngularMargin
-from paddlespeech.vector.layers.loss import LogSoftmaxWrapper
-from paddlespeech.vector.layers.lr import CyclicLRScheduler
+from paddlespeech.vector.io.batch import feature_normalize
+from paddlespeech.vector.io.batch import waveform_collate_fn
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
-from paddlespeech.vector.training.sid_model import SpeakerIdetification
+from paddlespeech.vector.modules.loss import AdditiveAngularMargin
+from paddlespeech.vector.modules.loss import LogSoftmaxWrapper
+from paddlespeech.vector.modules.lr import CyclicLRScheduler
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.utils.time import Timer
 
 # feat configuration
 cpu_feat_conf = {
     'n_mels': 80,
-    'window_size': 400,
-    'hop_length': 160,
+    'window_size': 400,  #ms
+    'hop_length': 160,  #ms
 }
 
+
 def main(args):
     # stage0: set the training device, cpu or gpu
     paddle.set_device(args.device)
diff --git a/paddleaudio/datasets/voxceleb.py b/paddleaudio/datasets/voxceleb.py
index 70cf3e7a..760db721 100644
--- a/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/datasets/voxceleb.py
@@ -76,6 +76,9 @@ class VoxCeleb1(Dataset):
         'META_INFO', ('id', 'duration', 'wav', 'start', 'stop', 'spk_id'))
     base_path = os.path.join(DATA_HOME, 'vox1')
     wav_path = os.path.join(base_path, 'wav')
+    meta_path = os.path.join(base_path, 'meta')
+    veri_test_file = os.path.join(meta_path, 'veri_test2.txt')
+    csv_path = os.path.join(base_path, 'csv')
     subsets = ['train', 'dev', 'enrol', 'test']
 
     def __init__(
diff --git a/paddleaudio/utils/download.py b/paddleaudio/utils/download.py
index a0c02ee1..0535249b 100644
--- a/paddleaudio/utils/download.py
+++ b/paddleaudio/utils/download.py
@@ -22,30 +22,22 @@ from .log import logger
 
 download.logger = logger
 
+__all__ = [
+    'decompress',
+    'download_and_decompress',
+    'load_state_dict_from_url',
+]
 
-def decompress(file: str, path: str=os.PathLike):
+
+def decompress(file: str):
     """
-    Extracts all files from a compressed file to specific path.
+    Extracts all files from a compressed file.
     """
     assert os.path.isfile(file), "File: {} not exists.".format(file)
+    download._decompress(file)
 
-    if path is None:
-        print("decompress the data: {}".format(file))
-        download._decompress(file)
-    else:
-        print("decompress the data: {} to {}".format(file, path))
-        if not os.path.isdir(path):
-            os.makedirs(path)
-
-        tmp_file = os.path.join(path, os.path.basename(file))
-        os.rename(file, tmp_file)
-        download._decompress(tmp_file)
-        os.rename(tmp_file, file)
 
-
-def download_and_decompress(archives: List[Dict[str, str]],
-                            path: str,
-                            decompress: bool=True):
+def download_and_decompress(archives: List[Dict[str, str]], path: str):
     """
     Download archieves and decompress to specific path.
     """
@@ -55,8 +47,8 @@ def download_and_decompress(archives: List[Dict[str, str]],
     for archive in archives:
         assert 'url' in archive and 'md5' in archive, \
             'Dictionary keys of "url" and "md5" are required in the archive, but got: {list(archieve.keys())}'
-        download.get_path_from_url(
-            archive['url'], path, archive['md5'], decompress=decompress)
+
+        download.get_path_from_url(archive['url'], path, archive['md5'])
 
 
 def load_state_dict_from_url(url: str, path: str, md5: str=None):
@@ -67,4 +59,4 @@ def load_state_dict_from_url(url: str, path: str, md5: str=None):
         os.makedirs(path)
 
     download.get_path_from_url(url, path, md5)
-    return load_state_dict(os.path.join(path, os.path.basename(url)))
+    return load_state_dict(os.path.join(path, os.path.basename(url)))
\ No newline at end of file
diff --git a/paddlespeech/vector/datasets/batch.py b/paddlespeech/vector/io/batch.py
similarity index 100%
rename from paddlespeech/vector/datasets/batch.py
rename to paddlespeech/vector/io/batch.py
diff --git a/paddlespeech/vector/layers/loss.py b/paddlespeech/vector/modules/loss.py
similarity index 99%
rename from paddlespeech/vector/layers/loss.py
rename to paddlespeech/vector/modules/loss.py
index bf632b13..1aa0599a 100644
--- a/paddlespeech/vector/layers/loss.py
+++ b/paddlespeech/vector/modules/loss.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import math
 
 import paddle
@@ -67,4 +66,4 @@ class LogSoftmaxWrapper(nn.Layer):
 
         predictions = F.log_softmax(predictions, axis=1)
         loss = self.criterion(predictions, targets) / targets.sum()
-        return loss
\ No newline at end of file
+        return loss
diff --git a/paddlespeech/vector/layers/lr.py b/paddlespeech/vector/modules/lr.py
similarity index 100%
rename from paddlespeech/vector/layers/lr.py
rename to paddlespeech/vector/modules/lr.py
diff --git a/paddlespeech/vector/training/sid_model.py b/paddlespeech/vector/modules/sid_model.py
similarity index 100%
rename from paddlespeech/vector/training/sid_model.py
rename to paddlespeech/vector/modules/sid_model.py
diff --git a/paddlespeech/vector/training/metrics.py b/paddlespeech/vector/training/metrics.py
new file mode 100644
index 00000000..65dc7a3c
--- /dev/null
+++ b/paddlespeech/vector/training/metrics.py
@@ -0,0 +1,28 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List
+
+import numpy as np
+from sklearn.metrics import roc_curve
+
+
+def compute_eer(labels: np.ndarray, scores: np.ndarray) -> List[float]:
+    '''
+    Compute EER and return score threshold.
+    '''
+    fpr, tpr, threshold = roc_curve(y_true=labels, y_score=scores)
+    fnr = 1 - tpr
+    eer_threshold = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
+    eer = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
+    return eer, eer_threshold
diff --git a/paddlespeech/vector/utils/download.py b/paddlespeech/vector/utils/download.py
new file mode 100644
index 00000000..476bfea7
--- /dev/null
+++ b/paddlespeech/vector/utils/download.py
@@ -0,0 +1,72 @@
+# Copyright (c) 2021  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from typing import Dict
+from typing import List
+
+from paddle.framework import load as load_state_dict
+from paddle.utils import download
+
+__all__ = [
+    'decompress',
+    'download_and_decompress',
+    'load_state_dict_from_url',
+]
+
+
+def decompress(file: str, path: str=os.PathLike):
+    """
+    Extracts all files from a compressed file to specific path.
+    """
+    assert os.path.isfile(file), "File: {} not exists.".format(file)
+
+    if path is None:
+        print("decompress the data: {}".format(file))
+        download._decompress(file)
+    else:
+        print("decompress the data: {} to {}".format(file, path))
+        if not os.path.isdir(path):
+            os.makedirs(path)
+
+        tmp_file = os.path.join(path, os.path.basename(file))
+        os.rename(file, tmp_file)
+        download._decompress(tmp_file)
+        os.rename(tmp_file, file)
+
+
+def download_and_decompress(archives: List[Dict[str, str]],
+                            path: str,
+                            decompress: bool=True):
+    """
+    Download archieves and decompress to specific path.
+    """
+    if not os.path.isdir(path):
+        os.makedirs(path)
+
+    for archive in archives:
+        assert 'url' in archive and 'md5' in archive, \
+            'Dictionary keys of "url" and "md5" are required in the archive, but got: {list(archieve.keys())}'
+        download.get_path_from_url(
+            archive['url'], path, archive['md5'], decompress=decompress)
+
+
+def load_state_dict_from_url(url: str, path: str, md5: str=None):
+    """
+    Download and load a state dict from url
+    """
+    if not os.path.isdir(path):
+        os.makedirs(path)
+
+    download.get_path_from_url(url, path, md5)
+    return load_state_dict(os.path.join(path, os.path.basename(url)))
diff --git a/paddlespeech/vector/training/time.py b/paddlespeech/vector/utils/time.py
similarity index 100%
rename from paddlespeech/vector/training/time.py
rename to paddlespeech/vector/utils/time.py

From ac4967e204e14f6b96efc69132deeeaa89d8e4cd Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sun, 6 Mar 2022 17:54:19 +0800
Subject: [PATCH 017/126] optimize the data prepare process

---
 paddlespeech/vector/utils/time.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/paddlespeech/vector/utils/time.py b/paddlespeech/vector/utils/time.py
index 3a4e183d..8e85b0e1 100644
--- a/paddlespeech/vector/utils/time.py
+++ b/paddlespeech/vector/utils/time.py
@@ -53,8 +53,7 @@ class Timer(object):
     def eta(self) -> str:
         if not self.is_running:
             return '00:00:00'
-        scale = self.total_step / self.current_step
-        remaining_time = (time.time() - self.start_time) * scale
+        remaining_time = time.time() - self.start_time
         return seconds_to_hms(remaining_time)
 
 

From 2d89c80e6f85cc9cae841baef529a769715eb51f Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 7 Mar 2022 21:56:38 +0800
Subject: [PATCH 018/126] add waveform augment pipeline, test=doc

---
 .../sv0/local/speaker_verification_cosine.py  |  76 +-
 examples/voxceleb/sv0/local/train.py          |  46 +-
 paddleaudio/datasets/rirs_noises.py           | 207 ++++
 paddleaudio/datasets/voxceleb.py              |  18 +-
 paddlespeech/vector/io/augment.py             | 899 ++++++++++++++++++
 paddlespeech/vector/io/signal_processing.py   | 219 +++++
 paddlespeech/vector/models/ecapa_tdnn.py      |  93 ++
 paddlespeech/vector/training/seeding.py       |  28 +
 8 files changed, 1543 insertions(+), 43 deletions(-)
 create mode 100644 paddleaudio/datasets/rirs_noises.py
 create mode 100644 paddlespeech/vector/io/augment.py
 create mode 100644 paddlespeech/vector/io/signal_processing.py
 create mode 100644 paddlespeech/vector/training/seeding.py

diff --git a/examples/voxceleb/sv0/local/speaker_verification_cosine.py b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
index 1959e85c..b0adcf66 100644
--- a/examples/voxceleb/sv0/local/speaker_verification_cosine.py
+++ b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
@@ -23,9 +23,13 @@ from paddle.io import DataLoader
 from tqdm import tqdm
 
 from paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
 from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 from paddlespeech.vector.training.metrics import compute_eer
+from paddlespeech.vector.training.seeding import seed_everything
+
+logger = Log(__name__).getlog()
 
 
 def pad_right_2d(x, target_length, axis=-1, mode='constant', **kwargs):
@@ -67,9 +71,19 @@ def feature_normalize(batch, mean_norm: bool=True, std_norm: bool=True):
     return {'ids': ids, 'feats': feats, 'lengths': lengths}
 
 
+# feat configuration
+cpu_feat_conf = {
+    'n_mels': 80,
+    'window_size': 400,  #ms
+    'hop_length': 160,  #ms
+}
+
+
 def main(args):
     # stage0: set the training device, cpu or gpu
     paddle.set_device(args.device)
+    # set the random seed, it is a must for multiprocess training
+    seed_everything(args.seed)
 
     # stage1: build the dnn backbone model network
     ##"channels": [1024, 1024, 1024, 1024, 3072],
@@ -95,19 +109,18 @@ def main(args):
     state_dict = paddle.load(
         os.path.join(args.load_checkpoint, 'model.pdparams'))
     model.set_state_dict(state_dict)
-    print(f'Checkpoint loaded from {args.load_checkpoint}')
+    logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
 
     # stage4: construct the enroll and test dataloader
     enrol_ds = VoxCeleb1(
         subset='enrol',
+        target_dir=args.data_dir,
         feat_type='melspectrogram',
         random_chunk=False,
-        n_mels=80,
-        window_size=400,
-        hop_length=160)
+        **cpu_feat_conf)
     enrol_sampler = BatchSampler(
         enrol_ds, batch_size=args.batch_size,
-        shuffle=True)  # Shuffle to make embedding normalization more robust.
+        shuffle=False)  # Shuffle to make embedding normalization more robust.
     enrol_loader = DataLoader(enrol_ds,
                     batch_sampler=enrol_sampler,
                     collate_fn=lambda x: feature_normalize(
@@ -117,14 +130,13 @@ def main(args):
 
     test_ds = VoxCeleb1(
         subset='test',
+        target_dir=args.data_dir,
         feat_type='melspectrogram',
         random_chunk=False,
-        n_mels=80,
-        window_size=400,
-        hop_length=160)
+        **cpu_feat_conf)
 
     test_sampler = BatchSampler(
-        test_ds, batch_size=args.batch_size, shuffle=True)
+        test_ds, batch_size=args.batch_size, shuffle=False)
     test_loader = DataLoader(test_ds,
                             batch_sampler=test_sampler,
                             collate_fn=lambda x: feature_normalize(
@@ -136,10 +148,10 @@ def main(args):
 
     # stage7: global embedding norm to imporve the performance
     if args.global_embedding_norm:
-        embedding_mean = None
-        embedding_std = None
-        mean_norm = args.embedding_mean_norm
-        std_norm = args.embedding_std_norm
+        global_embedding_mean = None
+        global_embedding_std = None
+        mean_norm_flag = args.embedding_mean_norm
+        std_norm_flag = args.embedding_std_norm
         batch_count = 0
 
     # stage8: Compute embeddings of audios in enrol and test dataset from model.
@@ -147,7 +159,7 @@ def main(args):
     # Run multi times to make embedding normalization more stable.
     for i in range(2):
         for dl in [enrol_loader, test_loader]:
-            print(
+            logger.info(
                 f'Loop {[i+1]}: Computing embeddings on {dl.dataset.subset} dataset'
             )
             with paddle.no_grad():
@@ -162,20 +174,24 @@ def main(args):
                     # Global embedding normalization.
                     if args.global_embedding_norm:
                         batch_count += 1
-                        mean = embeddings.mean(axis=0) if mean_norm else 0
-                        std = embeddings.std(axis=0) if std_norm else 1
+                        current_mean = embeddings.mean(
+                            axis=0) if mean_norm_flag else 0
+                        current_std = embeddings.std(
+                            axis=0) if std_norm_flag else 1
                         # Update global mean and std.
-                        if embedding_mean is None and embedding_std is None:
-                            embedding_mean, embedding_std = mean, std
+                        if global_embedding_mean is None and global_embedding_std is None:
+                            global_embedding_mean, global_embedding_std = current_mean, current_std
                         else:
                             weight = 1 / batch_count  # Weight decay by batches.
-                            embedding_mean = (1 - weight
-                                              ) * embedding_mean + weight * mean
-                            embedding_std = (1 - weight
-                                             ) * embedding_std + weight * std
+                            global_embedding_mean = (
+                                1 - weight
+                            ) * global_embedding_mean + weight * current_mean
+                            global_embedding_std = (
+                                1 - weight
+                            ) * global_embedding_std + weight * current_std
                         # Apply global embedding normalization.
-                        embeddings = (
-                            embeddings - embedding_mean) / embedding_std
+                        embeddings = (embeddings - global_embedding_mean
+                                      ) / global_embedding_std
 
                     # Update embedding dict.
                     id2embedding.update(dict(zip(ids, embeddings)))
@@ -198,7 +214,7 @@ def main(args):
                                              ])  # (N, emb_size)
     scores = cos_sim_func(enrol_embeddings, test_embeddings)
     EER, threshold = compute_eer(np.asarray(labels), scores.numpy())
-    print(
+    logger.info(
         f'EER of verification test: {EER*100:.4f}%, score threshold: {threshold:.5f}'
     )
 
@@ -210,10 +226,18 @@ if __name__ == "__main__":
                         choices=['cpu', 'gpu'],
                         default="gpu",
                         help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--seed",
+                        default=0,
+                        type=int,
+                        help="random seed for paddle, numpy and python random package")
+    parser.add_argument("--data-dir",
+                        default="./data/",
+                        type=str,
+                        help="data directory")
     parser.add_argument("--batch-size",
                         type=int,
                         default=16,
-                        help="Total examples' number in batch for training.")
+                        help="Total examples' number in batch for extract the embedding.")
     parser.add_argument("--num-workers",
                         type=int,
                         default=0,
diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index 4eabf94c..745d5eab 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -22,6 +22,9 @@ from paddle.io import DistributedBatchSampler
 
 from paddleaudio.datasets.voxceleb import VoxCeleb1
 from paddleaudio.features.core import melspectrogram
+from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.io.augment import build_augment_pipeline
+from paddlespeech.vector.io.augment import waveform_augment
 from paddlespeech.vector.io.batch import feature_normalize
 from paddlespeech.vector.io.batch import waveform_collate_fn
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
@@ -29,8 +32,11 @@ from paddlespeech.vector.modules.loss import AdditiveAngularMargin
 from paddlespeech.vector.modules.loss import LogSoftmaxWrapper
 from paddlespeech.vector.modules.lr import CyclicLRScheduler
 from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.seeding import seed_everything
 from paddlespeech.vector.utils.time import Timer
 
+logger = Log(__name__).getlog()
+
 # feat configuration
 cpu_feat_conf = {
     'n_mels': 80,
@@ -47,12 +53,19 @@ def main(args):
     paddle.distributed.init_parallel_env()
     nranks = paddle.distributed.get_world_size()
     local_rank = paddle.distributed.get_rank()
+    # set the random seed, it is a must for multiprocess training
+    seed_everything(args.seed)
 
-    # stage2: data prepare
-    # note: some cmd must do in rank==0
+    # stage2: data prepare, such vox1 and vox2 data, and augment data and pipline
+    # note: some cmd must do in rank==0, so wo will refactor the data prepare code
     train_ds = VoxCeleb1('train', target_dir=args.data_dir)
     dev_ds = VoxCeleb1('dev', target_dir=args.data_dir)
 
+    if args.augment:
+        augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
+    else:
+        augment_pipeline = []
+
     # stage3: build the dnn backbone model network
     #"channels": [1024, 1024, 1024, 1024, 3072],
     model_conf = {
@@ -83,7 +96,7 @@ def main(args):
     #         if pre-trained model exists, start epoch confirmed by the pre-trained model
     start_epoch = 0
     if args.load_checkpoint:
-        print("load the check point")
+        logger.info("load the check point")
         args.load_checkpoint = os.path.abspath(
             os.path.expanduser(args.load_checkpoint))
         try:
@@ -97,14 +110,14 @@ def main(args):
                 os.path.join(args.load_checkpoint, 'model.pdopt'))
             optimizer.set_state_dict(state_dict)
             if local_rank == 0:
-                print(f'Checkpoint loaded from {args.load_checkpoint}')
+                logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
         except FileExistsError:
             if local_rank == 0:
-                print('Train from scratch.')
+                logger.info('Train from scratch.')
 
         try:
             start_epoch = int(args.load_checkpoint[-1])
-            print(f'Restore training from epoch {start_epoch}.')
+            logger.info(f'Restore training from epoch {start_epoch}.')
         except ValueError:
             pass
 
@@ -137,7 +150,10 @@ def main(args):
             waveforms, labels = batch['waveforms'], batch['labels']
 
             # stage 9-2: audio sample augment method, which is done on the audio sample point
-            # todo
+            if len(augment_pipeline) != 0:
+                waveforms = waveform_augment(waveforms, augment_pipeline)
+                labels = paddle.concat(
+                    [labels for i in range(len(augment_pipeline) + 1)])
 
             # stage 9-3: extract the audio feats,such fbank, mfcc, spectrogram
             feats = []
@@ -185,7 +201,7 @@ def main(args):
                 print_msg += ' acc={:.4f}'.format(avg_acc)
                 print_msg += ' lr={:.4E} step/sec={:.2f} | ETA {}'.format(
                     lr, timer.timing, timer.eta)
-                print(print_msg)
+                logger.info(print_msg)
 
                 avg_loss = 0
                 num_corrects = 0
@@ -217,7 +233,7 @@ def main(args):
             num_samples = 0
 
             # stage 9-13: evaluation the valid dataset batch data
-            print('Evaluate on validation dataset')
+            logger.info('Evaluate on validation dataset')
             with paddle.no_grad():
                 for batch_idx, batch in enumerate(dev_loader):
                     waveforms, labels = batch['waveforms'], batch['labels']
@@ -238,12 +254,12 @@ def main(args):
 
             print_msg = '[Evaluation result]'
             print_msg += ' dev_acc={:.4f}'.format(num_corrects / num_samples)
-            print(print_msg)
+            logger.info(print_msg)
 
             # stage 9-14: Save model parameters
             save_dir = os.path.join(args.checkpoint_dir,
                                     'epoch_{}'.format(epoch))
-            print('Saving model checkpoint to {}'.format(save_dir))
+            logger.info('Saving model checkpoint to {}'.format(save_dir))
             paddle.save(model.state_dict(),
                         os.path.join(save_dir, 'model.pdparams'))
             paddle.save(optimizer.state_dict(),
@@ -260,6 +276,10 @@ if __name__ == "__main__":
                         choices=['cpu', 'gpu'],
                         default="cpu",
                         help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--seed",
+                        default=0,
+                        type=int,
+                        help="random seed for paddle, numpy and python random package")
     parser.add_argument("--data-dir",
                         default="./data/",
                         type=str,
@@ -295,6 +315,10 @@ if __name__ == "__main__":
                         type=str,
                         default='./checkpoint',
                         help="Directory to save model checkpoints.")
+    parser.add_argument("--augment",
+                        action="store_true",
+                        default=False,
+                        help="Apply audio augments.")
 
     args = parser.parse_args()
     # yapf: enable
diff --git a/paddleaudio/datasets/rirs_noises.py b/paddleaudio/datasets/rirs_noises.py
new file mode 100644
index 00000000..fa9e7f09
--- /dev/null
+++ b/paddleaudio/datasets/rirs_noises.py
@@ -0,0 +1,207 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import collections
+import csv
+import glob
+import os
+import random
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+from paddle.io import Dataset
+from tqdm import tqdm
+
+from paddleaudio.backends import load as load_audio
+from paddleaudio.backends import save_wav
+from paddleaudio.datasets.dataset import feat_funcs
+from paddleaudio.utils import DATA_HOME
+from paddleaudio.utils import decompress
+from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.utils.download import download_and_decompress
+
+logger = Log(__name__).getlog()
+
+__all__ = ['OpenRIRNoise']
+
+
+class OpenRIRNoise(Dataset):
+    archieves = [
+        {
+            'url': 'http://www.openslr.org/resources/28/rirs_noises.zip',
+            'md5': 'e6f48e257286e05de56413b4779d8ffb',
+        },
+    ]
+
+    sample_rate = 16000
+    meta_info = collections.namedtuple('META_INFO', ('id', 'duration', 'wav'))
+    base_path = os.path.join(DATA_HOME, 'open_rir_noise')
+    wav_path = os.path.join(base_path, 'RIRS_NOISES')
+    csv_path = os.path.join(base_path, 'csv')
+    subsets = ['rir', 'noise']
+
+    def __init__(self,
+                 subset: str='rir',
+                 feat_type: str='raw',
+                 target_dir=None,
+                 random_chunk: bool=True,
+                 chunk_duration: float=3.0,
+                 seed: int=0,
+                 **kwargs):
+
+        assert subset in self.subsets, \
+            'Dataset subset must be one in {}, but got {}'.format(self.subsets, subset)
+
+        self.subset = subset
+        self.feat_type = feat_type
+        self.feat_config = kwargs
+        self.random_chunk = random_chunk
+        self.chunk_duration = chunk_duration
+
+        self.csv_path = os.path.join(target_dir, "open_rir_noise",
+                                     "csv") if target_dir else self.csv_path
+        self._data = self._get_data()
+        super(OpenRIRNoise, self).__init__()
+
+        # Set up a seed to reproduce training or predicting result.
+        # random.seed(seed)
+
+    def _get_data(self):
+        # Download audio files.
+        logger.info(f"rirs noises base path: {self.base_path}")
+        if not os.path.isdir(self.base_path):
+            download_and_decompress(
+                self.archieves, self.base_path, decompress=True)
+        else:
+            logger.info(
+                f"{self.base_path} already exists, we will not download and decompress again"
+            )
+
+        # Data preparation.
+        logger.info(f"prepare the csv to {self.csv_path}")
+        if not os.path.isdir(self.csv_path):
+            os.makedirs(self.csv_path)
+            self.prepare_data()
+
+        data = []
+        with open(os.path.join(self.csv_path, f'{self.subset}.csv'), 'r') as rf:
+            for line in rf.readlines()[1:]:
+                audio_id, duration, wav = line.strip().split(',')
+                data.append(self.meta_info(audio_id, float(duration), wav))
+
+        random.shuffle(data)
+        return data
+
+    def _convert_to_record(self, idx: int):
+        sample = self._data[idx]
+
+        record = {}
+        # To show all fields in a namedtuple: `type(sample)._fields`
+        for field in type(sample)._fields:
+            record[field] = getattr(sample, field)
+
+        waveform, sr = load_audio(record['wav'])
+
+        assert self.feat_type in feat_funcs.keys(), \
+            f"Unknown feat_type: {self.feat_type}, it must be one in {list(feat_funcs.keys())}"
+        feat_func = feat_funcs[self.feat_type]
+        feat = feat_func(
+            waveform, sr=sr, **self.feat_config) if feat_func else waveform
+
+        record.update({'feat': feat})
+        return record
+
+    @staticmethod
+    def _get_chunks(seg_dur, audio_id, audio_duration):
+        num_chunks = int(audio_duration / seg_dur)  # all in milliseconds
+
+        chunk_lst = [
+            audio_id + "_" + str(i * seg_dur) + "_" + str(i * seg_dur + seg_dur)
+            for i in range(num_chunks)
+        ]
+        return chunk_lst
+
+    def _get_audio_info(self, wav_file: str,
+                        split_chunks: bool) -> List[List[str]]:
+        waveform, sr = load_audio(wav_file)
+        audio_id = wav_file.split("/open_rir_noise/")[-1].split(".")[0]
+        audio_duration = waveform.shape[0] / sr
+
+        ret = []
+        if split_chunks and audio_duration > self.chunk_duration:  # Split into pieces of self.chunk_duration seconds.
+            uniq_chunks_list = self._get_chunks(self.chunk_duration, audio_id,
+                                                audio_duration)
+
+            for idx, chunk in enumerate(uniq_chunks_list):
+                s, e = chunk.split("_")[-2:]  # Timestamps of start and end
+                start_sample = int(float(s) * sr)
+                end_sample = int(float(e) * sr)
+                new_wav_file = os.path.join(self.base_path,
+                                            audio_id + f'_chunk_{idx+1:02}.wav')
+                save_wav(waveform[start_sample:end_sample], sr, new_wav_file)
+                # id, duration, new_wav
+                ret.append([chunk, self.chunk_duration, new_wav_file])
+        else:  # Keep whole audio.
+            ret.append([audio_id, audio_duration, wav_file])
+        return ret
+
+    def generate_csv(self,
+                     wav_files: List[str],
+                     output_file: str,
+                     split_chunks: bool=True):
+        logger.info(f'Generating csv: {output_file}')
+        header = ["id", "duration", "wav"]
+
+        infos = list(
+            tqdm(
+                map(self._get_audio_info, wav_files, [split_chunks] * len(
+                    wav_files)),
+                total=len(wav_files)))
+
+        csv_lines = []
+        for info in infos:
+            csv_lines.extend(info)
+
+        with open(output_file, mode="w") as csv_f:
+            csv_writer = csv.writer(
+                csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
+            csv_writer.writerow(header)
+            for line in csv_lines:
+                csv_writer.writerow(line)
+
+    def prepare_data(self):
+        rir_list = os.path.join(self.wav_path, "real_rirs_isotropic_noises",
+                                "rir_list")
+        rir_files = []
+        with open(rir_list, 'r') as f:
+            for line in f.readlines():
+                rir_file = line.strip().split(' ')[-1]
+                rir_files.append(os.path.join(self.base_path, rir_file))
+
+        noise_list = os.path.join(self.wav_path, "pointsource_noises",
+                                  "noise_list")
+        noise_files = []
+        with open(noise_list, 'r') as f:
+            for line in f.readlines():
+                noise_file = line.strip().split(' ')[-1]
+                noise_files.append(os.path.join(self.base_path, noise_file))
+
+        self.generate_csv(rir_files, os.path.join(self.csv_path, 'rir.csv'))
+        self.generate_csv(noise_files, os.path.join(self.csv_path, 'noise.csv'))
+
+    def __getitem__(self, idx):
+        return self._convert_to_record(idx)
+
+    def __len__(self):
+        return len(self._data)
diff --git a/paddleaudio/datasets/voxceleb.py b/paddleaudio/datasets/voxceleb.py
index 760db721..28f6dfc6 100644
--- a/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/datasets/voxceleb.py
@@ -29,9 +29,12 @@ from paddleaudio.datasets.dataset import feat_funcs
 from paddleaudio.utils import DATA_HOME
 from paddleaudio.utils import decompress
 from paddleaudio.utils import download_and_decompress
+from paddlespeech.s2t.utils.log import Log
 from utils.utility import download
 from utils.utility import unpack
 
+logger = Log(__name__).getlog()
+
 __all__ = ['VoxCeleb1']
 
 
@@ -121,9 +124,9 @@ class VoxCeleb1(Dataset):
         # Download audio files.
         # We need the users to decompress all vox1/dev/wav and vox1/test/wav/ to vox1/wav/ dir
         # so, we check the vox1/wav dir status
-        print("wav base path: {}".format(self.wav_path))
+        logger.info(f"wav base path: {self.wav_path}")
         if not os.path.isdir(self.wav_path):
-            print("start to download the voxceleb1 dataset")
+            logger.info(f"start to download the voxceleb1 dataset")
             download_and_decompress(  # multi-zip parts concatenate to vox1_dev_wav.zip
                 self.archieves_audio_dev,
                 self.base_path,
@@ -135,7 +138,7 @@ class VoxCeleb1(Dataset):
 
             # Download all parts and concatenate the files into one zip file.
             dev_zipfile = os.path.join(self.base_path, 'vox1_dev_wav.zip')
-            print(f'Concatenating all parts to: {dev_zipfile}')
+            logger.info(f'Concatenating all parts to: {dev_zipfile}')
             os.system(
                 f'cat {os.path.join(self.base_path, "vox1_dev_wav_parta*")} > {dev_zipfile}'
             )
@@ -154,6 +157,9 @@ class VoxCeleb1(Dataset):
             self.prepare_data()
 
         data = []
+        logger.info(
+            f"read the {self.subset} from {os.path.join(self.csv_path, f'{self.subset}.csv')}"
+        )
         with open(os.path.join(self.csv_path, f'{self.subset}.csv'), 'r') as rf:
             for line in rf.readlines()[1:]:
                 audio_id, duration, wav, start, stop, spk_id = line.strip(
@@ -246,7 +252,7 @@ class VoxCeleb1(Dataset):
                      wav_files: List[str],
                      output_file: str,
                      split_chunks: bool=True):
-        print(f'Generating csv: {output_file}')
+        logger.info(f'Generating csv: {output_file}')
         header = ["id", "duration", "wav", "start", "stop", "spk_id"]
 
         with Pool(64) as p:
@@ -269,7 +275,7 @@ class VoxCeleb1(Dataset):
 
     def prepare_data(self):
         # Audio of speakers in veri_test_file should not be included in training set.
-        print("start to prepare the data csv file")
+        logger.info("start to prepare the data csv file")
         enrol_files = set()
         test_files = set()
         # get the enroll and test audio file path
@@ -299,7 +305,7 @@ class VoxCeleb1(Dataset):
                 speakers.add(spk)
                 audio_files.append(file)
 
-        print("start to generate the {}".format(
+        logger.info("start to generate the {}".format(
             os.path.join(self.meta_path, 'spk_id2label.txt')))
         # encode the train and dev speakers label to spk_id2label.txt
         with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'w') as f:
diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
new file mode 100644
index 00000000..d6bbc8a9
--- /dev/null
+++ b/paddlespeech/vector/io/augment.py
@@ -0,0 +1,899 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import os
+from typing import List
+
+import numpy as np
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+
+from paddleaudio.backends import load as load_audio
+from paddleaudio.datasets.rirs_noises import OpenRIRNoise
+from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.io.signal_processing import compute_amplitude
+from paddlespeech.vector.io.signal_processing import convolve1d
+from paddlespeech.vector.io.signal_processing import dB_to_amplitude
+from paddlespeech.vector.io.signal_processing import notch_filter
+from paddlespeech.vector.io.signal_processing import reverberate
+
+logger = Log(__name__).getlog()
+
+
+# TODO: Complete type-hint and doc string.
+class DropFreq(nn.Layer):
+    def __init__(
+            self,
+            drop_freq_low=1e-14,
+            drop_freq_high=1,
+            drop_count_low=1,
+            drop_count_high=2,
+            drop_width=0.05,
+            drop_prob=1, ):
+        super(DropFreq, self).__init__()
+        self.drop_freq_low = drop_freq_low
+        self.drop_freq_high = drop_freq_high
+        self.drop_count_low = drop_count_low
+        self.drop_count_high = drop_count_high
+        self.drop_width = drop_width
+        self.drop_prob = drop_prob
+
+    def forward(self, waveforms):
+        # Don't drop (return early) 1-`drop_prob` portion of the batches
+        dropped_waveform = waveforms.clone()
+        if paddle.rand([1]) > self.drop_prob:
+            return dropped_waveform
+
+        # Add channels dimension
+        if len(waveforms.shape) == 2:
+            dropped_waveform = dropped_waveform.unsqueeze(-1)
+
+        # Pick number of frequencies to drop
+        drop_count = paddle.randint(
+            low=self.drop_count_low, high=self.drop_count_high + 1, shape=[1])
+
+        # Pick a frequency to drop
+        drop_range = self.drop_freq_high - self.drop_freq_low
+        drop_frequency = (
+            paddle.rand([drop_count]) * drop_range + self.drop_freq_low)
+
+        # Filter parameters
+        filter_length = 101
+        pad = filter_length // 2
+
+        # Start with delta function
+        drop_filter = paddle.zeros([1, filter_length, 1])
+        drop_filter[0, pad, 0] = 1
+
+        # Subtract each frequency
+        for frequency in drop_frequency:
+            notch_kernel = notch_filter(frequency, filter_length,
+                                        self.drop_width)
+            drop_filter = convolve1d(drop_filter, notch_kernel, pad)
+
+        # Apply filter
+        dropped_waveform = convolve1d(dropped_waveform, drop_filter, pad)
+
+        # Remove channels dimension if added
+        return dropped_waveform.squeeze(-1)
+
+
+class DropChunk(nn.Layer):
+    def __init__(
+            self,
+            drop_length_low=100,
+            drop_length_high=1000,
+            drop_count_low=1,
+            drop_count_high=10,
+            drop_start=0,
+            drop_end=None,
+            drop_prob=1,
+            noise_factor=0.0, ):
+        super(DropChunk, self).__init__()
+        self.drop_length_low = drop_length_low
+        self.drop_length_high = drop_length_high
+        self.drop_count_low = drop_count_low
+        self.drop_count_high = drop_count_high
+        self.drop_start = drop_start
+        self.drop_end = drop_end
+        self.drop_prob = drop_prob
+        self.noise_factor = noise_factor
+
+        # Validate low < high
+        if drop_length_low > drop_length_high:
+            raise ValueError("Low limit must not be more than high limit")
+        if drop_count_low > drop_count_high:
+            raise ValueError("Low limit must not be more than high limit")
+
+        # Make sure the length doesn't exceed end - start
+        if drop_end is not None and drop_end >= 0:
+            if drop_start > drop_end:
+                raise ValueError("Low limit must not be more than high limit")
+
+            drop_range = drop_end - drop_start
+            self.drop_length_low = min(drop_length_low, drop_range)
+            self.drop_length_high = min(drop_length_high, drop_range)
+
+    def forward(self, waveforms, lengths):
+        # Reading input list
+        lengths = (lengths * waveforms.shape[1]).astype('int64')
+        batch_size = waveforms.shape[0]
+        dropped_waveform = waveforms.clone()
+
+        # Don't drop (return early) 1-`drop_prob` portion of the batches
+        if paddle.rand([1]) > self.drop_prob:
+            return dropped_waveform
+
+        # Store original amplitude for computing white noise amplitude
+        clean_amplitude = compute_amplitude(waveforms, lengths.unsqueeze(1))
+
+        # Pick a number of times to drop
+        drop_times = paddle.randint(
+            low=self.drop_count_low,
+            high=self.drop_count_high + 1,
+            shape=[batch_size], )
+
+        # Iterate batch to set mask
+        for i in range(batch_size):
+            if drop_times[i] == 0:
+                continue
+
+            # Pick lengths
+            length = paddle.randint(
+                low=self.drop_length_low,
+                high=self.drop_length_high + 1,
+                shape=[drop_times[i]], )
+
+            # Compute range of starting locations
+            start_min = self.drop_start
+            if start_min < 0:
+                start_min += lengths[i]
+            start_max = self.drop_end
+            if start_max is None:
+                start_max = lengths[i]
+            if start_max < 0:
+                start_max += lengths[i]
+            start_max = max(0, start_max - length.max())
+
+            # Pick starting locations
+            start = paddle.randint(
+                low=start_min,
+                high=start_max + 1,
+                shape=[drop_times[i]], )
+
+            end = start + length
+
+            # Update waveform
+            if not self.noise_factor:
+                for j in range(drop_times[i]):
+                    dropped_waveform[i, start[j]:end[j]] = 0.0
+            else:
+                # Uniform distribution of -2 to +2 * avg amplitude should
+                # preserve the average for normalization
+                noise_max = 2 * clean_amplitude[i] * self.noise_factor
+                for j in range(drop_times[i]):
+                    # zero-center the noise distribution
+                    noise_vec = paddle.rand([length[j]], dtype='float32')
+
+                    noise_vec = 2 * noise_max * noise_vec - noise_max
+                    dropped_waveform[i, int(start[j]):int(end[j])] = noise_vec
+
+        return dropped_waveform
+
+
+class Resample(nn.Layer):
+    def __init__(
+            self,
+            orig_freq=16000,
+            new_freq=16000,
+            lowpass_filter_width=6, ):
+        super(Resample, self).__init__()
+        self.orig_freq = orig_freq
+        self.new_freq = new_freq
+        self.lowpass_filter_width = lowpass_filter_width
+
+        # Compute rate for striding
+        self._compute_strides()
+        assert self.orig_freq % self.conv_stride == 0
+        assert self.new_freq % self.conv_transpose_stride == 0
+
+    def _compute_strides(self):
+        # Compute new unit based on ratio of in/out frequencies
+        base_freq = math.gcd(self.orig_freq, self.new_freq)
+        input_samples_in_unit = self.orig_freq // base_freq
+        self.output_samples = self.new_freq // base_freq
+
+        # Store the appropriate stride based on the new units
+        self.conv_stride = input_samples_in_unit
+        self.conv_transpose_stride = self.output_samples
+
+    def forward(self, waveforms):
+        if not hasattr(self, "first_indices"):
+            self._indices_and_weights(waveforms)
+
+        # Don't do anything if the frequencies are the same
+        if self.orig_freq == self.new_freq:
+            return waveforms
+
+        unsqueezed = False
+        if len(waveforms.shape) == 2:
+            waveforms = waveforms.unsqueeze(1)
+            unsqueezed = True
+        elif len(waveforms.shape) == 3:
+            waveforms = waveforms.transpose([0, 2, 1])
+        else:
+            raise ValueError("Input must be 2 or 3 dimensions")
+
+        # Do resampling
+        resampled_waveform = self._perform_resample(waveforms)
+
+        if unsqueezed:
+            resampled_waveform = resampled_waveform.squeeze(1)
+        else:
+            resampled_waveform = resampled_waveform.transpose([0, 2, 1])
+
+        return resampled_waveform
+
+    def _perform_resample(self, waveforms):
+        # Compute output size and initialize
+        batch_size, num_channels, wave_len = waveforms.shape
+        window_size = self.weights.shape[1]
+        tot_output_samp = self._output_samples(wave_len)
+        resampled_waveform = paddle.zeros((batch_size, num_channels,
+                                           tot_output_samp))
+
+        # eye size: (num_channels, num_channels, 1)
+        eye = paddle.eye(num_channels).unsqueeze(2)
+
+        # Iterate over the phases in the polyphase filter
+        for i in range(self.first_indices.shape[0]):
+            wave_to_conv = waveforms
+            first_index = int(self.first_indices[i].item())
+            if first_index >= 0:
+                # trim the signal as the filter will not be applied
+                # before the first_index
+                wave_to_conv = wave_to_conv[:, :, first_index:]
+
+            # pad the right of the signal to allow partial convolutions
+            # meaning compute values for partial windows (e.g. end of the
+            # window is outside the signal length)
+            max_index = (tot_output_samp - 1) // self.output_samples
+            end_index = max_index * self.conv_stride + window_size
+            current_wave_len = wave_len - first_index
+            right_padding = max(0, end_index + 1 - current_wave_len)
+            left_padding = max(0, -first_index)
+            wave_to_conv = paddle.nn.functional.pad(
+                wave_to_conv, [left_padding, right_padding], data_format='NCL')
+            conv_wave = paddle.nn.functional.conv1d(
+                x=wave_to_conv,
+                # weight=self.weights[i].repeat(num_channels, 1, 1),
+                weight=self.weights[i].expand((num_channels, 1, -1)),
+                stride=self.conv_stride,
+                groups=num_channels, )
+
+            # we want conv_wave[:, i] to be at
+            # output[:, i + n*conv_transpose_stride]
+            dilated_conv_wave = paddle.nn.functional.conv1d_transpose(
+                conv_wave, eye, stride=self.conv_transpose_stride)
+
+            # pad dilated_conv_wave so it reaches the output length if needed.
+            left_padding = i
+            previous_padding = left_padding + dilated_conv_wave.shape[-1]
+            right_padding = max(0, tot_output_samp - previous_padding)
+            dilated_conv_wave = paddle.nn.functional.pad(
+                dilated_conv_wave, [left_padding, right_padding],
+                data_format='NCL')
+            dilated_conv_wave = dilated_conv_wave[:, :, :tot_output_samp]
+
+            resampled_waveform += dilated_conv_wave
+
+        return resampled_waveform
+
+    def _output_samples(self, input_num_samp):
+        samp_in = int(self.orig_freq)
+        samp_out = int(self.new_freq)
+
+        tick_freq = abs(samp_in * samp_out) // math.gcd(samp_in, samp_out)
+        ticks_per_input_period = tick_freq // samp_in
+
+        # work out the number of ticks in the time interval
+        # [ 0, input_num_samp/samp_in ).
+        interval_length = input_num_samp * ticks_per_input_period
+        if interval_length <= 0:
+            return 0
+        ticks_per_output_period = tick_freq // samp_out
+
+        # Get the last output-sample in the closed interval,
+        # i.e. replacing [ ) with [ ]. Note: integer division rounds down.
+        # See http://en.wikipedia.org/wiki/Interval_(mathematics) for an
+        # explanation of the notation.
+        last_output_samp = interval_length // ticks_per_output_period
+
+        # We need the last output-sample in the open interval, so if it
+        # takes us to the end of the interval exactly, subtract one.
+        if last_output_samp * ticks_per_output_period == interval_length:
+            last_output_samp -= 1
+
+        # First output-sample index is zero, so the number of output samples
+        # is the last output-sample plus one.
+        num_output_samp = last_output_samp + 1
+
+        return num_output_samp
+
+    def _indices_and_weights(self, waveforms):
+        # Lowpass filter frequency depends on smaller of two frequencies
+        min_freq = min(self.orig_freq, self.new_freq)
+        lowpass_cutoff = 0.99 * 0.5 * min_freq
+
+        assert lowpass_cutoff * 2 <= min_freq
+        window_width = self.lowpass_filter_width / (2.0 * lowpass_cutoff)
+
+        assert lowpass_cutoff < min(self.orig_freq, self.new_freq) / 2
+        output_t = paddle.arange(start=0.0, end=self.output_samples)
+        output_t /= self.new_freq
+        min_t = output_t - window_width
+        max_t = output_t + window_width
+
+        min_input_index = paddle.ceil(min_t * self.orig_freq)
+        max_input_index = paddle.floor(max_t * self.orig_freq)
+        num_indices = max_input_index - min_input_index + 1
+
+        max_weight_width = num_indices.max()
+        j = paddle.arange(max_weight_width, dtype='float32')
+        input_index = min_input_index.unsqueeze(1) + j.unsqueeze(0)
+        delta_t = (input_index / self.orig_freq) - output_t.unsqueeze(1)
+
+        weights = paddle.zeros_like(delta_t)
+        inside_window_indices = delta_t.abs().less_than(
+            paddle.to_tensor(window_width))
+
+        # raised-cosine (Hanning) window with width `window_width`
+        weights[inside_window_indices] = 0.5 * (1 + paddle.cos(
+            2 * math.pi * lowpass_cutoff / self.lowpass_filter_width *
+            delta_t.masked_select(inside_window_indices)))
+
+        t_eq_zero_indices = delta_t.equal(paddle.zeros_like(delta_t))
+        t_not_eq_zero_indices = delta_t.not_equal(paddle.zeros_like(delta_t))
+
+        # sinc filter function
+        weights = paddle.where(
+            t_not_eq_zero_indices,
+            weights * paddle.sin(2 * math.pi * lowpass_cutoff * delta_t) /
+            (math.pi * delta_t), weights)
+
+        # limit of the function at t = 0
+        weights = paddle.where(t_eq_zero_indices, weights * 2 * lowpass_cutoff,
+                               weights)
+
+        # size (output_samples, max_weight_width)
+        weights /= self.orig_freq
+
+        self.first_indices = min_input_index
+        self.weights = weights
+
+
+class SpeedPerturb(nn.Layer):
+    def __init__(
+            self,
+            orig_freq,
+            speeds=[90, 100, 110],
+            perturb_prob=1.0, ):
+        super(SpeedPerturb, self).__init__()
+        self.orig_freq = orig_freq
+        self.speeds = speeds
+        self.perturb_prob = perturb_prob
+
+        # Initialize index of perturbation
+        self.samp_index = 0
+
+        # Initialize resamplers
+        self.resamplers = []
+        for speed in self.speeds:
+            config = {
+                "orig_freq": self.orig_freq,
+                "new_freq": self.orig_freq * speed // 100,
+            }
+            self.resamplers.append(Resample(**config))
+
+    def forward(self, waveform):
+        # Don't perturb (return early) 1-`perturb_prob` portion of the batches
+        if paddle.rand([1]) > self.perturb_prob:
+            return waveform.clone()
+
+        # Perform a random perturbation
+        self.samp_index = paddle.randint(len(self.speeds), shape=[1]).item()
+        perturbed_waveform = self.resamplers[self.samp_index](waveform)
+
+        return perturbed_waveform
+
+
+class AddNoise(nn.Layer):
+    def __init__(
+            self,
+            noise_dataset=None,  # None for white noise
+            num_workers=0,
+            snr_low=0,
+            snr_high=0,
+            mix_prob=1.0,
+            start_index=None,
+            normalize=False, ):
+        super(AddNoise, self).__init__()
+
+        self.num_workers = num_workers
+        self.snr_low = snr_low
+        self.snr_high = snr_high
+        self.mix_prob = mix_prob
+        self.start_index = start_index
+        self.normalize = normalize
+        self.noise_dataset = noise_dataset
+        self.noise_dataloader = None
+
+    def forward(self, waveforms, lengths=None):
+        if lengths is None:
+            lengths = paddle.ones([len(waveforms)])
+
+        # Copy clean waveform to initialize noisy waveform
+        noisy_waveform = waveforms.clone()
+        lengths = (lengths * waveforms.shape[1]).astype('int64').unsqueeze(1)
+
+        # Don't add noise (return early) 1-`mix_prob` portion of the batches
+        if paddle.rand([1]) > self.mix_prob:
+            return noisy_waveform
+
+        # Compute the average amplitude of the clean waveforms
+        clean_amplitude = compute_amplitude(waveforms, lengths)
+
+        # Pick an SNR and use it to compute the mixture amplitude factors
+        SNR = paddle.rand((len(waveforms), 1))
+        SNR = SNR * (self.snr_high - self.snr_low) + self.snr_low
+        noise_amplitude_factor = 1 / (dB_to_amplitude(SNR) + 1)
+        new_noise_amplitude = noise_amplitude_factor * clean_amplitude
+
+        # Scale clean signal appropriately
+        noisy_waveform *= 1 - noise_amplitude_factor
+
+        # Loop through clean samples and create mixture
+        if self.noise_dataset is None:
+            white_noise = paddle.normal(shape=waveforms.shape)
+            noisy_waveform += new_noise_amplitude * white_noise
+        else:
+            tensor_length = waveforms.shape[1]
+            noise_waveform, noise_length = self._load_noise(
+                lengths,
+                tensor_length, )
+
+            # Rescale and add
+            noise_amplitude = compute_amplitude(noise_waveform, noise_length)
+            noise_waveform *= new_noise_amplitude / (noise_amplitude + 1e-14)
+            noisy_waveform += noise_waveform
+
+        # Normalizing to prevent clipping
+        if self.normalize:
+            abs_max, _ = paddle.max(
+                paddle.abs(noisy_waveform), axis=1, keepdim=True)
+            noisy_waveform = noisy_waveform / abs_max.clip(min=1.0)
+
+        return noisy_waveform
+
+    def _load_noise(self, lengths, max_length):
+        """
+        Load a batch of noises
+
+        args
+        lengths(Paddle.Tensor): Num samples of waveforms with shape (N, 1).
+        max_length(int): Width of a batch.
+        """
+        lengths = lengths.squeeze(1)
+        batch_size = len(lengths)
+
+        # Load a noise batch
+        if self.noise_dataloader is None:
+
+            def noise_collate_fn(batch):
+                def pad(x, target_length, mode='constant', **kwargs):
+                    x = np.asarray(x)
+                    w = target_length - x.shape[0]
+                    assert w >= 0, f'Target length {target_length} is less than origin length {x.shape[0]}'
+                    return np.pad(x, [0, w], mode=mode, **kwargs)
+
+                ids = [item['id'] for item in batch]
+                lengths = np.asarray([item['feat'].shape[0] for item in batch])
+                waveforms = list(
+                    map(lambda x: pad(x, max(max_length, lengths.max().item())),
+                        [item['feat'] for item in batch]))
+                waveforms = np.stack(waveforms)
+                return {'ids': ids, 'feats': waveforms, 'lengths': lengths}
+
+            # Create noise data loader.
+            self.noise_dataloader = paddle.io.DataLoader(
+                self.noise_dataset,
+                batch_size=batch_size,
+                shuffle=True,
+                num_workers=self.num_workers,
+                collate_fn=noise_collate_fn,
+                return_list=True, )
+            self.noise_data = iter(self.noise_dataloader)
+
+        noise_batch, noise_len = self._load_noise_batch_of_size(batch_size)
+
+        # Select a random starting location in the waveform
+        start_index = self.start_index
+        if self.start_index is None:
+            start_index = 0
+            max_chop = (noise_len - lengths).min().clip(min=1)
+            start_index = paddle.randint(high=max_chop, shape=[1])
+
+        # Truncate noise_batch to max_length
+        noise_batch = noise_batch[:, start_index:start_index + max_length]
+        noise_len = (noise_len - start_index).clip(max=max_length).unsqueeze(1)
+        return noise_batch, noise_len
+
+    def _load_noise_batch_of_size(self, batch_size):
+        """Concatenate noise batches, then chop to correct size"""
+        noise_batch, noise_lens = self._load_noise_batch()
+
+        # Expand
+        while len(noise_batch) < batch_size:
+            noise_batch = paddle.concat((noise_batch, noise_batch))
+            noise_lens = paddle.concat((noise_lens, noise_lens))
+
+        # Contract
+        if len(noise_batch) > batch_size:
+            noise_batch = noise_batch[:batch_size]
+            noise_lens = noise_lens[:batch_size]
+
+        return noise_batch, noise_lens
+
+    def _load_noise_batch(self):
+        """Load a batch of noises, restarting iteration if necessary."""
+        try:
+            batch = next(self.noise_data)
+        except StopIteration:
+            self.noise_data = iter(self.noise_dataloader)
+            batch = next(self.noise_data)
+
+        noises, lens = batch['feats'], batch['lengths']
+        return noises, lens
+
+
+class AddReverb(nn.Layer):
+    def __init__(
+            self,
+            rir_dataset,
+            reverb_prob=1.0,
+            rir_scale_factor=1.0,
+            num_workers=0, ):
+        super(AddReverb, self).__init__()
+        self.rir_dataset = rir_dataset
+        self.reverb_prob = reverb_prob
+        self.rir_scale_factor = rir_scale_factor
+
+        # Create rir data loader.
+        def rir_collate_fn(batch):
+            def pad(x, target_length, mode='constant', **kwargs):
+                x = np.asarray(x)
+                w = target_length - x.shape[0]
+                assert w >= 0, f'Target length {target_length} is less than origin length {x.shape[0]}'
+                return np.pad(x, [0, w], mode=mode, **kwargs)
+
+            ids = [item['id'] for item in batch]
+            lengths = np.asarray([item['feat'].shape[0] for item in batch])
+            waveforms = list(
+                map(lambda x: pad(x, lengths.max().item()),
+                    [item['feat'] for item in batch]))
+            waveforms = np.stack(waveforms)
+            return {'ids': ids, 'feats': waveforms, 'lengths': lengths}
+
+        self.rir_dataloader = paddle.io.DataLoader(
+            self.rir_dataset,
+            collate_fn=rir_collate_fn,
+            num_workers=num_workers,
+            shuffle=True,
+            return_list=True, )
+
+        self.rir_data = iter(self.rir_dataloader)
+
+    def forward(self, waveforms, lengths=None):
+        """
+        Arguments
+        ---------
+        waveforms : tensor
+            Shape should be `[batch, time]` or `[batch, time, channels]`.
+        lengths : tensor
+            Shape should be a single dimension, `[batch]`.
+
+        Returns
+        -------
+        Tensor of shape `[batch, time]` or `[batch, time, channels]`.
+        """
+
+        if lengths is None:
+            lengths = paddle.ones([len(waveforms)])
+
+        # Don't add reverb (return early) 1-`reverb_prob` portion of the time
+        if paddle.rand([1]) > self.reverb_prob:
+            return waveforms.clone()
+
+        # Add channels dimension if necessary
+        channel_added = False
+        if len(waveforms.shape) == 2:
+            waveforms = waveforms.unsqueeze(-1)
+            channel_added = True
+
+        # Load and prepare RIR
+        rir_waveform = self._load_rir()
+
+        # Compress or dilate RIR
+        if self.rir_scale_factor != 1:
+            rir_waveform = F.interpolate(
+                rir_waveform.transpose([0, 2, 1]),
+                scale_factor=self.rir_scale_factor,
+                mode="linear",
+                align_corners=False,
+                data_format='NCW', )
+            # (N, C, L) -> (N, L, C)
+            rir_waveform = rir_waveform.transpose([0, 2, 1])
+
+        rev_waveform = reverberate(
+            waveforms,
+            rir_waveform,
+            self.rir_dataset.sample_rate,
+            rescale_amp="avg")
+
+        # Remove channels dimension if added
+        if channel_added:
+            return rev_waveform.squeeze(-1)
+
+        return rev_waveform
+
+    def _load_rir(self):
+        try:
+            batch = next(self.rir_data)
+        except StopIteration:
+            self.rir_data = iter(self.rir_dataloader)
+            batch = next(self.rir_data)
+
+        rir_waveform = batch['feats']
+
+        # Make sure RIR has correct channels
+        if len(rir_waveform.shape) == 2:
+            rir_waveform = rir_waveform.unsqueeze(-1)
+
+        return rir_waveform
+
+
+class AddBabble(nn.Layer):
+    def __init__(
+            self,
+            speaker_count=3,
+            snr_low=0,
+            snr_high=0,
+            mix_prob=1, ):
+        super(AddBabble, self).__init__()
+        self.speaker_count = speaker_count
+        self.snr_low = snr_low
+        self.snr_high = snr_high
+        self.mix_prob = mix_prob
+
+    def forward(self, waveforms, lengths=None):
+        if lengths is None:
+            lengths = paddle.ones([len(waveforms)])
+
+        babbled_waveform = waveforms.clone()
+        lengths = (lengths * waveforms.shape[1]).unsqueeze(1)
+        batch_size = len(waveforms)
+
+        # Don't mix (return early) 1-`mix_prob` portion of the batches
+        if paddle.rand([1]) > self.mix_prob:
+            return babbled_waveform
+
+        # Pick an SNR and use it to compute the mixture amplitude factors
+        clean_amplitude = compute_amplitude(waveforms, lengths)
+        SNR = paddle.rand((batch_size, 1))
+        SNR = SNR * (self.snr_high - self.snr_low) + self.snr_low
+        noise_amplitude_factor = 1 / (dB_to_amplitude(SNR) + 1)
+        new_noise_amplitude = noise_amplitude_factor * clean_amplitude
+
+        # Scale clean signal appropriately
+        babbled_waveform *= 1 - noise_amplitude_factor
+
+        # For each speaker in the mixture, roll and add
+        babble_waveform = waveforms.roll((1, ), axis=0)
+        babble_len = lengths.roll((1, ), axis=0)
+        for i in range(1, self.speaker_count):
+            babble_waveform += waveforms.roll((1 + i, ), axis=0)
+            babble_len = paddle.concat(
+                [babble_len, babble_len.roll((1, ), axis=0)], axis=-1).max(
+                    axis=-1, keepdim=True)
+
+        # Rescale and add to mixture
+        babble_amplitude = compute_amplitude(babble_waveform, babble_len)
+        babble_waveform *= new_noise_amplitude / (babble_amplitude + 1e-14)
+        babbled_waveform += babble_waveform
+
+        return babbled_waveform
+
+
+class TimeDomainSpecAugment(nn.Layer):
+    def __init__(
+            self,
+            perturb_prob=1.0,
+            drop_freq_prob=1.0,
+            drop_chunk_prob=1.0,
+            speeds=[95, 100, 105],
+            sample_rate=16000,
+            drop_freq_count_low=0,
+            drop_freq_count_high=3,
+            drop_chunk_count_low=0,
+            drop_chunk_count_high=5,
+            drop_chunk_length_low=1000,
+            drop_chunk_length_high=2000,
+            drop_chunk_noise_factor=0, ):
+        super(TimeDomainSpecAugment, self).__init__()
+        self.speed_perturb = SpeedPerturb(
+            perturb_prob=perturb_prob,
+            orig_freq=sample_rate,
+            speeds=speeds, )
+        self.drop_freq = DropFreq(
+            drop_prob=drop_freq_prob,
+            drop_count_low=drop_freq_count_low,
+            drop_count_high=drop_freq_count_high, )
+        self.drop_chunk = DropChunk(
+            drop_prob=drop_chunk_prob,
+            drop_count_low=drop_chunk_count_low,
+            drop_count_high=drop_chunk_count_high,
+            drop_length_low=drop_chunk_length_low,
+            drop_length_high=drop_chunk_length_high,
+            noise_factor=drop_chunk_noise_factor, )
+
+    def forward(self, waveforms, lengths=None):
+        if lengths is None:
+            lengths = paddle.ones([len(waveforms)])
+
+        with paddle.no_grad():
+            # Augmentation
+            waveforms = self.speed_perturb(waveforms)
+            waveforms = self.drop_freq(waveforms)
+            waveforms = self.drop_chunk(waveforms, lengths)
+
+        return waveforms
+
+
+class EnvCorrupt(nn.Layer):
+    def __init__(
+            self,
+            reverb_prob=1.0,
+            babble_prob=1.0,
+            noise_prob=1.0,
+            rir_dataset=None,
+            noise_dataset=None,
+            num_workers=0,
+            babble_speaker_count=0,
+            babble_snr_low=0,
+            babble_snr_high=0,
+            noise_snr_low=0,
+            noise_snr_high=0,
+            rir_scale_factor=1.0, ):
+        super(EnvCorrupt, self).__init__()
+
+        # Initialize corrupters
+        if rir_dataset is not None and reverb_prob > 0.0:
+            self.add_reverb = AddReverb(
+                rir_dataset=rir_dataset,
+                num_workers=num_workers,
+                reverb_prob=reverb_prob,
+                rir_scale_factor=rir_scale_factor, )
+
+        if babble_speaker_count > 0 and babble_prob > 0.0:
+            self.add_babble = AddBabble(
+                speaker_count=babble_speaker_count,
+                snr_low=babble_snr_low,
+                snr_high=babble_snr_high,
+                mix_prob=babble_prob, )
+
+        if noise_dataset is not None and noise_prob > 0.0:
+            self.add_noise = AddNoise(
+                noise_dataset=noise_dataset,
+                num_workers=num_workers,
+                snr_low=noise_snr_low,
+                snr_high=noise_snr_high,
+                mix_prob=noise_prob, )
+
+    def forward(self, waveforms, lengths=None):
+        if lengths is None:
+            lengths = paddle.ones([len(waveforms)])
+
+        # Augmentation
+        with paddle.no_grad():
+            if hasattr(self, "add_reverb"):
+                try:
+                    waveforms = self.add_reverb(waveforms, lengths)
+                except Exception:
+                    pass
+            if hasattr(self, "add_babble"):
+                waveforms = self.add_babble(waveforms, lengths)
+            if hasattr(self, "add_noise"):
+                waveforms = self.add_noise(waveforms, lengths)
+
+        return waveforms
+
+
+def build_augment_pipeline(target_dir=None) -> List[paddle.nn.Layer]:
+    """build augment pipeline
+    Note: this pipeline cannot be used in the paddle.DataLoader
+
+    Returns:
+        List[paddle.nn.Layer]: all augment process
+    """
+    logger.info("start to build the augment pipeline")
+    noise_dataset = OpenRIRNoise('noise', target_dir=target_dir)
+    rir_dataset = OpenRIRNoise('rir')
+
+    wavedrop = TimeDomainSpecAugment(
+        sample_rate=16000,
+        speeds=[100], )
+    speed_perturb = TimeDomainSpecAugment(
+        sample_rate=16000,
+        speeds=[95, 100, 105], )
+    add_noise = EnvCorrupt(
+        noise_dataset=noise_dataset,
+        reverb_prob=0.0,
+        noise_prob=1.0,
+        noise_snr_low=0,
+        noise_snr_high=15,
+        rir_scale_factor=1.0, )
+    add_rev = EnvCorrupt(
+        rir_dataset=rir_dataset,
+        reverb_prob=1.0,
+        noise_prob=0.0,
+        rir_scale_factor=1.0, )
+    add_rev_noise = EnvCorrupt(
+        noise_dataset=noise_dataset,
+        rir_dataset=rir_dataset,
+        reverb_prob=1.0,
+        noise_prob=1.0,
+        noise_snr_low=0,
+        noise_snr_high=15,
+        rir_scale_factor=1.0, )
+
+    return [wavedrop, speed_perturb, add_noise, add_rev, add_rev_noise]
+
+
+def waveform_augment(waveforms: paddle.Tensor,
+                     augment_pipeline: List[paddle.nn.Layer]) -> paddle.Tensor:
+    """process the augment pipeline and return all the waveforms
+
+    Args:
+        waveforms (paddle.Tensor): _description_
+        augment_pipeline (List[paddle.nn.Layer]): _description_
+
+    Returns:
+        paddle.Tensor: _description_
+    """
+    waveforms_aug_list = [waveforms]
+    for aug in augment_pipeline:
+        waveforms_aug = aug(waveforms)  # (N, L)
+        if waveforms_aug.shape[1] >= waveforms.shape[1]:
+            # Trunc
+            waveforms_aug = waveforms_aug[:, :waveforms.shape[1]]
+        else:
+            # Pad
+            lengths_to_pad = waveforms.shape[1] - waveforms_aug.shape[1]
+            waveforms_aug = F.pad(
+                waveforms_aug.unsqueeze(-1), [0, lengths_to_pad],
+                data_format='NLC').squeeze(-1)
+        waveforms_aug_list.append(waveforms_aug)
+
+    return paddle.concat(waveforms_aug_list, axis=0)
diff --git a/paddlespeech/vector/io/signal_processing.py b/paddlespeech/vector/io/signal_processing.py
new file mode 100644
index 00000000..a61bf554
--- /dev/null
+++ b/paddlespeech/vector/io/signal_processing.py
@@ -0,0 +1,219 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+
+import numpy as np
+import paddle
+
+# TODO: Complete type-hint and doc string.
+
+
+def blackman_window(win_len, dtype=np.float32):
+    arcs = np.pi * np.arange(win_len) / float(win_len)
+    win = np.asarray(
+        [0.42 - 0.5 * np.cos(2 * arc) + 0.08 * np.cos(4 * arc) for arc in arcs],
+        dtype=dtype)
+    return paddle.to_tensor(win)
+
+
+def compute_amplitude(waveforms, lengths=None, amp_type="avg", scale="linear"):
+    if len(waveforms.shape) == 1:
+        waveforms = waveforms.unsqueeze(0)
+
+    assert amp_type in ["avg", "peak"]
+    assert scale in ["linear", "dB"]
+
+    if amp_type == "avg":
+        if lengths is None:
+            out = paddle.mean(paddle.abs(waveforms), axis=1, keepdim=True)
+        else:
+            wav_sum = paddle.sum(paddle.abs(waveforms), axis=1, keepdim=True)
+            out = wav_sum / lengths
+    elif amp_type == "peak":
+        out = paddle.max(paddle.abs(waveforms), axis=1, keepdim=True)
+    else:
+        raise NotImplementedError
+
+    if scale == "linear":
+        return out
+    elif scale == "dB":
+        return paddle.clip(20 * paddle.log10(out), min=-80)
+    else:
+        raise NotImplementedError
+
+
+def dB_to_amplitude(SNR):
+    return 10**(SNR / 20)
+
+
+def convolve1d(
+        waveform,
+        kernel,
+        padding=0,
+        pad_type="constant",
+        stride=1,
+        groups=1, ):
+    if len(waveform.shape) != 3:
+        raise ValueError("Convolve1D expects a 3-dimensional tensor")
+
+    # Padding can be a tuple (left_pad, right_pad) or an int
+    if isinstance(padding, list):
+        waveform = paddle.nn.functional.pad(
+            x=waveform,
+            pad=padding,
+            mode=pad_type,
+            data_format='NLC', )
+
+    # Move time dimension last, which pad and fft and conv expect.
+    # (N, L, C) -> (N, C, L)
+    waveform = waveform.transpose([0, 2, 1])
+    kernel = kernel.transpose([0, 2, 1])
+
+    convolved = paddle.nn.functional.conv1d(
+        x=waveform,
+        weight=kernel,
+        stride=stride,
+        groups=groups,
+        padding=padding if not isinstance(padding, list) else 0, )
+
+    # Return time dimension to the second dimension.
+    return convolved.transpose([0, 2, 1])
+
+
+def notch_filter(notch_freq, filter_width=101, notch_width=0.05):
+    # Check inputs
+    assert 0 < notch_freq <= 1
+    assert filter_width % 2 != 0
+    pad = filter_width // 2
+    inputs = paddle.arange(filter_width, dtype='float32') - pad
+
+    # Avoid frequencies that are too low
+    notch_freq += notch_width
+
+    # Define sinc function, avoiding division by zero
+    def sinc(x):
+        def _sinc(x):
+            return paddle.sin(x) / x
+
+        # The zero is at the middle index
+        res = paddle.concat(
+            [_sinc(x[:pad]), paddle.ones([1]), _sinc(x[pad + 1:])])
+        return res
+
+    # Compute a low-pass filter with cutoff frequency notch_freq.
+    hlpf = sinc(3 * (notch_freq - notch_width) * inputs)
+    # import torch
+    # hlpf *= paddle.to_tensor(torch.blackman_window(filter_width).detach().numpy())
+    hlpf *= blackman_window(filter_width)
+    hlpf /= paddle.sum(hlpf)
+
+    # Compute a high-pass filter with cutoff frequency notch_freq.
+    hhpf = sinc(3 * (notch_freq + notch_width) * inputs)
+    # hhpf *= paddle.to_tensor(torch.blackman_window(filter_width).detach().numpy())
+    hhpf *= blackman_window(filter_width)
+    hhpf /= -paddle.sum(hhpf)
+    hhpf[pad] += 1
+
+    # Adding filters creates notch filter
+    return (hlpf + hhpf).reshape([1, -1, 1])
+
+
+def reverberate(waveforms,
+                rir_waveform,
+                sample_rate,
+                impulse_duration=0.3,
+                rescale_amp="avg"):
+    orig_shape = waveforms.shape
+
+    if len(waveforms.shape) > 3 or len(rir_waveform.shape) > 3:
+        raise NotImplementedError
+
+    # if inputs are mono tensors we reshape to 1, samples
+    if len(waveforms.shape) == 1:
+        waveforms = waveforms.unsqueeze(0).unsqueeze(-1)
+    elif len(waveforms.shape) == 2:
+        waveforms = waveforms.unsqueeze(-1)
+
+    if len(rir_waveform.shape) == 1:  # convolve1d expects a 3d tensor !
+        rir_waveform = rir_waveform.unsqueeze(0).unsqueeze(-1)
+    elif len(rir_waveform.shape) == 2:
+        rir_waveform = rir_waveform.unsqueeze(-1)
+
+    # Compute the average amplitude of the clean
+    orig_amplitude = compute_amplitude(waveforms, waveforms.shape[1],
+                                       rescale_amp)
+
+    # Compute index of the direct signal, so we can preserve alignment
+    impulse_index_start = rir_waveform.abs().argmax(axis=1).item()
+    impulse_index_end = min(
+        impulse_index_start + int(sample_rate * impulse_duration),
+        rir_waveform.shape[1])
+    rir_waveform = rir_waveform[:, impulse_index_start:impulse_index_end, :]
+    rir_waveform = rir_waveform / paddle.norm(rir_waveform, p=2)
+    rir_waveform = paddle.flip(rir_waveform, [1])
+
+    waveforms = convolve1d(
+        waveform=waveforms,
+        kernel=rir_waveform,
+        padding=[rir_waveform.shape[1] - 1, 0], )
+
+    # Rescale to the peak amplitude of the clean waveform
+    waveforms = rescale(waveforms, waveforms.shape[1], orig_amplitude,
+                        rescale_amp)
+
+    if len(orig_shape) == 1:
+        waveforms = waveforms.squeeze(0).squeeze(-1)
+    if len(orig_shape) == 2:
+        waveforms = waveforms.squeeze(-1)
+
+    return waveforms
+
+
+def rescale(waveforms, lengths, target_lvl, amp_type="avg", scale="linear"):
+    assert amp_type in ["peak", "avg"]
+    assert scale in ["linear", "dB"]
+
+    batch_added = False
+    if len(waveforms.shape) == 1:
+        batch_added = True
+        waveforms = waveforms.unsqueeze(0)
+
+    waveforms = normalize(waveforms, lengths, amp_type)
+
+    if scale == "linear":
+        out = target_lvl * waveforms
+    elif scale == "dB":
+        out = dB_to_amplitude(target_lvl) * waveforms
+
+    else:
+        raise NotImplementedError("Invalid scale, choose between dB and linear")
+
+    if batch_added:
+        out = out.squeeze(0)
+
+    return out
+
+
+def normalize(waveforms, lengths=None, amp_type="avg", eps=1e-14):
+    assert amp_type in ["avg", "peak"]
+
+    batch_added = False
+    if len(waveforms.shape) == 1:
+        batch_added = True
+        waveforms = waveforms.unsqueeze(0)
+
+    den = compute_amplitude(waveforms, lengths, amp_type) + eps
+    if batch_added:
+        waveforms = waveforms.squeeze(0)
+    return waveforms / den
diff --git a/paddlespeech/vector/models/ecapa_tdnn.py b/paddlespeech/vector/models/ecapa_tdnn.py
index e493b800..4c960e11 100644
--- a/paddlespeech/vector/models/ecapa_tdnn.py
+++ b/paddlespeech/vector/models/ecapa_tdnn.py
@@ -19,6 +19,16 @@ import paddle.nn.functional as F
 
 
 def length_to_mask(length, max_len=None, dtype=None):
+    """_summary_
+
+    Args:
+        length (_type_): _description_
+        max_len (_type_, optional): _description_. Defaults to None.
+        dtype (_type_, optional): _description_. Defaults to None.
+
+    Returns:
+        _type_: _description_
+    """
     assert len(length.shape) == 1
 
     if max_len is None:
@@ -47,6 +57,19 @@ class Conv1d(nn.Layer):
             groups=1,
             bias=True,
             padding_mode="reflect", ):
+        """_summary_
+
+        Args:
+            in_channels (_type_): _description_
+            out_channels (_type_): _description_
+            kernel_size (_type_): _description_
+            stride (int, optional): _description_. Defaults to 1.
+            padding (str, optional): _description_. Defaults to "same".
+            dilation (int, optional): _description_. Defaults to 1.
+            groups (int, optional): _description_. Defaults to 1.
+            bias (bool, optional): _description_. Defaults to True.
+            padding_mode (str, optional): _description_. Defaults to "reflect".
+        """
         super().__init__()
 
         self.kernel_size = kernel_size
@@ -66,6 +89,17 @@ class Conv1d(nn.Layer):
             bias_attr=bias, )
 
     def forward(self, x):
+        """_summary_
+
+        Args:
+            x (_type_): _description_
+
+        Raises:
+            ValueError: _description_
+
+        Returns:
+            _type_: _description_
+        """
         if self.padding == "same":
             x = self._manage_padding(x, self.kernel_size, self.dilation,
                                      self.stride)
@@ -75,6 +109,17 @@ class Conv1d(nn.Layer):
         return self.conv(x)
 
     def _manage_padding(self, x, kernel_size: int, dilation: int, stride: int):
+        """_summary_
+
+        Args:
+            x (_type_): _description_
+            kernel_size (int): _description_
+            dilation (int): _description_
+            stride (int): _description_
+
+        Returns:
+            _type_: _description_
+        """
         L_in = x.shape[-1]  # Detecting input shape
         padding = self._get_padding_elem(L_in, stride, kernel_size,
                                          dilation)  # Time padding
@@ -88,6 +133,17 @@ class Conv1d(nn.Layer):
                           stride: int,
                           kernel_size: int,
                           dilation: int):
+        """_summary_
+
+        Args:
+            L_in (int): _description_
+            stride (int): _description_
+            kernel_size (int): _description_
+            dilation (int): _description_
+
+        Returns:
+            _type_: _description_
+        """
         if stride > 1:
             n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1)
             L_out = stride * (n_steps - 1) + kernel_size * dilation
@@ -134,6 +190,15 @@ class TDNNBlock(nn.Layer):
             kernel_size,
             dilation,
             activation=nn.ReLU, ):
+        """Implementation of TDNN network
+
+        Args:
+            in_channels (int): input channels or input embedding dimensions
+            out_channels (int): output channels or output embedding dimensions
+            kernel_size (int): the kernel size of the TDNN network block
+            dilation (int): the dilation of the TDNN network block
+            activation (paddle class, optional): the activation layers. Defaults to nn.ReLU.
+        """
         super().__init__()
         self.conv = Conv1d(
             in_channels=in_channels,
@@ -149,6 +214,15 @@ class TDNNBlock(nn.Layer):
 
 class Res2NetBlock(nn.Layer):
     def __init__(self, in_channels, out_channels, scale=8, dilation=1):
+        """Implementation of Res2Net Block with dilation
+           The paper is refered as "Res2Net: A New Multi-scale Backbone Architecture",
+           whose url is https://arxiv.org/abs/1904.01169
+        Args:
+            in_channels (int): input channels or input dimensions
+            out_channels (int): output channels or output dimensions
+            scale (int, optional): _description_. Defaults to 8.
+            dilation (int, optional): _description_. Defaults to 1.
+        """
         super().__init__()
         assert in_channels % scale == 0
         assert out_channels % scale == 0
@@ -179,6 +253,14 @@ class Res2NetBlock(nn.Layer):
 
 class SEBlock(nn.Layer):
     def __init__(self, in_channels, se_channels, out_channels):
+        """Implementation of SEBlock
+           The paper is refered as "Squeeze-and-Excitation Networks"
+           whose url is https://arxiv.org/abs/1709.01507
+        Args:
+            in_channels (int): input channels or input data dimensions
+            se_channels (_type_): _description_
+            out_channels (int): output channels or output data dimensions
+        """
         super().__init__()
 
         self.conv1 = Conv1d(
@@ -275,6 +357,17 @@ class SERes2NetBlock(nn.Layer):
             kernel_size=1,
             dilation=1,
             activation=nn.ReLU, ):
+        """Implementation of Squeeze-Extraction Res2Blocks in ECAPA-TDNN network model
+
+        Args:
+            in_channels (int): input channels or input data dimensions
+            out_channels (_type_): _description_
+            res2net_scale (int, optional): _description_. Defaults to 8.
+            se_channels (int, optional): _description_. Defaults to 128.
+            kernel_size (int, optional): _description_. Defaults to 1.
+            dilation (int, optional): _description_. Defaults to 1.
+            activation (_type_, optional): _description_. Defaults to nn.ReLU.
+        """
         super().__init__()
         self.out_channels = out_channels
         self.tdnn1 = TDNNBlock(
diff --git a/paddlespeech/vector/training/seeding.py b/paddlespeech/vector/training/seeding.py
new file mode 100644
index 00000000..0778a27d
--- /dev/null
+++ b/paddlespeech/vector/training/seeding.py
@@ -0,0 +1,28 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from paddlespeech.s2t.utils.log import Log
+
+logger = Log(__name__).getlog()
+import random
+
+import numpy as np
+import paddle
+
+
+def seed_everything(seed: int):
+    """Seed paddle, random and np.random to help reproductivity."""
+    paddle.seed(seed)
+    random.seed(seed)
+    np.random.seed(seed)
+    logger.info(f"Set the seed of paddle, random, np.random to {seed}.")

From 7db7eb8993a1dcc986de859f4b0f0f9cbf6b589f Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 7 Mar 2022 22:58:15 +0800
Subject: [PATCH 019/126] add extract audio embedding api, test=doc

---
 .../sv0/local/extract_speaker_embedding.py    | 115 ++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100644 examples/voxceleb/sv0/local/extract_speaker_embedding.py

diff --git a/examples/voxceleb/sv0/local/extract_speaker_embedding.py b/examples/voxceleb/sv0/local/extract_speaker_embedding.py
new file mode 100644
index 00000000..8eb24e1d
--- /dev/null
+++ b/examples/voxceleb/sv0/local/extract_speaker_embedding.py
@@ -0,0 +1,115 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import ast
+import os
+
+import numpy as np
+import paddle
+import paddle.nn.functional as F
+from paddle.io import BatchSampler
+from paddle.io import DataLoader
+from tqdm import tqdm
+
+from paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddleaudio.features.core import melspectrogram
+from paddleaudio.backends import load as load_audio
+from paddlespeech.vector.io.batch import feature_normalize
+from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.metrics import compute_eer
+from paddlespeech.vector.training.seeding import seed_everything
+
+logger = Log(__name__).getlog()
+
+# feat configuration
+cpu_feat_conf = {
+    'n_mels': 80,
+    'window_size': 400,  #ms
+    'hop_length': 160,  #ms
+}
+
+def extract_audio_embedding(args):
+    # stage 0: set the training device, cpu or gpu
+    paddle.set_device(args.device)
+    # set the random seed, it is a must for multiprocess training
+    seed_everything(args.seed)
+
+    # stage 1: build the dnn backbone model network
+    ##"channels": [1024, 1024, 1024, 1024, 3072],
+    model_conf = {
+        "input_size": 80,
+        "channels": [512, 512, 512, 512, 1536],
+        "kernel_sizes": [5, 3, 3, 3, 1],
+        "dilations": [1, 2, 3, 4, 1],
+        "attention_channels": 128,
+        "lin_neurons": 192,
+    }
+    ecapa_tdnn = EcapaTdnn(**model_conf)
+
+    # stage 2: load the pre-trained model
+    args.load_checkpoint = os.path.abspath(
+        os.path.expanduser(args.load_checkpoint))
+
+    # load model checkpoint to sid model
+    state_dict = paddle.load(
+        os.path.join(args.load_checkpoint, 'model.pdparams'))
+    model.set_state_dict(state_dict)
+    logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
+
+    # stage 3: we must set the model to eval mode
+    model.eval()
+    
+    # stage 4: read the audio data and extract the embedding
+    waveform, sr = load_audio(args.audio_path)
+    feat = melspectrogram(x=waveform, **cpu_feat_conf)
+    feat = paddle.to_tensor(feat).unsqueeze(0)
+    lengths = paddle.ones([1]) # in paddle inference model, the lengths is all one without padding
+    feat = feature_normalize(feat, mean_norm=True, std_norm=False)
+    embedding = ecapa_tdnn(feat, lengths
+                ).squeeze().numpy() # (1, emb_size, 1) -> (emb_size)
+
+    # stage 5: do global norm with external mean and std
+    # todo
+    return embedding
+
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--device',
+                        choices=['cpu', 'gpu'],
+                        default="gpu",
+                        help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--seed",
+                        default=0,
+                        type=int,
+                        help="random seed for paddle, numpy and python random package")
+    parser.add_argument("--load-checkpoint",
+                        type=str,
+                        default='',
+                        help="Directory to load model checkpoint to contiune trainning.")
+    parser.add_argument("--global-embedding-norm",
+                        type=str,
+                        default=None,
+                        help="Apply global normalization on speaker embeddings.")
+    parser.add_argument("--audio-path",
+                        default="./data/demo.wav",
+                        type=str,
+                        help="Single audio file path")
+    args = parser.parse_args()
+    # yapf: enable
+
+    extract_audio_embedding(args)

From 386ef3f161531d6e9c7bace4d1097f72f71be5e2 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Tue, 8 Mar 2022 16:50:04 +0800
Subject: [PATCH 020/126] add voxceleb augment unit test, test=doc

---
 paddlespeech/vector/io/augment.py |   3 +-
 tests/unit/vector/conftest.py     |  11 +++
 tests/unit/vector/test_augment.py | 138 ++++++++++++++++++++++++++++++
 3 files changed, 151 insertions(+), 1 deletion(-)
 create mode 100644 tests/unit/vector/conftest.py
 create mode 100644 tests/unit/vector/test_augment.py

diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
index d6bbc8a9..af7aeb22 100644
--- a/paddlespeech/vector/io/augment.py
+++ b/paddlespeech/vector/io/augment.py
@@ -178,7 +178,8 @@ class DropChunk(nn.Layer):
             # Update waveform
             if not self.noise_factor:
                 for j in range(drop_times[i]):
-                    dropped_waveform[i, start[j]:end[j]] = 0.0
+                    if start[j] < end[j]:
+                        dropped_waveform[i, start[j]:end[j]] = 0.0
             else:
                 # Uniform distribution of -2 to +2 * avg amplitude should
                 # preserve the average for normalization
diff --git a/tests/unit/vector/conftest.py b/tests/unit/vector/conftest.py
new file mode 100644
index 00000000..7cac519b
--- /dev/null
+++ b/tests/unit/vector/conftest.py
@@ -0,0 +1,11 @@
+def pytest_addoption(parser):
+    parser.addoption("--device", action="store", default="cpu")
+
+
+def pytest_generate_tests(metafunc):
+    # This is called for every test. Only get/set command line arguments
+    # if the argument is specified in the list of test "fixturenames".
+    option_value = metafunc.config.option.device
+    if "device" in metafunc.fixturenames and option_value is not None:
+        metafunc.parametrize("device", [option_value])
+
diff --git a/tests/unit/vector/test_augment.py b/tests/unit/vector/test_augment.py
new file mode 100644
index 00000000..21d75bb3
--- /dev/null
+++ b/tests/unit/vector/test_augment.py
@@ -0,0 +1,138 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+import numpy as np
+import paddle
+import paddle.nn as nn
+import paddle.nn.functional as F
+from paddle.io import BatchSampler
+from paddle.io import DataLoader
+from paddle.io import Dataset
+
+
+def test_add_noise(tmpdir, device):
+    paddle.device.set_device(device)
+    from paddlespeech.vector.io.augment import AddNoise
+
+    test_waveform = paddle.sin(
+        paddle.arange(16000.0, dtype="float32")).unsqueeze(0)
+    test_noise = paddle.cos(
+        paddle.arange(16000.0, dtype="float32")).unsqueeze(0)
+    wav_lens = paddle.ones([1], dtype="float32")
+
+    # Edge cases
+    no_noise = AddNoise(mix_prob=0.0)
+    assert no_noise(test_waveform, wav_lens).allclose(test_waveform)
+
+
+def test_speed_perturb(device):
+    paddle.device.set_device(device)
+    from paddlespeech.vector.io.augment import SpeedPerturb
+
+    test_waveform = paddle.sin(
+        paddle.arange(16000.0, dtype="float32")).unsqueeze(0)
+
+    # Edge cases
+    no_perturb = SpeedPerturb(16000, perturb_prob=0.0)
+    assert no_perturb(test_waveform).allclose(test_waveform)
+    no_perturb = SpeedPerturb(16000, speeds=[100])
+    assert no_perturb(test_waveform).allclose(test_waveform)
+
+    # # Half speed
+    half_speed = SpeedPerturb(16000, speeds=[50])
+    assert half_speed(test_waveform).allclose(test_waveform[:, ::2], atol=3e-1)
+
+
+def test_babble(device):
+    paddle.device.set_device(device)
+    from paddlespeech.vector.io.augment import AddBabble
+
+    test_waveform = paddle.stack(
+        (paddle.sin(paddle.arange(16000.0, dtype="float32")),
+         paddle.cos(paddle.arange(16000.0, dtype="float32")), ))
+    lengths = paddle.ones([2])
+
+    # Edge cases
+    no_babble = AddBabble(mix_prob=0.0)
+    assert no_babble(test_waveform, lengths).allclose(test_waveform)
+    no_babble = AddBabble(speaker_count=1, snr_low=1000, snr_high=1000)
+    assert no_babble(test_waveform, lengths).allclose(test_waveform)
+
+    # One babbler just averages the two speakers
+    babble = AddBabble(speaker_count=1).to(device)
+    expected = (test_waveform + test_waveform.roll(1, 0)) / 2
+    assert babble(test_waveform, lengths).allclose(expected, atol=1e-4)
+
+
+def test_drop_freq(device):
+    paddle.device.set_device(device)
+    from paddlespeech.vector.io.augment import DropFreq
+
+    test_waveform = paddle.sin(
+        paddle.arange(16000.0, dtype="float32")).unsqueeze(0)
+
+    # Edge cases
+    no_drop = DropFreq(drop_prob=0.0)
+    assert no_drop(test_waveform).allclose(test_waveform)
+    no_drop = DropFreq(drop_count_low=0, drop_count_high=0)
+    assert no_drop(test_waveform).allclose(test_waveform)
+
+    # Check case where frequency range *does not* include signal frequency
+    drop_diff_freq = DropFreq(drop_freq_low=0.5, drop_freq_high=0.9)
+    assert drop_diff_freq(test_waveform).allclose(test_waveform, atol=1e-1)
+
+    # Check case where frequency range *does* include signal frequency
+    drop_same_freq = DropFreq(drop_freq_low=0.28, drop_freq_high=0.28)
+    assert drop_same_freq(test_waveform).allclose(
+        paddle.zeros([1, 16000]), atol=4e-1)
+
+
+def test_drop_chunk(device):
+    paddle.device.set_device(device)
+    from paddlespeech.vector.io.augment import DropChunk
+
+    test_waveform = paddle.sin(
+        paddle.arange(16000.0, dtype="float32")).unsqueeze(0)
+    lengths = paddle.ones([1])
+
+    # Edge cases
+    no_drop = DropChunk(drop_prob=0.0)
+    assert no_drop(test_waveform, lengths).allclose(test_waveform)
+    no_drop = DropChunk(drop_length_low=0, drop_length_high=0)
+    assert no_drop(test_waveform, lengths).allclose(test_waveform)
+    no_drop = DropChunk(drop_count_low=0, drop_count_high=0)
+    assert no_drop(test_waveform, lengths).allclose(test_waveform)
+    no_drop = DropChunk(drop_start=0, drop_end=0)
+    assert no_drop(test_waveform, lengths).allclose(test_waveform)
+
+    # Specify all parameters to ensure it is deterministic
+    dropper = DropChunk(
+        drop_length_low=100,
+        drop_length_high=100,
+        drop_count_low=1,
+        drop_count_high=1,
+        drop_start=100,
+        drop_end=200,
+        noise_factor=0.0, )
+    expected_waveform = test_waveform.clone()
+    expected_waveform[:, 100:200] = 0.0
+
+    assert dropper(test_waveform, lengths).allclose(expected_waveform)
+
+    # Make sure amplitude is similar before and after
+    dropper = DropChunk(noise_factor=1.0)
+    drop_amplitude = dropper(test_waveform, lengths).abs().mean()
+    orig_amplitude = test_waveform.abs().mean()
+    assert drop_amplitude.allclose(orig_amplitude, atol=1e-2)

From 14efbf5b15e45299f16eab594fcd1155a8b74742 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Tue, 8 Mar 2022 21:04:52 +0800
Subject: [PATCH 021/126] check extract embedding result, test=doc

---
 .../sv0/local/extract_speaker_embedding.py    | 28 +++++++++++----
 .../sv0/local/speaker_verification_cosine.py  |  4 +--
 examples/voxceleb/sv0/local/train.py          | 14 ++++----
 examples/voxceleb/sv0/run.sh                  | 35 +++++++++++++++----
 paddleaudio/datasets/voxceleb.py              |  9 ++---
 paddlespeech/vector/io/batch.py               | 17 ++++++---
 6 files changed, 76 insertions(+), 31 deletions(-)

diff --git a/examples/voxceleb/sv0/local/extract_speaker_embedding.py b/examples/voxceleb/sv0/local/extract_speaker_embedding.py
index 8eb24e1d..e7dad140 100644
--- a/examples/voxceleb/sv0/local/extract_speaker_embedding.py
+++ b/examples/voxceleb/sv0/local/extract_speaker_embedding.py
@@ -22,11 +22,11 @@ from paddle.io import BatchSampler
 from paddle.io import DataLoader
 from tqdm import tqdm
 
+from paddleaudio.backends import load as load_audio
 from paddleaudio.datasets.voxceleb import VoxCeleb1
 from paddleaudio.features.core import melspectrogram
-from paddleaudio.backends import load as load_audio
-from paddlespeech.vector.io.batch import feature_normalize
 from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.io.batch import feature_normalize
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
 from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 from paddlespeech.vector.training.metrics import compute_eer
@@ -41,6 +41,7 @@ cpu_feat_conf = {
     'hop_length': 160,  #ms
 }
 
+
 def extract_audio_embedding(args):
     # stage 0: set the training device, cpu or gpu
     paddle.set_device(args.device)
@@ -59,6 +60,8 @@ def extract_audio_embedding(args):
     }
     ecapa_tdnn = EcapaTdnn(**model_conf)
 
+    # stage4: build the speaker verification train instance with backbone model
+    model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=1211)
     # stage 2: load the pre-trained model
     args.load_checkpoint = os.path.abspath(
         os.path.expanduser(args.load_checkpoint))
@@ -71,18 +74,29 @@ def extract_audio_embedding(args):
 
     # stage 3: we must set the model to eval mode
     model.eval()
-    
+
     # stage 4: read the audio data and extract the embedding
+    # wavform is one dimension numpy array 
     waveform, sr = load_audio(args.audio_path)
+
+    # feat type is numpy array, whose shape is [dim, time]
+    # we need convert the audio feat to one-batch shape [batch, dim, time], where the batch is one
+    # so the final shape is [1, dim, time]
     feat = melspectrogram(x=waveform, **cpu_feat_conf)
     feat = paddle.to_tensor(feat).unsqueeze(0)
-    lengths = paddle.ones([1]) # in paddle inference model, the lengths is all one without padding
-    feat = feature_normalize(feat, mean_norm=True, std_norm=False)
-    embedding = ecapa_tdnn(feat, lengths
-                ).squeeze().numpy() # (1, emb_size, 1) -> (emb_size)
+
+    # in inference period, the lengths is all one without padding
+    lengths = paddle.ones([1])
+    feat = feature_normalize(
+        feat, mean_norm=True, std_norm=False, convert_to_numpy=True)
+
+    # model backbone network forward the feats and get the embedding
+    embedding = model.backbone(
+        feat, lengths).squeeze().numpy()  # (1, emb_size, 1) -> (emb_size)
 
     # stage 5: do global norm with external mean and std
     # todo
+    # np.save("audio-embedding", embedding)
     return embedding
 
 
diff --git a/examples/voxceleb/sv0/local/speaker_verification_cosine.py b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
index b0adcf66..417e8aa3 100644
--- a/examples/voxceleb/sv0/local/speaker_verification_cosine.py
+++ b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
@@ -120,7 +120,7 @@ def main(args):
         **cpu_feat_conf)
     enrol_sampler = BatchSampler(
         enrol_ds, batch_size=args.batch_size,
-        shuffle=False)  # Shuffle to make embedding normalization more robust.
+        shuffle=True)  # Shuffle to make embedding normalization more robust.
     enrol_loader = DataLoader(enrol_ds,
                     batch_sampler=enrol_sampler,
                     collate_fn=lambda x: feature_normalize(
@@ -136,7 +136,7 @@ def main(args):
         **cpu_feat_conf)
 
     test_sampler = BatchSampler(
-        test_ds, batch_size=args.batch_size, shuffle=False)
+        test_ds, batch_size=args.batch_size, shuffle=True)
     test_loader = DataLoader(test_ds,
                             batch_sampler=test_sampler,
                             collate_fn=lambda x: feature_normalize(
diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
index 745d5eab..3fe67c8e 100644
--- a/examples/voxceleb/sv0/local/train.py
+++ b/examples/voxceleb/sv0/local/train.py
@@ -56,10 +56,10 @@ def main(args):
     # set the random seed, it is a must for multiprocess training
     seed_everything(args.seed)
 
-    # stage2: data prepare, such vox1 and vox2 data, and augment data and pipline
+    # stage2: data prepare, such vox1 and vox2 data, and augment noise data and pipline
     # note: some cmd must do in rank==0, so wo will refactor the data prepare code
-    train_ds = VoxCeleb1('train', target_dir=args.data_dir)
-    dev_ds = VoxCeleb1('dev', target_dir=args.data_dir)
+    train_dataset = VoxCeleb1('train', target_dir=args.data_dir)
+    dev_dataset = VoxCeleb1('dev', target_dir=args.data_dir)
 
     if args.augment:
         augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
@@ -123,9 +123,9 @@ def main(args):
 
     # stage8: we build the batch sampler for paddle.DataLoader
     train_sampler = DistributedBatchSampler(
-        train_ds, batch_size=args.batch_size, shuffle=True, drop_last=False)
+        train_dataset, batch_size=args.batch_size, shuffle=True, drop_last=False)
     train_loader = DataLoader(
-        train_ds,
+        train_dataset,
         batch_sampler=train_sampler,
         num_workers=args.num_workers,
         collate_fn=waveform_collate_fn,
@@ -216,12 +216,12 @@ def main(args):
 
             # stage 9-12: construct the valid dataset dataloader
             dev_sampler = BatchSampler(
-                dev_ds,
+                dev_dataset,
                 batch_size=args.batch_size // 4,
                 shuffle=False,
                 drop_last=False)
             dev_loader = DataLoader(
-                dev_ds,
+                dev_dataset,
                 batch_sampler=dev_sampler,
                 collate_fn=waveform_collate_fn,
                 num_workers=args.num_workers,
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index c3b31ce5..34a1cbd4 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -3,6 +3,8 @@
 set -e
 
 #######################################################################
+# stage 0: data prepare, including voxceleb1 download and generate {train,dev,enroll,test}.csv
+#          voxceleb2 data is m4a format, so we need user to convert the m4a to wav yourselves as described in Readme.md
 # stage 1: train the speaker identification model
 # stage 2: test speaker identification 
 # stage 3: extract the training embeding to train the LDA and PLDA
@@ -12,23 +14,42 @@ set -e
 # default the dataset is the ~/.paddleaudio/
 # export PPAUDIO_HOME=
 
-stage=2
-dir=data/                     # data directory
-exp_dir=exp/ecapa-tdnn/       # experiment directory
+stage=0
+dir=data.bak/                     # data directory
+exp_dir=exp/ecapa-tdnn/           # experiment directory
 mkdir -p ${dir}
+mkdir -p ${exp_dir}
+
+# if [ $stage -le 0 ]; then 
+#      # stage 0: data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
+#      # todo
+# fi 
 
 if [ $stage -le 1 ]; then
      # stage 1: train the speaker identification model
      python3 \
           -m paddle.distributed.launch --gpus=0,1,2,3 \
-          local/train.py --device "gpu" --checkpoint-dir ${exp_dir} \
-          --save-freq 10 --data-dir ${dir} --batch-size 256 --epochs 60
+          local/train.py --device "gpu" --checkpoint-dir ${exp_dir} --augment \
+          --save-freq 10 --data-dir ${dir} --batch-size 64 --epochs 100
 fi
 
 if [ $stage -le 2 ]; then
      # stage 1: train the speaker identification model
+     # you can set the variable PPAUDIO_HOME to specifiy the downloaded the vox1 and vox2 dataset
+     python3 \
+          local/speaker_verification_cosine.py\
+          --batch-size 4 --data-dir ${dir} --load-checkpoint ${exp_dir}/epoch_10/
+fi
+
+if [ $stage -le 3 ]; then
+     # stage 1: train the speaker identification model
+     # you can set the variable PPAUDIO_HOME to specifiy the downloaded the vox1 and vox2 dataset
      python3 \
-          local/speaker_verification_cosine.py \
-          --load-checkpoint ${exp_dir}/epoch_40/
+          local/extract_speaker_embedding.py\
+          --audio-path "demo/csv/00001.wav" --load-checkpoint ${exp_dir}/epoch_60/
 fi
 
+# if [ $stage -le 3 ]; then
+#      # stage 2: extract the training embeding to train the LDA and PLDA
+#      # todo: extract the training embedding
+# fi 
diff --git a/paddleaudio/datasets/voxceleb.py b/paddleaudio/datasets/voxceleb.py
index 28f6dfc6..c97e825e 100644
--- a/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/datasets/voxceleb.py
@@ -28,7 +28,7 @@ from paddleaudio.backends import load as load_audio
 from paddleaudio.datasets.dataset import feat_funcs
 from paddleaudio.utils import DATA_HOME
 from paddleaudio.utils import decompress
-from paddleaudio.utils import download_and_decompress
+from paddlespeech.vector.utils.download import download_and_decompress
 from paddlespeech.s2t.utils.log import Log
 from utils.utility import download
 from utils.utility import unpack
@@ -106,13 +106,14 @@ class VoxCeleb1(Dataset):
         self.chunk_duration = chunk_duration
         self.split_ratio = split_ratio
         self.target_dir = target_dir if target_dir else self.base_path
-        self.csv_path = os.path.join(
+        VoxCeleb1.csv_path = os.path.join(
             target_dir, 'csv') if target_dir else os.path.join(self.base_path,
                                                                'csv')
-        self.meta_path = os.path.join(
+        VoxCeleb1.meta_path = os.path.join(
             target_dir, 'meta') if target_dir else os.path.join(self.base_path,
                                                                 'meta')
-        self.veri_test_file = os.path.join(self.meta_path, 'veri_test2.txt')
+        VoxCeleb1.veri_test_file = os.path.join(self.meta_path,
+                                                'veri_test2.txt')
         # self._data = self._get_data()[:1000]  # KP: Small dataset test.
         self._data = self._get_data()
         super(VoxCeleb1, self).__init__()
diff --git a/paddlespeech/vector/io/batch.py b/paddlespeech/vector/io/batch.py
index 9db615f6..879cde3a 100644
--- a/paddlespeech/vector/io/batch.py
+++ b/paddlespeech/vector/io/batch.py
@@ -24,10 +24,19 @@ def waveform_collate_fn(batch):
 
 def feature_normalize(feats: paddle.Tensor,
                       mean_norm: bool=True,
-                      std_norm: bool=True):
+                      std_norm: bool=True,
+                      convert_to_numpy: bool=False):
     # Features normalization if needed
-    mean = feats.mean(axis=-1, keepdim=True) if mean_norm else 0
-    std = feats.std(axis=-1, keepdim=True) if std_norm else 1
-    feats = (feats - mean) / std
+    # numpy.mean is a little with paddle.mean about 1e-6
+    if convert_to_numpy:
+        feats_np = feats.numpy()
+        mean = feats_np.mean(axis=-1, keepdims=True) if mean_norm else 0
+        std = feats_np.std(axis=-1, keepdims=True) if std_norm else 1
+        feats_np = (feats_np - mean) / std
+        feats = paddle.to_tensor(feats_np, dtype=feats.dtype)
+    else:
+        mean = feats.mean(axis=-1, keepdim=True) if mean_norm else 0
+        std = feats.std(axis=-1, keepdim=True) if std_norm else 1
+        feats = (feats - mean) / std
 
     return feats

From 60d73bb7bd5af81001c0b90837fc3fb01cd65da9 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 9 Mar 2022 12:10:02 +0800
Subject: [PATCH 022/126] add state 0 to  prepare the voxcele data and augment
 data

---
 examples/voxceleb/README.md                 | 53 ++++++++++++++++++
 examples/voxceleb/sv0/local/data_prepare.py | 60 +++++++++++++++++++++
 examples/voxceleb/sv0/run.sh                |  8 +--
 paddleaudio/datasets/rirs_noises.py         |  5 +-
 paddleaudio/datasets/voxceleb.py            | 37 +++++++------
 paddlespeech/vector/io/augment.py           |  2 +-
 6 files changed, 142 insertions(+), 23 deletions(-)
 create mode 100644 examples/voxceleb/sv0/local/data_prepare.py

diff --git a/examples/voxceleb/README.md b/examples/voxceleb/README.md
index 2c8ad138..59fb491c 100644
--- a/examples/voxceleb/README.md
+++ b/examples/voxceleb/README.md
@@ -6,3 +6,56 @@ sv0 - speaker verfication with softmax backend etc, all python code
 
 sv1 - dependence on kaldi, speaker verfication with plda/sc backend, 
       more info refer to the sv1/readme.txt
+
+
+## VoxCeleb2 preparation
+
+VoxCeleb2 audio files are released in m4a format. All the VoxCeleb2 m4a audio files must be converted in wav files before feeding them in PaddleSpeech. 
+Please, follow these steps to prepare the dataset correctly:
+
+1. Download Voxceleb2.
+You can find download instructions here: http://www.robots.ox.ac.uk/~vgg/data/voxceleb/
+
+2. Convert .m4a to wav
+VoxCeleb2 stores files with the m4a audio format. To use them in PaddleSpeech,  you have to convert all the m4a audio files into wav files.
+
+``` shell
+ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 %s
+```
+
+``` shell
+# copy this to root directory of data and 
+# chmod a+x convert.sh
+# ./convert.sh
+# https://unix.stackexchange.com/questions/103920/parallelize-a-bash-for-loop
+
+open_sem(){
+    mkfifo pipe-$$
+    exec 3<>pipe-$$
+    rm pipe-$$
+    local i=$1
+    for((;i>0;i--)); do
+        printf %s 000 >&3
+    done
+}
+run_with_lock(){
+    local x
+    read -u 3 -n 3 x && ((0==x)) || exit $x
+    (
+     ( "$@"; )
+    printf '%.3d' $? >&3
+    )&
+}
+
+N=32 # number of vCPU
+open_sem $N
+for f in $(find . -name "*.m4a"); do
+    run_with_lock ffmpeg -loglevel panic -i "$f" -ar 16000 "${f%.*}.wav"
+done
+```
+
+You can do the conversion using ffmpeg  https://gist.github.com/seungwonpark/4f273739beef2691cd53b5c39629d830). This operation might take several hours and should be only once.
+
+3. Put all the wav files in a folder called `wav`. You should have something like `voxceleb2/wav/id*/*.wav` (e.g, `voxceleb2/wav/id00012/21Uxsk56VDQ/00001.wav`)
+
+4. 
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/local/data_prepare.py b/examples/voxceleb/sv0/local/data_prepare.py
new file mode 100644
index 00000000..ca707fc2
--- /dev/null
+++ b/examples/voxceleb/sv0/local/data_prepare.py
@@ -0,0 +1,60 @@
+import argparse
+import os
+
+import numpy as np
+import paddle
+from paddle.io import BatchSampler
+from paddle.io import DataLoader
+from paddle.io import DistributedBatchSampler
+
+from paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddleaudio.features.core import melspectrogram
+from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.io.augment import build_augment_pipeline
+from paddlespeech.vector.io.augment import waveform_augment
+from paddlespeech.vector.io.batch import feature_normalize
+from paddlespeech.vector.io.batch import waveform_collate_fn
+from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
+from paddlespeech.vector.modules.loss import AdditiveAngularMargin
+from paddlespeech.vector.modules.loss import LogSoftmaxWrapper
+from paddlespeech.vector.modules.lr import CyclicLRScheduler
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.seeding import seed_everything
+from paddlespeech.vector.utils.time import Timer
+
+logger = Log(__name__).getlog()
+
+def main(args):
+    # stage0: set the cpu device, all data prepare process will be done in cpu mode
+    paddle.set_device("cpu")
+    # set the random seed, it is a must for multiprocess training
+    seed_everything(args.seed)
+
+    # stage 1: generate the voxceleb csv file
+    # Note: this may occurs c++ execption, but the program will execute fine
+    # so we can ignore the execption 
+    train_dataset = VoxCeleb1('train', target_dir=args.data_dir)
+    dev_dataset = VoxCeleb1('dev', target_dir=args.data_dir)
+
+    # stage 2: generate the augment noise csv file
+    if args.augment:
+        augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument("--seed",
+                        default=0,
+                        type=int,
+                        help="random seed for paddle, numpy and python random package")
+    parser.add_argument("--data-dir",
+                        default="./data/",
+                        type=str,
+                        help="data directory")
+    parser.add_argument("--augment",
+                        action="store_true",
+                        default=False,
+                        help="Apply audio augments.")
+    args = parser.parse_args()
+    # yapf: enable
+    main(args)                    
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index 34a1cbd4..7ad3a36f 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -20,10 +20,10 @@ exp_dir=exp/ecapa-tdnn/           # experiment directory
 mkdir -p ${dir}
 mkdir -p ${exp_dir}
 
-# if [ $stage -le 0 ]; then 
-#      # stage 0: data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
-#      # todo
-# fi 
+if [ $stage -le 0 ]; then 
+     # stage 0: data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
+     python3 local/data_prepare.py --data-dir ${dir} --augment
+fi 
 
 if [ $stage -le 1 ]; then
      # stage 1: train the speaker identification model
diff --git a/paddleaudio/datasets/rirs_noises.py b/paddleaudio/datasets/rirs_noises.py
index fa9e7f09..6af9fd9d 100644
--- a/paddleaudio/datasets/rirs_noises.py
+++ b/paddleaudio/datasets/rirs_noises.py
@@ -69,8 +69,9 @@ class OpenRIRNoise(Dataset):
         self.random_chunk = random_chunk
         self.chunk_duration = chunk_duration
 
-        self.csv_path = os.path.join(target_dir, "open_rir_noise",
-                                     "csv") if target_dir else self.csv_path
+        OpenRIRNoise.csv_path = os.path.join(
+            target_dir, "open_rir_noise",
+            "csv") if target_dir else self.csv_path
         self._data = self._get_data()
         super(OpenRIRNoise, self).__init__()
 
diff --git a/paddleaudio/datasets/voxceleb.py b/paddleaudio/datasets/voxceleb.py
index c97e825e..0011340e 100644
--- a/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/datasets/voxceleb.py
@@ -16,6 +16,7 @@ import csv
 import glob
 import os
 import random
+from multiprocessing import cpu_count
 from typing import Dict
 from typing import List
 from typing import Tuple
@@ -28,8 +29,8 @@ from paddleaudio.backends import load as load_audio
 from paddleaudio.datasets.dataset import feat_funcs
 from paddleaudio.utils import DATA_HOME
 from paddleaudio.utils import decompress
-from paddlespeech.vector.utils.download import download_and_decompress
 from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.utils.download import download_and_decompress
 from utils.utility import download
 from utils.utility import unpack
 
@@ -105,14 +106,15 @@ class VoxCeleb1(Dataset):
         self.random_chunk = random_chunk
         self.chunk_duration = chunk_duration
         self.split_ratio = split_ratio
-        self.target_dir = target_dir if target_dir else self.base_path
+        self.target_dir = target_dir if target_dir else VoxCeleb1.base_path
+
+        # if we set the target dir, we will change the vox data info data from base path to target dir
         VoxCeleb1.csv_path = os.path.join(
-            target_dir, 'csv') if target_dir else os.path.join(self.base_path,
-                                                               'csv')
+            target_dir, "voxceleb", 'csv') if target_dir else VoxCeleb1.csv_path
         VoxCeleb1.meta_path = os.path.join(
-            target_dir, 'meta') if target_dir else os.path.join(self.base_path,
-                                                                'meta')
-        VoxCeleb1.veri_test_file = os.path.join(self.meta_path,
+            target_dir, "voxceleb",
+            'meta') if target_dir else VoxCeleb1.meta_path
+        VoxCeleb1.veri_test_file = os.path.join(VoxCeleb1.meta_path,
                                                 'veri_test2.txt')
         # self._data = self._get_data()[:1000]  # KP: Small dataset test.
         self._data = self._get_data()
@@ -255,8 +257,9 @@ class VoxCeleb1(Dataset):
                      split_chunks: bool=True):
         logger.info(f'Generating csv: {output_file}')
         header = ["id", "duration", "wav", "start", "stop", "spk_id"]
-
-        with Pool(64) as p:
+        # Note: this may occurs c++ execption, but the program will execute fine
+        # so we can ignore the execption 
+        with Pool(cpu_count()) as p:
             infos = list(
                 tqdm(
                     p.imap(lambda x: self._get_audio_info(x, split_chunks),
@@ -277,20 +280,20 @@ class VoxCeleb1(Dataset):
     def prepare_data(self):
         # Audio of speakers in veri_test_file should not be included in training set.
         logger.info("start to prepare the data csv file")
-        enrol_files = set()
+        enroll_files = set()
         test_files = set()
         # get the enroll and test audio file path
         with open(self.veri_test_file, 'r') as f:
             for line in f.readlines():
                 _, enrol_file, test_file = line.strip().split(' ')
-                enrol_files.add(os.path.join(self.wav_path, enrol_file))
+                enroll_files.add(os.path.join(self.wav_path, enrol_file))
                 test_files.add(os.path.join(self.wav_path, test_file))
-            enrol_files = sorted(enrol_files)
+            enroll_files = sorted(enroll_files)
             test_files = sorted(test_files)
 
         # get the enroll and test speakers
         test_spks = set()
-        for file in (enrol_files + test_files):
+        for file in (enroll_files + test_files):
             spk = file.split('/wav/')[1].split('/')[0]
             test_spks.add(spk)
 
@@ -306,8 +309,9 @@ class VoxCeleb1(Dataset):
                 speakers.add(spk)
                 audio_files.append(file)
 
-        logger.info("start to generate the {}".format(
-            os.path.join(self.meta_path, 'spk_id2label.txt')))
+        logger.info(
+            f"start to generate the {os.path.join(self.meta_path, 'spk_id2label.txt')}"
+        )
         # encode the train and dev speakers label to spk_id2label.txt
         with open(os.path.join(self.meta_path, 'spk_id2label.txt'), 'w') as f:
             for label, spk_id in enumerate(
@@ -323,8 +327,9 @@ class VoxCeleb1(Dataset):
 
         self.generate_csv(train_files, os.path.join(self.csv_path, 'train.csv'))
         self.generate_csv(dev_files, os.path.join(self.csv_path, 'dev.csv'))
+
         self.generate_csv(
-            enrol_files,
+            enroll_files,
             os.path.join(self.csv_path, 'enrol.csv'),
             split_chunks=False)
         self.generate_csv(
diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
index af7aeb22..366c0cff 100644
--- a/paddlespeech/vector/io/augment.py
+++ b/paddlespeech/vector/io/augment.py
@@ -840,7 +840,7 @@ def build_augment_pipeline(target_dir=None) -> List[paddle.nn.Layer]:
     """
     logger.info("start to build the augment pipeline")
     noise_dataset = OpenRIRNoise('noise', target_dir=target_dir)
-    rir_dataset = OpenRIRNoise('rir')
+    rir_dataset = OpenRIRNoise('rir', target_dir=target_dir)
 
     wavedrop = TimeDomainSpecAugment(
         sample_rate=16000,

From 0e87037f2c9fc67a87c665e7a2f1c8058666646b Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 9 Mar 2022 14:03:47 +0800
Subject: [PATCH 023/126] refactor to compilance paddleaudio

---
 examples/voxceleb/sv0/local/data_prepare.py   |  15 +-
 .../sv0/local/extract_speaker_embedding.py    | 129 -------
 .../sv0/local/speaker_verification_cosine.py  | 264 --------------
 examples/voxceleb/sv0/local/train.py          | 326 ------------------
 examples/voxceleb/sv0/path.sh                 |   3 +
 examples/voxceleb/sv0/run.sh                  |  12 +-
 paddleaudio/paddleaudio/datasets/__init__.py  |   2 +
 .../{ => paddleaudio}/datasets/rirs_noises.py |  10 +-
 .../{ => paddleaudio}/datasets/voxceleb.py    |  12 +-
 paddleaudio/paddleaudio/metric/__init__.py    |   1 +
 .../paddleaudio/metric/eer.py                 |   0
 paddlespeech/vector/io/augment.py             |   4 +-
 paddlespeech/vector/io/batch.py               |  38 ++
 .../{modules/lr.py => training/scheduler.py}  |   0
 14 files changed, 63 insertions(+), 753 deletions(-)
 delete mode 100644 examples/voxceleb/sv0/local/extract_speaker_embedding.py
 delete mode 100644 examples/voxceleb/sv0/local/speaker_verification_cosine.py
 delete mode 100644 examples/voxceleb/sv0/local/train.py
 rename paddleaudio/{ => paddleaudio}/datasets/rirs_noises.py (97%)
 rename paddleaudio/{ => paddleaudio}/datasets/voxceleb.py (97%)
 rename paddlespeech/vector/training/metrics.py => paddleaudio/paddleaudio/metric/eer.py (100%)
 rename paddlespeech/vector/{modules/lr.py => training/scheduler.py} (100%)

diff --git a/examples/voxceleb/sv0/local/data_prepare.py b/examples/voxceleb/sv0/local/data_prepare.py
index ca707fc2..1a0a6392 100644
--- a/examples/voxceleb/sv0/local/data_prepare.py
+++ b/examples/voxceleb/sv0/local/data_prepare.py
@@ -3,24 +3,11 @@ import os
 
 import numpy as np
 import paddle
-from paddle.io import BatchSampler
-from paddle.io import DataLoader
-from paddle.io import DistributedBatchSampler
 
-from paddleaudio.datasets.voxceleb import VoxCeleb1
-from paddleaudio.features.core import melspectrogram
+from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb1
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.augment import build_augment_pipeline
-from paddlespeech.vector.io.augment import waveform_augment
-from paddlespeech.vector.io.batch import feature_normalize
-from paddlespeech.vector.io.batch import waveform_collate_fn
-from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
-from paddlespeech.vector.modules.loss import AdditiveAngularMargin
-from paddlespeech.vector.modules.loss import LogSoftmaxWrapper
-from paddlespeech.vector.modules.lr import CyclicLRScheduler
-from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 from paddlespeech.vector.training.seeding import seed_everything
-from paddlespeech.vector.utils.time import Timer
 
 logger = Log(__name__).getlog()
 
diff --git a/examples/voxceleb/sv0/local/extract_speaker_embedding.py b/examples/voxceleb/sv0/local/extract_speaker_embedding.py
deleted file mode 100644
index e7dad140..00000000
--- a/examples/voxceleb/sv0/local/extract_speaker_embedding.py
+++ /dev/null
@@ -1,129 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import ast
-import os
-
-import numpy as np
-import paddle
-import paddle.nn.functional as F
-from paddle.io import BatchSampler
-from paddle.io import DataLoader
-from tqdm import tqdm
-
-from paddleaudio.backends import load as load_audio
-from paddleaudio.datasets.voxceleb import VoxCeleb1
-from paddleaudio.features.core import melspectrogram
-from paddlespeech.s2t.utils.log import Log
-from paddlespeech.vector.io.batch import feature_normalize
-from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
-from paddlespeech.vector.modules.sid_model import SpeakerIdetification
-from paddlespeech.vector.training.metrics import compute_eer
-from paddlespeech.vector.training.seeding import seed_everything
-
-logger = Log(__name__).getlog()
-
-# feat configuration
-cpu_feat_conf = {
-    'n_mels': 80,
-    'window_size': 400,  #ms
-    'hop_length': 160,  #ms
-}
-
-
-def extract_audio_embedding(args):
-    # stage 0: set the training device, cpu or gpu
-    paddle.set_device(args.device)
-    # set the random seed, it is a must for multiprocess training
-    seed_everything(args.seed)
-
-    # stage 1: build the dnn backbone model network
-    ##"channels": [1024, 1024, 1024, 1024, 3072],
-    model_conf = {
-        "input_size": 80,
-        "channels": [512, 512, 512, 512, 1536],
-        "kernel_sizes": [5, 3, 3, 3, 1],
-        "dilations": [1, 2, 3, 4, 1],
-        "attention_channels": 128,
-        "lin_neurons": 192,
-    }
-    ecapa_tdnn = EcapaTdnn(**model_conf)
-
-    # stage4: build the speaker verification train instance with backbone model
-    model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=1211)
-    # stage 2: load the pre-trained model
-    args.load_checkpoint = os.path.abspath(
-        os.path.expanduser(args.load_checkpoint))
-
-    # load model checkpoint to sid model
-    state_dict = paddle.load(
-        os.path.join(args.load_checkpoint, 'model.pdparams'))
-    model.set_state_dict(state_dict)
-    logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
-
-    # stage 3: we must set the model to eval mode
-    model.eval()
-
-    # stage 4: read the audio data and extract the embedding
-    # wavform is one dimension numpy array 
-    waveform, sr = load_audio(args.audio_path)
-
-    # feat type is numpy array, whose shape is [dim, time]
-    # we need convert the audio feat to one-batch shape [batch, dim, time], where the batch is one
-    # so the final shape is [1, dim, time]
-    feat = melspectrogram(x=waveform, **cpu_feat_conf)
-    feat = paddle.to_tensor(feat).unsqueeze(0)
-
-    # in inference period, the lengths is all one without padding
-    lengths = paddle.ones([1])
-    feat = feature_normalize(
-        feat, mean_norm=True, std_norm=False, convert_to_numpy=True)
-
-    # model backbone network forward the feats and get the embedding
-    embedding = model.backbone(
-        feat, lengths).squeeze().numpy()  # (1, emb_size, 1) -> (emb_size)
-
-    # stage 5: do global norm with external mean and std
-    # todo
-    # np.save("audio-embedding", embedding)
-    return embedding
-
-
-if __name__ == "__main__":
-    # yapf: disable
-    parser = argparse.ArgumentParser(__doc__)
-    parser.add_argument('--device',
-                        choices=['cpu', 'gpu'],
-                        default="gpu",
-                        help="Select which device to train model, defaults to gpu.")
-    parser.add_argument("--seed",
-                        default=0,
-                        type=int,
-                        help="random seed for paddle, numpy and python random package")
-    parser.add_argument("--load-checkpoint",
-                        type=str,
-                        default='',
-                        help="Directory to load model checkpoint to contiune trainning.")
-    parser.add_argument("--global-embedding-norm",
-                        type=str,
-                        default=None,
-                        help="Apply global normalization on speaker embeddings.")
-    parser.add_argument("--audio-path",
-                        default="./data/demo.wav",
-                        type=str,
-                        help="Single audio file path")
-    args = parser.parse_args()
-    # yapf: enable
-
-    extract_audio_embedding(args)
diff --git a/examples/voxceleb/sv0/local/speaker_verification_cosine.py b/examples/voxceleb/sv0/local/speaker_verification_cosine.py
deleted file mode 100644
index 417e8aa3..00000000
--- a/examples/voxceleb/sv0/local/speaker_verification_cosine.py
+++ /dev/null
@@ -1,264 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import ast
-import os
-
-import numpy as np
-import paddle
-import paddle.nn.functional as F
-from paddle.io import BatchSampler
-from paddle.io import DataLoader
-from tqdm import tqdm
-
-from paddleaudio.datasets.voxceleb import VoxCeleb1
-from paddlespeech.s2t.utils.log import Log
-from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
-from paddlespeech.vector.modules.sid_model import SpeakerIdetification
-from paddlespeech.vector.training.metrics import compute_eer
-from paddlespeech.vector.training.seeding import seed_everything
-
-logger = Log(__name__).getlog()
-
-
-def pad_right_2d(x, target_length, axis=-1, mode='constant', **kwargs):
-    x = np.asarray(x)
-    assert len(
-        x.shape) == 2, f'Only 2D arrays supported, but got shape: {x.shape}'
-
-    w = target_length - x.shape[axis]
-    assert w >= 0, f'Target length {target_length} is less than origin length {x.shape[axis]}'
-
-    if axis == 0:
-        pad_width = [[0, w], [0, 0]]
-    else:
-        pad_width = [[0, 0], [0, w]]
-
-    return np.pad(x, pad_width, mode=mode, **kwargs)
-
-
-def feature_normalize(batch, mean_norm: bool=True, std_norm: bool=True):
-    ids = [item['id'] for item in batch]
-    lengths = np.asarray([item['feat'].shape[1] for item in batch])
-    feats = list(
-        map(lambda x: pad_right_2d(x, lengths.max()),
-            [item['feat'] for item in batch]))
-    feats = np.stack(feats)
-
-    # Features normalization if needed
-    for i in range(len(feats)):
-        feat = feats[i][:, :lengths[i]]  # Excluding pad values.
-        mean = feat.mean(axis=-1, keepdims=True) if mean_norm else 0
-        std = feat.std(axis=-1, keepdims=True) if std_norm else 1
-        feats[i][:, :lengths[i]] = (feat - mean) / std
-        assert feats[i][:, lengths[
-            i]:].sum() == 0  # Padding valus should all be 0.
-
-    # Converts into ratios.
-    lengths = (lengths / lengths.max()).astype(np.float32)
-
-    return {'ids': ids, 'feats': feats, 'lengths': lengths}
-
-
-# feat configuration
-cpu_feat_conf = {
-    'n_mels': 80,
-    'window_size': 400,  #ms
-    'hop_length': 160,  #ms
-}
-
-
-def main(args):
-    # stage0: set the training device, cpu or gpu
-    paddle.set_device(args.device)
-    # set the random seed, it is a must for multiprocess training
-    seed_everything(args.seed)
-
-    # stage1: build the dnn backbone model network
-    ##"channels": [1024, 1024, 1024, 1024, 3072],
-    model_conf = {
-        "input_size": 80,
-        "channels": [512, 512, 512, 512, 1536],
-        "kernel_sizes": [5, 3, 3, 3, 1],
-        "dilations": [1, 2, 3, 4, 1],
-        "attention_channels": 128,
-        "lin_neurons": 192,
-    }
-    ecapa_tdnn = EcapaTdnn(**model_conf)
-
-    # stage2: build the speaker verification eval instance with backbone model
-    model = SpeakerIdetification(
-        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
-
-    # stage3: load the pre-trained model
-    args.load_checkpoint = os.path.abspath(
-        os.path.expanduser(args.load_checkpoint))
-
-    # load model checkpoint to sid model
-    state_dict = paddle.load(
-        os.path.join(args.load_checkpoint, 'model.pdparams'))
-    model.set_state_dict(state_dict)
-    logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
-
-    # stage4: construct the enroll and test dataloader
-    enrol_ds = VoxCeleb1(
-        subset='enrol',
-        target_dir=args.data_dir,
-        feat_type='melspectrogram',
-        random_chunk=False,
-        **cpu_feat_conf)
-    enrol_sampler = BatchSampler(
-        enrol_ds, batch_size=args.batch_size,
-        shuffle=True)  # Shuffle to make embedding normalization more robust.
-    enrol_loader = DataLoader(enrol_ds,
-                    batch_sampler=enrol_sampler,
-                    collate_fn=lambda x: feature_normalize(
-                            x, mean_norm=True, std_norm=False),
-                    num_workers=args.num_workers,
-                    return_list=True,)
-
-    test_ds = VoxCeleb1(
-        subset='test',
-        target_dir=args.data_dir,
-        feat_type='melspectrogram',
-        random_chunk=False,
-        **cpu_feat_conf)
-
-    test_sampler = BatchSampler(
-        test_ds, batch_size=args.batch_size, shuffle=True)
-    test_loader = DataLoader(test_ds,
-                            batch_sampler=test_sampler,
-                            collate_fn=lambda x: feature_normalize(
-                                x, mean_norm=True, std_norm=False),
-                            num_workers=args.num_workers,
-                            return_list=True,)
-    # stage6: we must set the model to eval mode
-    model.eval()
-
-    # stage7: global embedding norm to imporve the performance
-    if args.global_embedding_norm:
-        global_embedding_mean = None
-        global_embedding_std = None
-        mean_norm_flag = args.embedding_mean_norm
-        std_norm_flag = args.embedding_std_norm
-        batch_count = 0
-
-    # stage8: Compute embeddings of audios in enrol and test dataset from model.
-    id2embedding = {}
-    # Run multi times to make embedding normalization more stable.
-    for i in range(2):
-        for dl in [enrol_loader, test_loader]:
-            logger.info(
-                f'Loop {[i+1]}: Computing embeddings on {dl.dataset.subset} dataset'
-            )
-            with paddle.no_grad():
-                for batch_idx, batch in enumerate(tqdm(dl)):
-
-                    # stage 8-1: extrac the audio embedding
-                    ids, feats, lengths = batch['ids'], batch['feats'], batch[
-                        'lengths']
-                    embeddings = model.backbone(feats, lengths).squeeze(
-                        -1).numpy()  # (N, emb_size, 1) -> (N, emb_size)
-
-                    # Global embedding normalization.
-                    if args.global_embedding_norm:
-                        batch_count += 1
-                        current_mean = embeddings.mean(
-                            axis=0) if mean_norm_flag else 0
-                        current_std = embeddings.std(
-                            axis=0) if std_norm_flag else 1
-                        # Update global mean and std.
-                        if global_embedding_mean is None and global_embedding_std is None:
-                            global_embedding_mean, global_embedding_std = current_mean, current_std
-                        else:
-                            weight = 1 / batch_count  # Weight decay by batches.
-                            global_embedding_mean = (
-                                1 - weight
-                            ) * global_embedding_mean + weight * current_mean
-                            global_embedding_std = (
-                                1 - weight
-                            ) * global_embedding_std + weight * current_std
-                        # Apply global embedding normalization.
-                        embeddings = (embeddings - global_embedding_mean
-                                      ) / global_embedding_std
-
-                    # Update embedding dict.
-                    id2embedding.update(dict(zip(ids, embeddings)))
-
-    # stage 9: Compute cosine scores.
-    labels = []
-    enrol_ids = []
-    test_ids = []
-    with open(VoxCeleb1.veri_test_file, 'r') as f:
-        for line in f.readlines():
-            label, enrol_id, test_id = line.strip().split(' ')
-            labels.append(int(label))
-            enrol_ids.append(enrol_id.split('.')[0].replace('/', '-'))
-            test_ids.append(test_id.split('.')[0].replace('/', '-'))
-
-    cos_sim_func = paddle.nn.CosineSimilarity(axis=1)
-    enrol_embeddings, test_embeddings = map(lambda ids: paddle.to_tensor(
-        np.asarray([id2embedding[id] for id in ids], dtype='float32')),
-                                            [enrol_ids, test_ids
-                                             ])  # (N, emb_size)
-    scores = cos_sim_func(enrol_embeddings, test_embeddings)
-    EER, threshold = compute_eer(np.asarray(labels), scores.numpy())
-    logger.info(
-        f'EER of verification test: {EER*100:.4f}%, score threshold: {threshold:.5f}'
-    )
-
-
-if __name__ == "__main__":
-    # yapf: disable
-    parser = argparse.ArgumentParser(__doc__)
-    parser.add_argument('--device',
-                        choices=['cpu', 'gpu'],
-                        default="gpu",
-                        help="Select which device to train model, defaults to gpu.")
-    parser.add_argument("--seed",
-                        default=0,
-                        type=int,
-                        help="random seed for paddle, numpy and python random package")
-    parser.add_argument("--data-dir",
-                        default="./data/",
-                        type=str,
-                        help="data directory")
-    parser.add_argument("--batch-size",
-                        type=int,
-                        default=16,
-                        help="Total examples' number in batch for extract the embedding.")
-    parser.add_argument("--num-workers",
-                        type=int,
-                        default=0,
-                        help="Number of workers in dataloader.")
-    parser.add_argument("--load-checkpoint",
-                        type=str,
-                        default='',
-                        help="Directory to load model checkpoint to contiune trainning.")
-    parser.add_argument("--global-embedding-norm",
-                        type=bool,
-                        default=True,
-                        help="Apply global normalization on speaker embeddings.")
-    parser.add_argument("--embedding-mean-norm",
-                        type=bool,
-                        default=True,
-                        help="Apply mean normalization on speaker embeddings.")
-    parser.add_argument("--embedding-std-norm",
-                        type=bool,
-                        default=False,
-                        help="Apply std normalization on speaker embeddings.")
-    args = parser.parse_args()
-    # yapf: enable
-
-    main(args)
diff --git a/examples/voxceleb/sv0/local/train.py b/examples/voxceleb/sv0/local/train.py
deleted file mode 100644
index 3fe67c8e..00000000
--- a/examples/voxceleb/sv0/local/train.py
+++ /dev/null
@@ -1,326 +0,0 @@
-# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import os
-
-import numpy as np
-import paddle
-from paddle.io import BatchSampler
-from paddle.io import DataLoader
-from paddle.io import DistributedBatchSampler
-
-from paddleaudio.datasets.voxceleb import VoxCeleb1
-from paddleaudio.features.core import melspectrogram
-from paddlespeech.s2t.utils.log import Log
-from paddlespeech.vector.io.augment import build_augment_pipeline
-from paddlespeech.vector.io.augment import waveform_augment
-from paddlespeech.vector.io.batch import feature_normalize
-from paddlespeech.vector.io.batch import waveform_collate_fn
-from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
-from paddlespeech.vector.modules.loss import AdditiveAngularMargin
-from paddlespeech.vector.modules.loss import LogSoftmaxWrapper
-from paddlespeech.vector.modules.lr import CyclicLRScheduler
-from paddlespeech.vector.modules.sid_model import SpeakerIdetification
-from paddlespeech.vector.training.seeding import seed_everything
-from paddlespeech.vector.utils.time import Timer
-
-logger = Log(__name__).getlog()
-
-# feat configuration
-cpu_feat_conf = {
-    'n_mels': 80,
-    'window_size': 400,  #ms
-    'hop_length': 160,  #ms
-}
-
-
-def main(args):
-    # stage0: set the training device, cpu or gpu
-    paddle.set_device(args.device)
-
-    # stage1: we must call the paddle.distributed.init_parallel_env() api at the begining
-    paddle.distributed.init_parallel_env()
-    nranks = paddle.distributed.get_world_size()
-    local_rank = paddle.distributed.get_rank()
-    # set the random seed, it is a must for multiprocess training
-    seed_everything(args.seed)
-
-    # stage2: data prepare, such vox1 and vox2 data, and augment noise data and pipline
-    # note: some cmd must do in rank==0, so wo will refactor the data prepare code
-    train_dataset = VoxCeleb1('train', target_dir=args.data_dir)
-    dev_dataset = VoxCeleb1('dev', target_dir=args.data_dir)
-
-    if args.augment:
-        augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
-    else:
-        augment_pipeline = []
-
-    # stage3: build the dnn backbone model network
-    #"channels": [1024, 1024, 1024, 1024, 3072],
-    model_conf = {
-        "input_size": 80,
-        "channels": [512, 512, 512, 512, 1536],
-        "kernel_sizes": [5, 3, 3, 3, 1],
-        "dilations": [1, 2, 3, 4, 1],
-        "attention_channels": 128,
-        "lin_neurons": 192,
-    }
-    ecapa_tdnn = EcapaTdnn(**model_conf)
-
-    # stage4: build the speaker verification train instance with backbone model
-    model = SpeakerIdetification(
-        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
-
-    # stage5: build the optimizer, we now only construct the AdamW optimizer
-    lr_schedule = CyclicLRScheduler(
-        base_lr=args.learning_rate, max_lr=1e-3, step_size=140000 // nranks)
-    optimizer = paddle.optimizer.AdamW(
-        learning_rate=lr_schedule, parameters=model.parameters())
-
-    # stage6: build the loss function, we now only support LogSoftmaxWrapper
-    criterion = LogSoftmaxWrapper(
-        loss_fn=AdditiveAngularMargin(margin=0.2, scale=30))
-
-    # stage7: confirm training start epoch
-    #         if pre-trained model exists, start epoch confirmed by the pre-trained model
-    start_epoch = 0
-    if args.load_checkpoint:
-        logger.info("load the check point")
-        args.load_checkpoint = os.path.abspath(
-            os.path.expanduser(args.load_checkpoint))
-        try:
-            # load model checkpoint
-            state_dict = paddle.load(
-                os.path.join(args.load_checkpoint, 'model.pdparams'))
-            model.set_state_dict(state_dict)
-
-            # load optimizer checkpoint
-            state_dict = paddle.load(
-                os.path.join(args.load_checkpoint, 'model.pdopt'))
-            optimizer.set_state_dict(state_dict)
-            if local_rank == 0:
-                logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
-        except FileExistsError:
-            if local_rank == 0:
-                logger.info('Train from scratch.')
-
-        try:
-            start_epoch = int(args.load_checkpoint[-1])
-            logger.info(f'Restore training from epoch {start_epoch}.')
-        except ValueError:
-            pass
-
-    # stage8: we build the batch sampler for paddle.DataLoader
-    train_sampler = DistributedBatchSampler(
-        train_dataset, batch_size=args.batch_size, shuffle=True, drop_last=False)
-    train_loader = DataLoader(
-        train_dataset,
-        batch_sampler=train_sampler,
-        num_workers=args.num_workers,
-        collate_fn=waveform_collate_fn,
-        return_list=True,
-        use_buffer_reader=True, )
-
-    # stage9: start to train
-    #         we will comment the training process
-    steps_per_epoch = len(train_sampler)
-    timer = Timer(steps_per_epoch * args.epochs)
-    timer.start()
-
-    for epoch in range(start_epoch + 1, args.epochs + 1):
-        # at the begining, model must set to train mode
-        model.train()
-
-        avg_loss = 0
-        num_corrects = 0
-        num_samples = 0
-        for batch_idx, batch in enumerate(train_loader):
-            # stage 9-1: batch data is audio sample points and speaker id label
-            waveforms, labels = batch['waveforms'], batch['labels']
-
-            # stage 9-2: audio sample augment method, which is done on the audio sample point
-            if len(augment_pipeline) != 0:
-                waveforms = waveform_augment(waveforms, augment_pipeline)
-                labels = paddle.concat(
-                    [labels for i in range(len(augment_pipeline) + 1)])
-
-            # stage 9-3: extract the audio feats,such fbank, mfcc, spectrogram
-            feats = []
-            for waveform in waveforms.numpy():
-                feat = melspectrogram(x=waveform, **cpu_feat_conf)
-                feats.append(feat)
-            feats = paddle.to_tensor(np.asarray(feats))
-
-            # stage 9-4: feature normalize, which help converge and imporve the performance
-            feats = feature_normalize(
-                feats, mean_norm=True, std_norm=False)  # Features normalization
-
-            # stage 9-5: model forward, such ecapa-tdnn, x-vector
-            logits = model(feats)
-
-            # stage 9-6: loss function criterion, such AngularMargin, AdditiveAngularMargin
-            loss = criterion(logits, labels)
-
-            # stage 9-7: update the gradient and clear the gradient cache
-            loss.backward()
-            optimizer.step()
-            if isinstance(optimizer._learning_rate,
-                          paddle.optimizer.lr.LRScheduler):
-                optimizer._learning_rate.step()
-            optimizer.clear_grad()
-
-            # stage 9-8: Calculate average loss per batch
-            avg_loss += loss.numpy()[0]
-
-            # stage 9-9: Calculate metrics, which is one-best accuracy
-            preds = paddle.argmax(logits, axis=1)
-            num_corrects += (preds == labels).numpy().sum()
-            num_samples += feats.shape[0]
-            timer.count()  # step plus one in timer
-
-            # stage 9-10: print the log information only on 0-rank per log-freq batchs
-            if (batch_idx + 1) % args.log_freq == 0 and local_rank == 0:
-                lr = optimizer.get_lr()
-                avg_loss /= args.log_freq
-                avg_acc = num_corrects / num_samples
-
-                print_msg = 'Train Epoch={}/{}, Step={}/{}'.format(
-                    epoch, args.epochs, batch_idx + 1, steps_per_epoch)
-                print_msg += ' loss={:.4f}'.format(avg_loss)
-                print_msg += ' acc={:.4f}'.format(avg_acc)
-                print_msg += ' lr={:.4E} step/sec={:.2f} | ETA {}'.format(
-                    lr, timer.timing, timer.eta)
-                logger.info(print_msg)
-
-                avg_loss = 0
-                num_corrects = 0
-                num_samples = 0
-
-        # stage 9-11: save the model parameters only on 0-rank per save-freq batchs
-        if epoch % args.save_freq == 0 and batch_idx + 1 == steps_per_epoch:
-            if local_rank != 0:
-                paddle.distributed.barrier(
-                )  # Wait for valid step in main process
-                continue  # Resume trainning on other process
-
-            # stage 9-12: construct the valid dataset dataloader
-            dev_sampler = BatchSampler(
-                dev_dataset,
-                batch_size=args.batch_size // 4,
-                shuffle=False,
-                drop_last=False)
-            dev_loader = DataLoader(
-                dev_dataset,
-                batch_sampler=dev_sampler,
-                collate_fn=waveform_collate_fn,
-                num_workers=args.num_workers,
-                return_list=True, )
-
-            # set the model to eval mode
-            model.eval()
-            num_corrects = 0
-            num_samples = 0
-
-            # stage 9-13: evaluation the valid dataset batch data
-            logger.info('Evaluate on validation dataset')
-            with paddle.no_grad():
-                for batch_idx, batch in enumerate(dev_loader):
-                    waveforms, labels = batch['waveforms'], batch['labels']
-
-                    feats = []
-                    for waveform in waveforms.numpy():
-                        feat = melspectrogram(x=waveform, **cpu_feat_conf)
-                        feats.append(feat)
-
-                    feats = paddle.to_tensor(np.asarray(feats))
-                    feats = feature_normalize(
-                        feats, mean_norm=True, std_norm=False)
-                    logits = model(feats)
-
-                    preds = paddle.argmax(logits, axis=1)
-                    num_corrects += (preds == labels).numpy().sum()
-                    num_samples += feats.shape[0]
-
-            print_msg = '[Evaluation result]'
-            print_msg += ' dev_acc={:.4f}'.format(num_corrects / num_samples)
-            logger.info(print_msg)
-
-            # stage 9-14: Save model parameters
-            save_dir = os.path.join(args.checkpoint_dir,
-                                    'epoch_{}'.format(epoch))
-            logger.info('Saving model checkpoint to {}'.format(save_dir))
-            paddle.save(model.state_dict(),
-                        os.path.join(save_dir, 'model.pdparams'))
-            paddle.save(optimizer.state_dict(),
-                        os.path.join(save_dir, 'model.pdopt'))
-
-            if nranks > 1:
-                paddle.distributed.barrier()  # Main process
-
-
-if __name__ == "__main__":
-    # yapf: disable
-    parser = argparse.ArgumentParser(__doc__)
-    parser.add_argument('--device',
-                        choices=['cpu', 'gpu'],
-                        default="cpu",
-                        help="Select which device to train model, defaults to gpu.")
-    parser.add_argument("--seed",
-                        default=0,
-                        type=int,
-                        help="random seed for paddle, numpy and python random package")
-    parser.add_argument("--data-dir",
-                        default="./data/",
-                        type=str,
-                        help="data directory")
-    parser.add_argument("--learning-rate",
-                        type=float,
-                        default=1e-8,
-                        help="Learning rate used to train with warmup.")
-    parser.add_argument("--load-checkpoint",
-                        type=str,
-                        default=None,
-                        help="Directory to load model checkpoint to contiune trainning.")
-    parser.add_argument("--batch-size",
-                        type=int, default=64,
-                        help="Total examples' number in batch for training.")
-    parser.add_argument("--num-workers",
-                        type=int,
-                        default=0,
-                        help="Number of workers in dataloader.")
-    parser.add_argument("--epochs",
-                        type=int,
-                        default=50,
-                        help="Number of epoches for fine-tuning.")
-    parser.add_argument("--log-freq",
-                        type=int,
-                        default=10,
-                        help="Log the training infomation every n steps.")
-    parser.add_argument("--save-freq",
-                        type=int,
-                        default=1,
-                        help="Save checkpoint every n epoch.")
-    parser.add_argument("--checkpoint-dir",
-                        type=str,
-                        default='./checkpoint',
-                        help="Directory to save model checkpoints.")
-    parser.add_argument("--augment",
-                        action="store_true",
-                        default=False,
-                        help="Apply audio augments.")
-
-    args = parser.parse_args()
-    # yapf: enable
-
-    main(args)
diff --git a/examples/voxceleb/sv0/path.sh b/examples/voxceleb/sv0/path.sh
index 38a242a4..6d19f994 100755
--- a/examples/voxceleb/sv0/path.sh
+++ b/examples/voxceleb/sv0/path.sh
@@ -9,3 +9,6 @@ export PYTHONIOENCODING=UTF-8
 export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
 
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/
+
+MODEL=ecapa-tdnn
+export BIN_DIR=${MAIN_ROOT}/paddlespeech/vector/exps/${MODEL}
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index a2336fb6..a6346cd5 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -30,23 +30,21 @@ if [ $stage -le 1 ]; then
      # stage 1: train the speaker identification model
      python3 \
           -m paddle.distributed.launch --gpus=0,1,2,3 \
-          local/train.py --device "gpu" --checkpoint-dir ${exp_dir} --augment \
+          ${BIN_DIR}/train.py --device "gpu" --checkpoint-dir ${exp_dir} --augment \
           --save-freq 10 --data-dir ${dir} --batch-size 64 --epochs 100
 fi
 
 if [ $stage -le 2 ]; then
-     # stage 1: train the speaker identification model
-     # you can set the variable PPAUDIO_HOME to specifiy the downloaded the vox1 and vox2 dataset
+     # stage 1: get the speaker verification scores with cosine function
      python3 \
-          local/speaker_verification_cosine.py\
+          ${BIN_DIR}/speaker_verification_cosine.py\
           --batch-size 4 --data-dir ${dir} --load-checkpoint ${exp_dir}/epoch_10/
 fi
 
 if [ $stage -le 3 ]; then
-     # stage 1: train the speaker identification model
-     # you can set the variable PPAUDIO_HOME to specifiy the downloaded the vox1 and vox2 dataset
+     # stage 3: extract the audio embedding
      python3 \
-          local/extract_speaker_embedding.py\
+          ${BIN_DIR}/extract_speaker_embedding.py\
           --audio-path "demo/csv/00001.wav" --load-checkpoint ${exp_dir}/epoch_60/
 fi
 
diff --git a/paddleaudio/paddleaudio/datasets/__init__.py b/paddleaudio/paddleaudio/datasets/__init__.py
index 5c5f0369..cbf9b3ae 100644
--- a/paddleaudio/paddleaudio/datasets/__init__.py
+++ b/paddleaudio/paddleaudio/datasets/__init__.py
@@ -15,3 +15,5 @@ from .esc50 import ESC50
 from .gtzan import GTZAN
 from .tess import TESS
 from .urban_sound import UrbanSound8K
+from .voxceleb import VoxCeleb1
+from .rirs_noises import OpenRIRNoise
diff --git a/paddleaudio/datasets/rirs_noises.py b/paddleaudio/paddleaudio/datasets/rirs_noises.py
similarity index 97%
rename from paddleaudio/datasets/rirs_noises.py
rename to paddleaudio/paddleaudio/datasets/rirs_noises.py
index 6af9fd9d..df5dec61 100644
--- a/paddleaudio/datasets/rirs_noises.py
+++ b/paddleaudio/paddleaudio/datasets/rirs_noises.py
@@ -23,11 +23,11 @@ from typing import Tuple
 from paddle.io import Dataset
 from tqdm import tqdm
 
-from paddleaudio.backends import load as load_audio
-from paddleaudio.backends import save_wav
-from paddleaudio.datasets.dataset import feat_funcs
-from paddleaudio.utils import DATA_HOME
-from paddleaudio.utils import decompress
+from ..backends import load as load_audio
+from ..backends import save as save_wav
+from .dataset import feat_funcs
+from ..utils import DATA_HOME
+from ..utils import decompress
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.utils.download import download_and_decompress
 
diff --git a/paddleaudio/datasets/voxceleb.py b/paddleaudio/paddleaudio/datasets/voxceleb.py
similarity index 97%
rename from paddleaudio/datasets/voxceleb.py
rename to paddleaudio/paddleaudio/datasets/voxceleb.py
index 0011340e..4989accb 100644
--- a/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/paddleaudio/datasets/voxceleb.py
@@ -25,10 +25,10 @@ from paddle.io import Dataset
 from pathos.multiprocessing import Pool
 from tqdm import tqdm
 
-from paddleaudio.backends import load as load_audio
-from paddleaudio.datasets.dataset import feat_funcs
-from paddleaudio.utils import DATA_HOME
-from paddleaudio.utils import decompress
+from .dataset import feat_funcs
+from ..backends import load as load_audio
+from ..utils import DATA_HOME
+from ..utils import decompress
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.utils.download import download_and_decompress
 from utils.utility import download
@@ -83,7 +83,7 @@ class VoxCeleb1(Dataset):
     meta_path = os.path.join(base_path, 'meta')
     veri_test_file = os.path.join(meta_path, 'veri_test2.txt')
     csv_path = os.path.join(base_path, 'csv')
-    subsets = ['train', 'dev', 'enrol', 'test']
+    subsets = ['train', 'dev', 'enroll', 'test']
 
     def __init__(
             self,
@@ -330,7 +330,7 @@ class VoxCeleb1(Dataset):
 
         self.generate_csv(
             enroll_files,
-            os.path.join(self.csv_path, 'enrol.csv'),
+            os.path.join(self.csv_path, 'enroll.csv'),
             split_chunks=False)
         self.generate_csv(
             test_files,
diff --git a/paddleaudio/paddleaudio/metric/__init__.py b/paddleaudio/paddleaudio/metric/__init__.py
index a96530ff..b435571d 100644
--- a/paddleaudio/paddleaudio/metric/__init__.py
+++ b/paddleaudio/paddleaudio/metric/__init__.py
@@ -13,3 +13,4 @@
 # limitations under the License.
 from .dtw import dtw_distance
 from .mcd import mcd_distance
+from .eer import compute_eer
diff --git a/paddlespeech/vector/training/metrics.py b/paddleaudio/paddleaudio/metric/eer.py
similarity index 100%
rename from paddlespeech/vector/training/metrics.py
rename to paddleaudio/paddleaudio/metric/eer.py
diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
index 366c0cff..76312978 100644
--- a/paddlespeech/vector/io/augment.py
+++ b/paddlespeech/vector/io/augment.py
@@ -20,8 +20,8 @@ import paddle
 import paddle.nn as nn
 import paddle.nn.functional as F
 
-from paddleaudio.backends import load as load_audio
-from paddleaudio.datasets.rirs_noises import OpenRIRNoise
+from paddleaudio.paddleaudio import load as load_audio
+from paddleaudio.paddleaudio.datasets.rirs_noises import OpenRIRNoise
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.signal_processing import compute_amplitude
 from paddlespeech.vector.io.signal_processing import convolve1d
diff --git a/paddlespeech/vector/io/batch.py b/paddlespeech/vector/io/batch.py
index 879cde3a..811775e2 100644
--- a/paddlespeech/vector/io/batch.py
+++ b/paddlespeech/vector/io/batch.py
@@ -40,3 +40,41 @@ def feature_normalize(feats: paddle.Tensor,
         feats = (feats - mean) / std
 
     return feats
+
+
+def pad_right_2d(x, target_length, axis=-1, mode='constant', **kwargs):
+    x = np.asarray(x)
+    assert len(
+        x.shape) == 2, f'Only 2D arrays supported, but got shape: {x.shape}'
+
+    w = target_length - x.shape[axis]
+    assert w >= 0, f'Target length {target_length} is less than origin length {x.shape[axis]}'
+
+    if axis == 0:
+        pad_width = [[0, w], [0, 0]]
+    else:
+        pad_width = [[0, 0], [0, w]]
+
+    return np.pad(x, pad_width, mode=mode, **kwargs)
+
+def batch_feature_normalize(batch, mean_norm: bool=True, std_norm: bool=True):
+    ids = [item['id'] for item in batch]
+    lengths = np.asarray([item['feat'].shape[1] for item in batch])
+    feats = list(
+        map(lambda x: pad_right_2d(x, lengths.max()),
+            [item['feat'] for item in batch]))
+    feats = np.stack(feats)
+
+    # Features normalization if needed
+    for i in range(len(feats)):
+        feat = feats[i][:, :lengths[i]]  # Excluding pad values.
+        mean = feat.mean(axis=-1, keepdims=True) if mean_norm else 0
+        std = feat.std(axis=-1, keepdims=True) if std_norm else 1
+        feats[i][:, :lengths[i]] = (feat - mean) / std
+        assert feats[i][:, lengths[
+            i]:].sum() == 0  # Padding valus should all be 0.
+
+    # Converts into ratios.
+    lengths = (lengths / lengths.max()).astype(np.float32)
+
+    return {'ids': ids, 'feats': feats, 'lengths': lengths}
\ No newline at end of file
diff --git a/paddlespeech/vector/modules/lr.py b/paddlespeech/vector/training/scheduler.py
similarity index 100%
rename from paddlespeech/vector/modules/lr.py
rename to paddlespeech/vector/training/scheduler.py

From 993d6783d7ca15d190892037e92111d2e50d3326 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 9 Mar 2022 15:39:57 +0800
Subject: [PATCH 024/126] remove unused code, test=doc

---
 examples/voxceleb/sv0/local/data.sh         |  25 ----
 paddlespeech/vector/__init__.py             |  30 +---
 paddlespeech/vector/datasets/ark_dataset.py | 142 -------------------
 paddlespeech/vector/datasets/dataset.py     | 143 --------------------
 paddlespeech/vector/datasets/egs_dataset.py |  91 -------------
 paddlespeech/vector/utils/data_utils.py     | 125 -----------------
 paddlespeech/vector/utils/utils.py          | 132 ------------------
 7 files changed, 1 insertion(+), 687 deletions(-)
 delete mode 100755 examples/voxceleb/sv0/local/data.sh
 delete mode 100755 paddlespeech/vector/datasets/ark_dataset.py
 delete mode 100644 paddlespeech/vector/datasets/dataset.py
 delete mode 100644 paddlespeech/vector/datasets/egs_dataset.py
 delete mode 100755 paddlespeech/vector/utils/data_utils.py
 delete mode 100755 paddlespeech/vector/utils/utils.py

diff --git a/examples/voxceleb/sv0/local/data.sh b/examples/voxceleb/sv0/local/data.sh
deleted file mode 100755
index 6df9c3b8..00000000
--- a/examples/voxceleb/sv0/local/data.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-stage=-1
-stop_stage=100
-TARGET_DIR=${MAIN_ROOT}/dataset
-
-. utils/parse_options.sh || exit -1;
-
-src=$1
-mkdir -p data/{dev,test}
-if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
-    # download data, generate manifests
-    # create data/{dev,test} directory to store the manifest files
-    python3 ${TARGET_DIR}/voxceleb/voxceleb1.py \
-    --manifest_prefix="data/manifest" \
-    --target_dir="${src}"
-
-    if [ $? -ne 0 ]; then
-        echo "Prepare Voxceleb failed. Terminated."
-        exit 1
-    fi
-    mv data/manifest.dev data/dev
-    mv data/voxceleb1.dev.meta data/dev
-
-    mv data/manifest.test data/test
-    mv data/voxceleb1.test.meta data/test
-fi
diff --git a/paddlespeech/vector/__init__.py b/paddlespeech/vector/__init__.py
index 5c846193..61d5aa21 100644
--- a/paddlespeech/vector/__init__.py
+++ b/paddlespeech/vector/__init__.py
@@ -10,32 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-__init__ file for sidt package.
-"""
-
-import logging as sidt_logging
-import colorlog
-
-LOG_COLOR_CONFIG = {
-    'DEBUG': 'white',
-    'INFO': 'white',
-    'WARNING': 'yellow',
-    'ERROR': 'red',
-    'CRITICAL': 'purple',
-}
-
-# 设置全局的logger
-colored_formatter = colorlog.ColoredFormatter(
-    '%(log_color)s [%(levelname)s] [%(asctime)s] [%(filename)s:%(lineno)d] - %(message)s',
-    datefmt="%Y-%m-%d %H:%M:%S",
-    log_colors=LOG_COLOR_CONFIG)  # 日志输出格式
-_logger = sidt_logging.getLogger("sidt")
-handler = colorlog.StreamHandler()
-handler.setLevel(sidt_logging.INFO)
-handler.setFormatter(colored_formatter)
-_logger.addHandler(handler)
-_logger.setLevel(sidt_logging.INFO)
-
+# limitations under the License.
\ No newline at end of file
diff --git a/paddlespeech/vector/datasets/ark_dataset.py b/paddlespeech/vector/datasets/ark_dataset.py
deleted file mode 100755
index 7a00e7ba..00000000
--- a/paddlespeech/vector/datasets/ark_dataset.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-import random
-import numpy as np
-import kaldi_python_io as k_io
-from paddle.io import Dataset
-from paddlespeech.vector.utils.data_utils import batch_pad_right
-import paddlespeech.vector.utils as utils
-from paddlespeech.vector.utils.utils import read_map_file
-from paddlespeech.vector import _logger as log
-
-def ark_collate_fn(batch):
-    """
-    Custom collate function] for kaldi feats dataset
-
-    Args:
-        min_chunk_size: min chunk size of a utterance
-        max_chunk_size: max chunk size of a utterance
-
-    Returns:
-        ark_collate_fn: collate funtion for dataloader
-    """
-
-    data = []
-    target = []
-    for items in batch:
-        for x, y in zip(items[0], items[1]):
-            data.append(np.array(x))
-            target.append(y)
-
-    data, lengths = batch_pad_right(data)
-    return np.array(data, dtype=np.float32), \
-           np.array(lengths, dtype=np.float32), \
-           np.array(target, dtype=np.long).reshape((len(target), 1))
-
-
-class KaldiArkDataset(Dataset):
-    """
-    Dataset used to load kaldi ark/scp files.
-    """
-    def __init__(self, scp_file, label2utt, min_item_size=1,
-                 max_item_size=1, repeat=50, min_chunk_size=200,
-                 max_chunk_size=400, select_by_speaker=True):
-        self.scp_file = scp_file
-        self.scp_reader = None
-        self.repeat = repeat
-        self.min_item_size = min_item_size
-        self.max_item_size = max_item_size
-        self.min_chunk_size = min_chunk_size
-        self.max_chunk_size = max_chunk_size
-        self._collate_fn = ark_collate_fn
-        self._is_select_by_speaker = select_by_speaker
-        if utils.is_exist(self.scp_file):
-            self.scp_reader = k_io.ScriptReader(self.scp_file)
-
-        label2utts, utt2label = read_map_file(label2utt, key_func=int)
-        self.utt_info = list(label2utts.items()) if self._is_select_by_speaker else list(utt2label.items())
-
-    @property
-    def collate_fn(self):
-        """
-        Return a collate funtion.
-        """
-        return self._collate_fn
-
-    def _random_chunk(self, length):
-        chunk_size = random.randint(self.min_chunk_size, self.max_chunk_size)
-        if chunk_size >= length:
-            return 0, length
-        start = random.randint(0, length - chunk_size)
-        end = start + chunk_size
-
-        return start, end
-
-    def _select_by_speaker(self, index):
-        if self.scp_reader is None or not self.utt_info:
-            return []
-        index = index % (len(self.utt_info))
-        inputs = []
-        labels = []
-        item_size = random.randint(self.min_item_size, self.max_item_size)
-        for loop_idx in range(item_size):
-            try:
-                utt_index = random.randint(0, len(self.utt_info[index][1])) \
-                        % len(self.utt_info[index][1])
-                key = self.utt_info[index][1][utt_index]
-            except:
-                print(index, utt_index, len(self.utt_info[index][1]))
-                sys.exit(-1)
-            x = self.scp_reader[key]
-            x = np.transpose(x)
-            bg, end = self._random_chunk(x.shape[-1])
-            inputs.append(x[:, bg: end])
-            labels.append(self.utt_info[index][0])
-        return inputs, labels
-
-    def _select_by_utt(self, index):
-        if self.scp_reader is None or len(self.utt_info) == 0:
-            return {}
-        index = index % (len(self.utt_info))
-        key = self.utt_info[index][0]
-        x = self.scp_reader[key]
-        x = np.transpose(x)
-        bg, end = self._random_chunk(x.shape[-1])
-
-        y = self.utt_info[index][1]
-
-        return [x[:, bg: end]], [y]
-
-    def __getitem__(self, index):
-        if self._is_select_by_speaker:
-            return self._select_by_speaker(index)
-        else:
-            return self._select_by_utt(index)
-
-    def __len__(self):
-        return len(self.utt_info) * self.repeat
-
-    def __iter__(self):
-        self._start = 0
-        return self
-
-    def __next__(self):
-        if self._start < len(self):
-            ret = self[self._start]
-            self._start += 1
-            return ret
-        else:
-            raise StopIteration
diff --git a/paddlespeech/vector/datasets/dataset.py b/paddlespeech/vector/datasets/dataset.py
deleted file mode 100644
index e7030053..00000000
--- a/paddlespeech/vector/datasets/dataset.py
+++ /dev/null
@@ -1,143 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-import random
-import numpy as np
-import kaldi_python_io as k_io
-from paddle.io import Dataset
-from paddlespeech.vector.utils.data_utils import batch_pad_right
-import paddlespeech.vector.utils as utils
-from paddlespeech.vector.utils.utils import read_map_file
-
-def ark_collate_fn(batch):
-    """
-    Custom collate function for kaldi feats dataset
-
-    Args:
-        min_chunk_size: min chunk size of a utterance
-        max_chunk_size: max chunk size of a utterance
-
-    Returns:
-        ark_collate_fn: collate funtion for dataloader
-    """
-
-    data = []
-    target = []
-    for items in batch:
-        for x, y in zip(items[0], items[1]):
-            data.append(np.array(x))
-            target.append(y)
-
-    data, lengths = batch_pad_right(data)
-    return np.array(data, dtype=np.float32), \
-           np.array(lengths, dtype=np.float32), \
-           np.array(target, dtype=np.long).reshape((len(target), 1))
-
-
-class KaldiArkDataset(Dataset):
-    """
-    Dataset used to load kaldi ark/scp files.
-    """
-    def __init__(self, scp_file, label2utt, min_item_size=1,
-                 max_item_size=1, repeat=50, min_chunk_size=200,
-                 max_chunk_size=400, select_by_speaker=True):
-        self.scp_file = scp_file
-        self.scp_reader = None
-        self.repeat = repeat
-        self.min_item_size = min_item_size
-        self.max_item_size = max_item_size
-        self.min_chunk_size = min_chunk_size
-        self.max_chunk_size = max_chunk_size
-        self._collate_fn = ark_collate_fn
-        self._is_select_by_speaker = select_by_speaker
-        if utils.is_exist(self.scp_file):
-            self.scp_reader = k_io.ScriptReader(self.scp_file)
-
-        label2utts, utt2label = read_map_file(label2utt, key_func=int)
-        self.utt_info = list(label2utts.items()) if self._is_select_by_speaker else list(utt2label.items())
-
-    @property
-    def collate_fn(self):
-        """
-        Return a collate funtion.
-        """
-        return self._collate_fn
-
-    def _random_chunk(self, length):
-        chunk_size = random.randint(self.min_chunk_size, self.max_chunk_size)
-        if chunk_size >= length:
-            return 0, length
-        start = random.randint(0, length - chunk_size)
-        end = start + chunk_size
-
-        return start, end
-
-    def _select_by_speaker(self, index):
-        if self.scp_reader is None or not self.utt_info:
-            return []
-        index = index % (len(self.utt_info))
-        inputs = []
-        labels = []
-        item_size = random.randint(self.min_item_size, self.max_item_size)
-        for loop_idx in range(item_size):
-            try:
-                utt_index = random.randint(0, len(self.utt_info[index][1])) \
-                        % len(self.utt_info[index][1])
-                key = self.utt_info[index][1][utt_index]
-            except:
-                print(index, utt_index, len(self.utt_info[index][1]))
-                sys.exit(-1)
-            x = self.scp_reader[key]
-            x = np.transpose(x)
-            bg, end = self._random_chunk(x.shape[-1])
-            inputs.append(x[:, bg: end])
-            labels.append(self.utt_info[index][0])
-        return inputs, labels
-
-    def _select_by_utt(self, index):
-        if self.scp_reader is None or len(self.utt_info) == 0:
-            return {}
-        index = index % (len(self.utt_info))
-        key = self.utt_info[index][0]
-        x = self.scp_reader[key]
-        x = np.transpose(x)
-        bg, end = self._random_chunk(x.shape[-1])
-
-        y = self.utt_info[index][1]
-
-        return [x[:, bg: end]], [y]
-
-    def __getitem__(self, index):
-        if self._is_select_by_speaker:
-            return self._select_by_speaker(index)
-        else:
-            return self._select_by_utt(index)
-
-    def __len__(self):
-        return len(self.utt_info) * self.repeat
-
-    def __iter__(self):
-        self._start = 0
-        return self
-
-    def __next__(self):
-        if self._start < len(self):
-            ret = self[self._start]
-            self._start += 1
-            return ret
-        else:
-            raise StopIteration
-
-return KaldiArkDataset
diff --git a/paddlespeech/vector/datasets/egs_dataset.py b/paddlespeech/vector/datasets/egs_dataset.py
deleted file mode 100644
index 53130d5f..00000000
--- a/paddlespeech/vector/datasets/egs_dataset.py
+++ /dev/null
@@ -1,91 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Load nnet3 training egs which generated by kaldi
-"""
-
-import random
-import numpy as np
-import kaldi_python_io as k_io
-from paddle.io import Dataset
-import paddlespeech.vector.utils.utils as utils
-from paddlespeech.vector import _logger as log
-class KaldiEgsDataset(Dataset):
-    """
-    Dataset used to load kaldi nnet3 egs files.
-    """
-    def __init__(self, egs_list_file, egs_idx, transforms=None):
-        self.scp_reader = None
-        self.subset_idx = egs_idx - 1
-        self.transforms = transforms
-        if not utils.is_exist(egs_list_file):
-            return
-
-        self.egs_files = []
-        with open(egs_list_file, 'r') as in_fh:
-            for line in in_fh:
-                if line.strip():
-                    self.egs_files.append(line.strip())
-
-        self.next_subset()
-
-    def next_subset(self, target_index=None, delta_index=None):
-        """
-        Use next specific subset
-
-        Args:
-            target_index: target egs index
-            delta_index: incremental value of egs index
-        """
-        if self.egs_files:
-            if target_index:
-                self.subset_idx = target_index
-            else:
-                delta_index = delta_index if delta_index else 1
-                self.subset_idx += delta_index
-            log.info("egs dataset subset index: %d" % (self.subset_idx))
-            egs_file = self.egs_files[self.subset_idx % len(self.egs_files)]
-            if utils.is_exist(egs_file):
-                self.scp_reader = k_io.Nnet3EgsScriptReader(egs_file)
-            else:
-                log.warning("No such file or directory: %s" % (egs_file))
-
-    def __getitem__(self, index):
-        if self.scp_reader is None:
-            return {}
-        index %= len(self)
-        in_dict, out_dict = self.scp_reader[index]
-        x = np.array(in_dict['matrix'])
-        x = np.transpose(x)
-        y = np.array(out_dict['matrix'][0][0][0], dtype=np.int).reshape((1,))
-        if self.transforms is not None:
-            idx = random.randint(0, len(self.transforms) - 1)
-            x = self.transforms[idx](x)
-        return x, y
-
-    def __len__(self):
-        return len(self.scp_reader)
-
-    def __iter__(self):
-        self._start = 0
-        return self
-
-    def __next__(self):
-        if self._start < len(self):
-            ret = self[self._start]
-            self._start += 1
-            return ret
-        else:
-            raise StopIteration
\ No newline at end of file
diff --git a/paddlespeech/vector/utils/data_utils.py b/paddlespeech/vector/utils/data_utils.py
deleted file mode 100755
index 4a33a795..00000000
--- a/paddlespeech/vector/utils/data_utils.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-data utilities
-"""
-import os
-import sys
-import numpy
-import paddle
-
-
-def pad_right_to(array, target_shape, mode="constant", value=0):
-    """
-    This function takes a numpy array of arbitrary shape and pads it to target
-    shape by appending values on the right.
-
-    Args:
-        array: input numpy array. Input array whose dimension we need to pad.
-    target_shape : (list, tuple). Target shape we want for the target array its len must be equal to array.ndim
-    mode : str. Pad mode, please refer to numpy.pad documentation.
-    value : float. Pad value, please refer to numpy.pad documentation.
-
-    Returns:
-        array: numpy.array. Padded array.
-        valid_vals : list. List containing proportion for each dimension of original, non-padded values.
-    """
-    assert len(target_shape) == array.ndim
-    pads = []  # this contains the abs length of the padding for each dimension.
-    valid_vals = []  # thic contains the relative lengths for each dimension.
-    i = 0 # iterating over target_shape ndims
-    while i < len(target_shape):
-        assert (
-            target_shape[i] >= array.shape[i]
-        ), "Target shape must be >= original shape for every dim"
-        pads.append([0, target_shape[i] - array.shape[i]])
-        valid_vals.append(array.shape[i] / target_shape[i])
-        i += 1
-
-    array = numpy.pad(array, pads, mode=mode, constant_values=value)
-
-    return array, valid_vals
-
-
-def batch_pad_right(arrays, mode="constant", value=0):
-    """Given a list of numpy arrays it batches them together by padding to the right
-    on each dimension in order to get same length for all.
-
-    Args:
-        arrays : list. List of array we wish to pad together.
-        mode : str. Padding mode see numpy.pad documentation.
-        value : float. Padding value see numpy.pad documentation.
-
-    Returns:
-        array : numpy.array. Padded array.
-        valid_vals : list. List containing proportion for each dimension of original, non-padded values.
-    """
-
-    if not len(arrays):
-        raise IndexError("arrays list must not be empty")
-
-    if len(arrays) == 1:
-        # if there is only one array in the batch we simply unsqueeze it.
-        return numpy.expand_dims(arrays[0], axis=0), numpy.array([1.0])
-
-    if not (
-        any(
-            [arrays[i].ndim == arrays[0].ndim for i in range(1, len(arrays))]
-        )
-    ):
-        raise IndexError("All arrays must have same number of dimensions")
-
-    # FIXME we limit the support here: we allow padding of only the last dimension
-    # need to remove this when feat extraction is updated to handle multichannel.
-    max_shape = []
-    for dim in range(arrays[0].ndim):
-        if dim != (arrays[0].ndim - 1):
-            if not all(
-                [x.shape[dim] == arrays[0].shape[dim] for x in arrays[1:]]
-            ):
-                raise EnvironmentError(
-                    "arrays should have same dimensions except for last one"
-                )
-        max_shape.append(max([x.shape[dim] for x in arrays]))
-
-    batched = []
-    valid = []
-    for t in arrays:
-        # for each array we apply pad_right_to
-        padded, valid_percent = pad_right_to(
-            t, max_shape, mode=mode, value=value
-        )
-        batched.append(padded)
-        valid.append(valid_percent[-1])
-
-    batched = numpy.stack(batched)
-
-    return batched, numpy.array(valid)
-
-
-def length_to_mask(length, max_len=None, dtype=None):
-    """Creates a binary mask for each sequence.
-    """
-    assert len(length.shape) == 1
-
-    if max_len is None:
-        max_len = paddle.cast(paddle.max(length), dtype="int64")  # using arange to generate mask
-    mask = paddle.arange(max_len, dtype=length.dtype).expand([paddle.shape(length)[0], max_len]) < length.unsqueeze(1)
-
-    if dtype is None:
-        dtype = length.dtype
-
-    mask = paddle.cast(mask, dtype=dtype)
-    return mask
diff --git a/paddlespeech/vector/utils/utils.py b/paddlespeech/vector/utils/utils.py
deleted file mode 100755
index a28cb526..00000000
--- a/paddlespeech/vector/utils/utils.py
+++ /dev/null
@@ -1,132 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-utilities
-"""
-import os
-import sys
-import paddle
-import numpy as np
-
-from paddlespeech.vector import _logger as log
-
-
-def exit_if_not_exist(in_path):
-    """
-    Check the existence of a file or directory, if not exit, exit the program.
-
-    Args:
-        in_path: input dicrector
-    """
-    if not is_exist(in_path):
-        sys.exit(-1)
-
-
-def is_exist(in_path):
-    """
-    Check the existence of a file or directory
-
-    Args:
-        in_path: input dicrector
-
-    Returns:
-        True or False
-    """
-    if not os.path.exists(in_path):
-        log.error("No such file or directory: %s" % (in_path))
-        return False
-
-    return True
-
-
-def get_latest_file(target_dir):
-    """
-    Get the latest file in target directory
-
-    Args:
-        target_dir: target directory
-
-    Returns:
-        latest_file: a string or None
-    """
-    items = os.listdir(target_dir)
-    items.sort(key=lambda fn: os.path.getmtime(os.path.join(target_dir, fn)) \
-               if not os.path.isdir(os.path.join(target_dir, fn)) else 0)
-    latest_file = None if not items else os.path.join(target_dir, items[-1])
-    return latest_file
-
-
-def avg_models(models):
-    """
-    merge multiple models
-    """
-    checkpoint_dict = paddle.load(models[0])
-    final_state_dict = checkpoint_dict
-
-    if len(models) > 1:
-        for model in models[1:]:
-            checkpoint_dict = paddle.load(model)
-            for k, v in checkpoint_dict.items():
-                final_state_dict[k] += v
-        for k in final_state_dict.keys():
-            final_state_dict[k] /= float(len(models))
-            if np.any(np.isnan(final_state_dict[k])):
-                print("Nan in %s" % (k))
-
-    return final_state_dict
-
-def Q_from_tokens(token_num):
-    """
-    get prior model, data from uniform, would support others(guassian) in future
-    """
-    freq = [1] * token_num
-    Q = paddle.to_tensor(freq, dtype = 'float64')
-    return Q / Q.sum()
-
-
-def read_map_file(map_file, key_func=None, value_func=None, values_func=None):
-    """ Read map file. First colume is key, the rest columes are values.
-
-    Args:
-        map_file: map file
-        key_func: convert function for key
-        value_func: convert function for each value
-        values_func: convert function for values
-
-    Returns:
-        dict: key 2 value
-        dict: value 2 key
-    """
-    if not is_exist(map_file):
-        sys.exit(0)
-
-    key2val = {}
-    val2key = {}
-    with open(map_file, 'r') as f:
-        for line in f:
-            line = line.strip()
-            if not line:
-                continue
-            items = line.split()
-            assert len(items) >= 2
-            key = items[0] if not key_func else key_func(items[0])
-            values = items[1:] if not value_func else [value_func(item) for item in items[1:]]
-            if values_func:
-                values = values_func(values)
-            key2val[key] = values
-            for value in values:
-                val2key[value] = key
-
-    return key2val, val2key

From 584a2c0e39ab73b4a5826077528eccb4edf7afbd Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 9 Mar 2022 20:46:57 +0800
Subject: [PATCH 025/126] add ecapa-tdnn config yaml file

---
 examples/voxceleb/sv0/conf/ecapa_tdnn.yaml    |  35 ++
 examples/voxceleb/sv0/run.sh                  |   6 +-
 .../ecapa-tdnn/extract_speaker_embedding.py   | 112 +++++++
 .../ecapa-tdnn/speaker_verification_cosine.py | 207 ++++++++++++
 paddlespeech/vector/exps/ecapa-tdnn/train.py  | 298 ++++++++++++++++++
 5 files changed, 656 insertions(+), 2 deletions(-)
 create mode 100644 examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
 create mode 100644 paddlespeech/vector/exps/ecapa-tdnn/extract_speaker_embedding.py
 create mode 100644 paddlespeech/vector/exps/ecapa-tdnn/speaker_verification_cosine.py
 create mode 100644 paddlespeech/vector/exps/ecapa-tdnn/train.py

diff --git a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
new file mode 100644
index 00000000..33304054
--- /dev/null
+++ b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
@@ -0,0 +1,35 @@
+###########################################################
+#                FEATURE EXTRACTION SETTING               #
+###########################################################
+# currently, we only support fbank
+feature:
+  n_mels: 80
+  window_size: 400     #25ms, sample rate 16000, 25 * 16000 / 1000 = 400 
+  hop_length: 160     #10ms, sample rate 16000, 10 * 16000 / 1000 = 160
+
+
+###########################################################
+#                       MODEL SETTING                     #
+###########################################################
+# currently, we only support ecapa-tdnn in the ecapa_tdnn.yaml
+# if we want use another model, please choose another configuration yaml file
+model:
+  input_size: 80
+  ##"channels": [1024, 1024, 1024, 1024, 3072],
+  # "channels": [512, 512, 512, 512, 1536],
+  channels: [512, 512, 512, 512, 1536]
+  kernel_sizes: [5, 3, 3, 3, 1]
+  dilations: [1, 2, 3, 4, 1]
+  attention_channels: 128
+  lin_neurons: 192
+
+###########################################
+#                Training                 #
+###########################################
+seed: 0
+epochs: 10
+batch_size: 32
+num_workers: 2
+save_freq: 10
+log_freq: 10
+learning_rate: 1e-8
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index a6346cd5..2c0e55a6 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -31,20 +31,22 @@ if [ $stage -le 1 ]; then
      python3 \
           -m paddle.distributed.launch --gpus=0,1,2,3 \
           ${BIN_DIR}/train.py --device "gpu" --checkpoint-dir ${exp_dir} --augment \
-          --save-freq 10 --data-dir ${dir} --batch-size 64 --epochs 100
+          --data-dir ${dir} --config conf/ecapa_tdnn.yaml
 fi
 
 if [ $stage -le 2 ]; then
      # stage 1: get the speaker verification scores with cosine function
      python3 \
           ${BIN_DIR}/speaker_verification_cosine.py\
-          --batch-size 4 --data-dir ${dir} --load-checkpoint ${exp_dir}/epoch_10/
+          --config conf/ecapa_tdnn.yaml \
+          --data-dir ${dir} --load-checkpoint ${exp_dir}/epoch_10/
 fi
 
 if [ $stage -le 3 ]; then
      # stage 3: extract the audio embedding
      python3 \
           ${BIN_DIR}/extract_speaker_embedding.py\
+          --config conf/ecapa_tdnn.yaml \
           --audio-path "demo/csv/00001.wav" --load-checkpoint ${exp_dir}/epoch_60/
 fi
 
diff --git a/paddlespeech/vector/exps/ecapa-tdnn/extract_speaker_embedding.py b/paddlespeech/vector/exps/ecapa-tdnn/extract_speaker_embedding.py
new file mode 100644
index 00000000..78498c61
--- /dev/null
+++ b/paddlespeech/vector/exps/ecapa-tdnn/extract_speaker_embedding.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import os
+
+import numpy as np
+import paddle
+from yacs.config import CfgNode
+
+from paddleaudio.paddleaudio.backends import load as load_audio
+from paddleaudio.paddleaudio.compliance.librosa import melspectrogram
+from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.io.batch import feature_normalize
+from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.seeding import seed_everything
+
+logger = Log(__name__).getlog()
+
+def extract_audio_embedding(args, config):
+    # stage 0: set the training device, cpu or gpu
+    paddle.set_device(args.device)
+    # set the random seed, it is a must for multiprocess training
+    seed_everything(config.seed)
+
+    # stage 1: build the dnn backbone model network
+    ecapa_tdnn = EcapaTdnn(**config.model)
+
+    # stage4: build the speaker verification train instance with backbone model
+    model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=1211)
+    # stage 2: load the pre-trained model
+    args.load_checkpoint = os.path.abspath(
+        os.path.expanduser(args.load_checkpoint))
+
+    # load model checkpoint to sid model
+    state_dict = paddle.load(
+        os.path.join(args.load_checkpoint, 'model.pdparams'))
+    model.set_state_dict(state_dict)
+    logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
+
+    # stage 3: we must set the model to eval mode
+    model.eval()
+
+    # stage 4: read the audio data and extract the embedding
+    # wavform is one dimension numpy array 
+    waveform, sr = load_audio(args.audio_path)
+
+    # feat type is numpy array, whose shape is [dim, time]
+    # we need convert the audio feat to one-batch shape [batch, dim, time], where the batch is one
+    # so the final shape is [1, dim, time]
+    feat = melspectrogram(x=waveform, **config.feature)
+    feat = paddle.to_tensor(feat).unsqueeze(0)
+
+    # in inference period, the lengths is all one without padding
+    lengths = paddle.ones([1])
+    feat = feature_normalize(
+        feat, mean_norm=True, std_norm=False, convert_to_numpy=True)
+
+    # model backbone network forward the feats and get the embedding
+    embedding = model.backbone(
+        feat, lengths).squeeze().numpy()  # (1, emb_size, 1) -> (emb_size)
+
+    # stage 5: do global norm with external mean and std
+    # todo
+    return embedding
+
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--device',
+                        choices=['cpu', 'gpu'],
+                        default="gpu",
+                        help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--config", 
+                        default=None,
+                        type=str,
+                        help="configuration file")
+    parser.add_argument("--load-checkpoint",
+                        type=str,
+                        default='',
+                        help="Directory to load model checkpoint to contiune trainning.")
+    parser.add_argument("--global-embedding-norm",
+                        type=str,
+                        default=None,
+                        help="Apply global normalization on speaker embeddings.")
+    parser.add_argument("--audio-path",
+                        default="./data/demo.wav",
+                        type=str,
+                        help="Single audio file path")
+    args = parser.parse_args()
+    # yapf: enable
+    # https://yaml.org/type/float.html
+    config = CfgNode(new_allowed=True)
+    if args.config:
+        config.merge_from_file(args.config)
+
+    config.freeze()
+    print(config)
+
+    extract_audio_embedding(args, config)
diff --git a/paddlespeech/vector/exps/ecapa-tdnn/speaker_verification_cosine.py b/paddlespeech/vector/exps/ecapa-tdnn/speaker_verification_cosine.py
new file mode 100644
index 00000000..4d85bd62
--- /dev/null
+++ b/paddlespeech/vector/exps/ecapa-tdnn/speaker_verification_cosine.py
@@ -0,0 +1,207 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import ast
+import os
+
+import numpy as np
+import paddle
+from yacs.config import CfgNode
+import paddle.nn.functional as F
+from paddle.io import BatchSampler
+from paddle.io import DataLoader
+from tqdm import tqdm
+
+from paddleaudio.paddleaudio.datasets import VoxCeleb1
+from paddlespeech.s2t.utils.log import Log
+from paddleaudio.paddleaudio.metric import compute_eer
+from paddlespeech.vector.io.batch import batch_feature_normalize
+from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.seeding import seed_everything
+
+logger = Log(__name__).getlog()
+
+def main(args, config):
+    # stage0: set the training device, cpu or gpu
+    paddle.set_device(args.device)
+    # set the random seed, it is a must for multiprocess training
+    seed_everything(config.seed)
+
+    # stage1: build the dnn backbone model network
+    ecapa_tdnn = EcapaTdnn(**config.model)
+
+    # stage2: build the speaker verification eval instance with backbone model
+    model = SpeakerIdetification(
+        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
+
+    # stage3: load the pre-trained model
+    args.load_checkpoint = os.path.abspath(
+        os.path.expanduser(args.load_checkpoint))
+
+    # load model checkpoint to sid model
+    state_dict = paddle.load(
+        os.path.join(args.load_checkpoint, 'model.pdparams'))
+    model.set_state_dict(state_dict)
+    logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
+
+    # stage4: construct the enroll and test dataloader
+    enroll_dataset = VoxCeleb1(
+        subset='enroll',
+        target_dir=args.data_dir,
+        feat_type='melspectrogram',
+        random_chunk=False,
+        **config.feature)
+    enroll_sampler = BatchSampler(
+        enroll_dataset, batch_size=config.batch_size,
+        shuffle=True)  # Shuffle to make embedding normalization more robust.
+    enrol_loader = DataLoader(enroll_dataset,
+                    batch_sampler=enroll_sampler,
+                    collate_fn=lambda x: batch_feature_normalize(
+                            x, mean_norm=True, std_norm=False),
+                    num_workers=config.num_workers,
+                    return_list=True,)
+
+    test_dataset = VoxCeleb1(
+        subset='test',
+        target_dir=args.data_dir,
+        feat_type='melspectrogram',
+        random_chunk=False,
+        **config.feature)
+
+    test_sampler = BatchSampler(
+        test_dataset, batch_size=config.batch_size, shuffle=True)
+    test_loader = DataLoader(test_dataset,
+                            batch_sampler=test_sampler,
+                            collate_fn=lambda x: batch_feature_normalize(
+                                x, mean_norm=True, std_norm=False),
+                            num_workers=config.num_workers,
+                            return_list=True,)
+    # stage6: we must set the model to eval mode
+    model.eval()
+
+    # stage7: global embedding norm to imporve the performance
+    if args.global_embedding_norm:
+        global_embedding_mean = None
+        global_embedding_std = None
+        mean_norm_flag = args.embedding_mean_norm
+        std_norm_flag = args.embedding_std_norm
+        batch_count = 0
+
+    # stage8: Compute embeddings of audios in enrol and test dataset from model.
+    id2embedding = {}
+    # Run multi times to make embedding normalization more stable.
+    for i in range(2):
+        for dl in [enrol_loader, test_loader]:
+            logger.info(
+                f'Loop {[i+1]}: Computing embeddings on {dl.dataset.subset} dataset'
+            )
+            with paddle.no_grad():
+                for batch_idx, batch in enumerate(tqdm(dl)):
+
+                    # stage 8-1: extrac the audio embedding
+                    ids, feats, lengths = batch['ids'], batch['feats'], batch[
+                        'lengths']
+                    embeddings = model.backbone(feats, lengths).squeeze(
+                        -1).numpy()  # (N, emb_size, 1) -> (N, emb_size)
+
+                    # Global embedding normalization.
+                    if args.global_embedding_norm:
+                        batch_count += 1
+                        current_mean = embeddings.mean(
+                            axis=0) if mean_norm_flag else 0
+                        current_std = embeddings.std(
+                            axis=0) if std_norm_flag else 1
+                        # Update global mean and std.
+                        if global_embedding_mean is None and global_embedding_std is None:
+                            global_embedding_mean, global_embedding_std = current_mean, current_std
+                        else:
+                            weight = 1 / batch_count  # Weight decay by batches.
+                            global_embedding_mean = (
+                                1 - weight
+                            ) * global_embedding_mean + weight * current_mean
+                            global_embedding_std = (
+                                1 - weight
+                            ) * global_embedding_std + weight * current_std
+                        # Apply global embedding normalization.
+                        embeddings = (embeddings - global_embedding_mean
+                                      ) / global_embedding_std
+
+                    # Update embedding dict.
+                    id2embedding.update(dict(zip(ids, embeddings)))
+
+    # stage 9: Compute cosine scores.
+    labels = []
+    enrol_ids = []
+    test_ids = []
+    with open(VoxCeleb1.veri_test_file, 'r') as f:
+        for line in f.readlines():
+            label, enrol_id, test_id = line.strip().split(' ')
+            labels.append(int(label))
+            enrol_ids.append(enrol_id.split('.')[0].replace('/', '-'))
+            test_ids.append(test_id.split('.')[0].replace('/', '-'))
+
+    cos_sim_func = paddle.nn.CosineSimilarity(axis=1)
+    enrol_embeddings, test_embeddings = map(lambda ids: paddle.to_tensor(
+        np.asarray([id2embedding[id] for id in ids], dtype='float32')),
+                                            [enrol_ids, test_ids
+                                             ])  # (N, emb_size)
+    scores = cos_sim_func(enrol_embeddings, test_embeddings)
+    EER, threshold = compute_eer(np.asarray(labels), scores.numpy())
+    logger.info(
+        f'EER of verification test: {EER*100:.4f}%, score threshold: {threshold:.5f}'
+    )
+
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--device',
+                        choices=['cpu', 'gpu'],
+                        default="gpu",
+                        help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--config", 
+                        default=None,
+                        type=str,
+                        help="configuration file")
+    parser.add_argument("--data-dir",
+                        default="./data/",
+                        type=str,
+                        help="data directory")
+    parser.add_argument("--load-checkpoint",
+                        type=str,
+                        default='',
+                        help="Directory to load model checkpoint to contiune trainning.")
+    parser.add_argument("--global-embedding-norm",
+                        type=bool,
+                        default=True,
+                        help="Apply global normalization on speaker embeddings.")
+    parser.add_argument("--embedding-mean-norm",
+                        type=bool,
+                        default=True,
+                        help="Apply mean normalization on speaker embeddings.")
+    parser.add_argument("--embedding-std-norm",
+                        type=bool,
+                        default=False,
+                        help="Apply std normalization on speaker embeddings.")
+    args = parser.parse_args()
+    # yapf: enable
+    # https://yaml.org/type/float.html
+    config = CfgNode(new_allowed=True)
+    if args.config:
+        config.merge_from_file(args.config)
+
+    config.freeze()
+    print(config)
+    main(args, config)
diff --git a/paddlespeech/vector/exps/ecapa-tdnn/train.py b/paddlespeech/vector/exps/ecapa-tdnn/train.py
new file mode 100644
index 00000000..08a4ac1c
--- /dev/null
+++ b/paddlespeech/vector/exps/ecapa-tdnn/train.py
@@ -0,0 +1,298 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import os
+
+import numpy as np
+import paddle
+from paddle.io import BatchSampler
+from paddle.io import DataLoader
+from paddle.io import DistributedBatchSampler
+from yacs.config import CfgNode
+from paddleaudio.paddleaudio.compliance.librosa import melspectrogram
+from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddlespeech.s2t.utils.log import Log
+from paddlespeech.vector.io.augment import build_augment_pipeline
+from paddlespeech.vector.io.augment import waveform_augment
+from paddlespeech.vector.io.batch import feature_normalize
+from paddlespeech.vector.io.batch import waveform_collate_fn
+from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
+from paddlespeech.vector.modules.loss import AdditiveAngularMargin
+from paddlespeech.vector.modules.loss import LogSoftmaxWrapper
+from paddlespeech.vector.training.scheduler import CyclicLRScheduler
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.seeding import seed_everything
+from paddlespeech.vector.utils.time import Timer
+
+logger = Log(__name__).getlog()
+
+def main(args, config):
+    # stage0: set the training device, cpu or gpu
+    paddle.set_device(args.device)
+
+    # stage1: we must call the paddle.distributed.init_parallel_env() api at the begining
+    paddle.distributed.init_parallel_env()
+    nranks = paddle.distributed.get_world_size()
+    local_rank = paddle.distributed.get_rank()
+    # set the random seed, it is a must for multiprocess training
+    seed_everything(config.seed)
+
+    # stage2: data prepare, such vox1 and vox2 data, and augment noise data and pipline
+    # note: some cmd must do in rank==0, so wo will refactor the data prepare code
+    train_dataset = VoxCeleb1('train', target_dir=args.data_dir)
+    dev_dataset = VoxCeleb1('dev', target_dir=args.data_dir)
+
+    if args.augment:
+        augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
+    else:
+        augment_pipeline = []
+
+    # stage3: build the dnn backbone model network
+    ecapa_tdnn = EcapaTdnn(**config.model)
+
+    # stage4: build the speaker verification train instance with backbone model
+    model = SpeakerIdetification(
+        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
+
+    # stage5: build the optimizer, we now only construct the AdamW optimizer
+    lr_schedule = CyclicLRScheduler(
+        base_lr=config.learning_rate, max_lr=1e-3, step_size=140000 // nranks)
+    optimizer = paddle.optimizer.AdamW(
+        learning_rate=lr_schedule, parameters=model.parameters())
+
+    # stage6: build the loss function, we now only support LogSoftmaxWrapper
+    criterion = LogSoftmaxWrapper(
+        loss_fn=AdditiveAngularMargin(margin=0.2, scale=30))
+
+    # stage7: confirm training start epoch
+    #         if pre-trained model exists, start epoch confirmed by the pre-trained model
+    start_epoch = 0
+    if args.load_checkpoint:
+        logger.info("load the check point")
+        args.load_checkpoint = os.path.abspath(
+            os.path.expanduser(args.load_checkpoint))
+        try:
+            # load model checkpoint
+            state_dict = paddle.load(
+                os.path.join(args.load_checkpoint, 'model.pdparams'))
+            model.set_state_dict(state_dict)
+
+            # load optimizer checkpoint
+            state_dict = paddle.load(
+                os.path.join(args.load_checkpoint, 'model.pdopt'))
+            optimizer.set_state_dict(state_dict)
+            if local_rank == 0:
+                logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
+        except FileExistsError:
+            if local_rank == 0:
+                logger.info('Train from scratch.')
+
+        try:
+            start_epoch = int(args.load_checkpoint[-1])
+            logger.info(f'Restore training from epoch {start_epoch}.')
+        except ValueError:
+            pass
+
+    # stage8: we build the batch sampler for paddle.DataLoader
+    train_sampler = DistributedBatchSampler(
+        train_dataset,
+        batch_size=config.batch_size,
+        shuffle=True,
+        drop_last=False)
+    train_loader = DataLoader(
+        train_dataset,
+        batch_sampler=train_sampler,
+        num_workers=config.num_workers,
+        collate_fn=waveform_collate_fn,
+        return_list=True,
+        use_buffer_reader=True, )
+
+    # stage9: start to train
+    #         we will comment the training process
+    steps_per_epoch = len(train_sampler)
+    timer = Timer(steps_per_epoch * config.epochs)
+    timer.start()
+
+    for epoch in range(start_epoch + 1, config.epochs + 1):
+        # at the begining, model must set to train mode
+        model.train()
+
+        avg_loss = 0
+        num_corrects = 0
+        num_samples = 0
+        for batch_idx, batch in enumerate(train_loader):
+            # stage 9-1: batch data is audio sample points and speaker id label
+            waveforms, labels = batch['waveforms'], batch['labels']
+
+            # stage 9-2: audio sample augment method, which is done on the audio sample point
+            if len(augment_pipeline) != 0:
+                waveforms = waveform_augment(waveforms, augment_pipeline)
+                labels = paddle.concat(
+                    [labels for i in range(len(augment_pipeline) + 1)])
+
+            # stage 9-3: extract the audio feats,such fbank, mfcc, spectrogram
+            feats = []
+            for waveform in waveforms.numpy():
+                feat = melspectrogram(x=waveform, **config.feature)
+                feats.append(feat)
+            feats = paddle.to_tensor(np.asarray(feats))
+
+            # stage 9-4: feature normalize, which help converge and imporve the performance
+            feats = feature_normalize(
+                feats, mean_norm=True, std_norm=False)  # Features normalization
+
+            # stage 9-5: model forward, such ecapa-tdnn, x-vector
+            logits = model(feats)
+
+            # stage 9-6: loss function criterion, such AngularMargin, AdditiveAngularMargin
+            loss = criterion(logits, labels)
+
+            # stage 9-7: update the gradient and clear the gradient cache
+            loss.backward()
+            optimizer.step()
+            if isinstance(optimizer._learning_rate,
+                          paddle.optimizer.lr.LRScheduler):
+                optimizer._learning_rate.step()
+            optimizer.clear_grad()
+
+            # stage 9-8: Calculate average loss per batch
+            avg_loss += loss.numpy()[0]
+
+            # stage 9-9: Calculate metrics, which is one-best accuracy
+            preds = paddle.argmax(logits, axis=1)
+            num_corrects += (preds == labels).numpy().sum()
+            num_samples += feats.shape[0]
+            timer.count()  # step plus one in timer
+
+            # stage 9-10: print the log information only on 0-rank per log-freq batchs
+            if (batch_idx + 1) % config.log_freq == 0 and local_rank == 0:
+                lr = optimizer.get_lr()
+                avg_loss /= config.log_freq
+                avg_acc = num_corrects / num_samples
+
+                print_msg = 'Train Epoch={}/{}, Step={}/{}'.format(
+                    epoch, config.epochs, batch_idx + 1, steps_per_epoch)
+                print_msg += ' loss={:.4f}'.format(avg_loss)
+                print_msg += ' acc={:.4f}'.format(avg_acc)
+                print_msg += ' lr={:.4E} step/sec={:.2f} | ETA {}'.format(
+                    lr, timer.timing, timer.eta)
+                logger.info(print_msg)
+
+                avg_loss = 0
+                num_corrects = 0
+                num_samples = 0
+
+        # stage 9-11: save the model parameters only on 0-rank per save-freq batchs
+        if epoch % config.save_freq == 0 and batch_idx + 1 == steps_per_epoch:
+            if local_rank != 0:
+                paddle.distributed.barrier(
+                )  # Wait for valid step in main process
+                continue  # Resume trainning on other process
+
+            # stage 9-12: construct the valid dataset dataloader
+            dev_sampler = BatchSampler(
+                dev_dataset,
+                batch_size=config.batch_size // 4,
+                shuffle=False,
+                drop_last=False)
+            dev_loader = DataLoader(
+                dev_dataset,
+                batch_sampler=dev_sampler,
+                collate_fn=waveform_collate_fn,
+                num_workers=config.num_workers,
+                return_list=True, )
+
+            # set the model to eval mode
+            model.eval()
+            num_corrects = 0
+            num_samples = 0
+
+            # stage 9-13: evaluation the valid dataset batch data
+            logger.info('Evaluate on validation dataset')
+            with paddle.no_grad():
+                for batch_idx, batch in enumerate(dev_loader):
+                    waveforms, labels = batch['waveforms'], batch['labels']
+
+                    feats = []
+                    for waveform in waveforms.numpy():
+                        # feat = melspectrogram(x=waveform, **cpu_feat_conf)
+                        feat = melspectrogram(x=waveform, **config.feature)
+                        feats.append(feat)
+
+                    feats = paddle.to_tensor(np.asarray(feats))
+                    feats = feature_normalize(
+                        feats, mean_norm=True, std_norm=False)
+                    logits = model(feats)
+
+                    preds = paddle.argmax(logits, axis=1)
+                    num_corrects += (preds == labels).numpy().sum()
+                    num_samples += feats.shape[0]
+
+            print_msg = '[Evaluation result]'
+            print_msg += ' dev_acc={:.4f}'.format(num_corrects / num_samples)
+            logger.info(print_msg)
+
+            # stage 9-14: Save model parameters
+            save_dir = os.path.join(args.checkpoint_dir,
+                                    'epoch_{}'.format(epoch))
+            logger.info('Saving model checkpoint to {}'.format(save_dir))
+            paddle.save(model.state_dict(),
+                        os.path.join(save_dir, 'model.pdparams'))
+            paddle.save(optimizer.state_dict(),
+                        os.path.join(save_dir, 'model.pdopt'))
+
+            if nranks > 1:
+                paddle.distributed.barrier()  # Main process
+
+
+if __name__ == "__main__":
+    # yapf: disable
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument('--device',
+                        choices=['cpu', 'gpu'],
+                        default="cpu",
+                        help="Select which device to train model, defaults to gpu.")
+    parser.add_argument("--config", 
+                        default=None,
+                        type=str,
+                        help="configuration file")
+    parser.add_argument("--data-dir",
+                        default="./data/",
+                        type=str,
+                        help="data directory")
+    parser.add_argument("--load-checkpoint",
+                        type=str,
+                        default=None,
+                        help="Directory to load model checkpoint to contiune trainning.")
+    parser.add_argument("--checkpoint-dir",
+                        type=str,
+                        default='./checkpoint',
+                        help="Directory to save model checkpoints.")
+    parser.add_argument("--augment",
+                        action="store_true",
+                        default=False,
+                        help="Apply audio augments.")
+
+    args = parser.parse_args()
+    # yapf: enable
+
+    # https://yaml.org/type/float.html
+    config = CfgNode(new_allowed=True)
+    if args.config:
+        config.merge_from_file(args.config)
+
+    config.freeze()
+    print(config)
+
+    main(args, config)

From 8ed5c287a323b0d59b3ef44855f579f8a03102f9 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 10 Mar 2022 16:54:48 +0800
Subject: [PATCH 026/126] add vox2 data into VoxCeleb class

---
 examples/voxceleb/README.md                   | 33 -------
 examples/voxceleb/sv0/local/data_prepare.py   | 33 ++++++-
 examples/voxceleb/sv0/path.sh                 | 16 +++-
 examples/voxceleb/sv0/run.sh                  | 34 ++++++-
 paddleaudio/paddleaudio/datasets/__init__.py  |  2 +-
 paddleaudio/paddleaudio/datasets/voxceleb.py  | 44 ++++++---
 .../extract_speaker_embedding.py              |  3 +-
 .../speaker_verification_cosine.py            | 17 ++--
 .../exps/{ecapa-tdnn => ecapa_tdnn}/train.py  | 14 +--
 paddlespeech/vector/io/augment.py             |  1 +
 paddlespeech/vector/models/ecapa_tdnn.py      | 96 +++++++------------
 11 files changed, 158 insertions(+), 135 deletions(-)
 rename paddlespeech/vector/exps/{ecapa-tdnn => ecapa_tdnn}/extract_speaker_embedding.py (99%)
 rename paddlespeech/vector/exps/{ecapa-tdnn => ecapa_tdnn}/speaker_verification_cosine.py (96%)
 rename paddlespeech/vector/exps/{ecapa-tdnn => ecapa_tdnn}/train.py (97%)

diff --git a/examples/voxceleb/README.md b/examples/voxceleb/README.md
index 59fb491c..fc847cd8 100644
--- a/examples/voxceleb/README.md
+++ b/examples/voxceleb/README.md
@@ -23,39 +23,6 @@ VoxCeleb2 stores files with the m4a audio format. To use them in PaddleSpeech,
 ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 %s
 ```
 
-``` shell
-# copy this to root directory of data and 
-# chmod a+x convert.sh
-# ./convert.sh
-# https://unix.stackexchange.com/questions/103920/parallelize-a-bash-for-loop
-
-open_sem(){
-    mkfifo pipe-$$
-    exec 3<>pipe-$$
-    rm pipe-$$
-    local i=$1
-    for((;i>0;i--)); do
-        printf %s 000 >&3
-    done
-}
-run_with_lock(){
-    local x
-    read -u 3 -n 3 x && ((0==x)) || exit $x
-    (
-     ( "$@"; )
-    printf '%.3d' $? >&3
-    )&
-}
-
-N=32 # number of vCPU
-open_sem $N
-for f in $(find . -name "*.m4a"); do
-    run_with_lock ffmpeg -loglevel panic -i "$f" -ar 16000 "${f%.*}.wav"
-done
-```
-
 You can do the conversion using ffmpeg  https://gist.github.com/seungwonpark/4f273739beef2691cd53b5c39629d830). This operation might take several hours and should be only once.
 
 3. Put all the wav files in a folder called `wav`. You should have something like `voxceleb2/wav/id*/*.wav` (e.g, `voxceleb2/wav/id00012/21Uxsk56VDQ/00001.wav`)
-
-4. 
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/local/data_prepare.py b/examples/voxceleb/sv0/local/data_prepare.py
index 1a0a6392..b906b5da 100644
--- a/examples/voxceleb/sv0/local/data_prepare.py
+++ b/examples/voxceleb/sv0/local/data_prepare.py
@@ -1,17 +1,32 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 import argparse
 import os
 
 import numpy as np
 import paddle
 
-from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.augment import build_augment_pipeline
 from paddlespeech.vector.training.seeding import seed_everything
 
 logger = Log(__name__).getlog()
 
+
 def main(args):
+
     # stage0: set the cpu device, all data prepare process will be done in cpu mode
     paddle.set_device("cpu")
     # set the random seed, it is a must for multiprocess training
@@ -19,14 +34,18 @@ def main(args):
 
     # stage 1: generate the voxceleb csv file
     # Note: this may occurs c++ execption, but the program will execute fine
-    # so we can ignore the execption 
-    train_dataset = VoxCeleb1('train', target_dir=args.data_dir)
-    dev_dataset = VoxCeleb1('dev', target_dir=args.data_dir)
+    # so we ignore the execption 
+    # we explicitly pass the vox2 base path to data prepare and generate the audio info
+    train_dataset = VoxCeleb(
+        'train', target_dir=args.data_dir, vox2_base_path=args.vox2_base_path)
+    dev_dataset = VoxCeleb(
+        'dev', target_dir=args.data_dir, vox2_base_path=args.vox2_base_path)
 
     # stage 2: generate the augment noise csv file
     if args.augment:
         augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
 
+
 if __name__ == "__main__":
     # yapf: disable
     parser = argparse.ArgumentParser(__doc__)
@@ -38,10 +57,14 @@ if __name__ == "__main__":
                         default="./data/",
                         type=str,
                         help="data directory")
+    parser.add_argument("--vox2-base-path",
+                        default=None,
+                        type=str,
+                        help="vox2 base path, where is store the wav audio")
     parser.add_argument("--augment",
                         action="store_true",
                         default=False,
                         help="Apply audio augments.")
     args = parser.parse_args()
     # yapf: enable
-    main(args)                    
\ No newline at end of file
+    main(args)
diff --git a/examples/voxceleb/sv0/path.sh b/examples/voxceleb/sv0/path.sh
index 6d19f994..2be098e0 100755
--- a/examples/voxceleb/sv0/path.sh
+++ b/examples/voxceleb/sv0/path.sh
@@ -1,3 +1,17 @@
+#!/bin/bash
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 export MAIN_ROOT=`realpath ${PWD}/../../../`
 
 export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
@@ -10,5 +24,5 @@ export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
 
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/
 
-MODEL=ecapa-tdnn
+MODEL=ecapa_tdnn
 export BIN_DIR=${MAIN_ROOT}/paddlespeech/vector/exps/${MODEL}
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index 2c0e55a6..769332eb 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -1,4 +1,17 @@
 #!/bin/bash
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 . ./path.sh
 set -e
@@ -11,19 +24,30 @@ set -e
 # stage 3: extract the training embeding to train the LDA and PLDA
 ######################################################################
 
-# you can set the variable PPAUDIO_HOME to specifiy the downloaded the vox1 and vox2 dataset
-# default the dataset is the ~/.paddleaudio/
+# we can set the variable PPAUDIO_HOME to specifiy the root directory of the downloaded vox1 and vox2 dataset 
+# default the dataset will be stored in the ~/.paddleaudio/
+# the vox2 dataset is stored in m4a format, we need to convert the audio from m4a to wav yourself
+# and put all of them to ${PPAUDIO_HOME}/datasets/vox2
+# we will find the wav from ${PPAUDIO_HOME}/datasets/vox1/wav and ${PPAUDIO_HOME}/datasets/vox2/wav
 # export PPAUDIO_HOME=
 
 stage=0
-dir=data.bak/                     # data directory
-exp_dir=exp/ecapa-tdnn/           # experiment directory
+# data directory
+# if we set the variable ${dir}, we will store the wav info to this directory
+# otherwise, we will store the wav info to vox1 and vox2 directory respectively
+dir=data/                          
+exp_dir=exp/ecapa-tdnn/            # experiment directory
+
+# vox2 wav path, we must convert the m4a format to wav format 
+# and store them in the ${PPAUDIO_HOME}/datasets/vox2/wav/ directory
+vox2_base_path=${PPAUDIO_HOME}/datasets/vox2/wav/
 mkdir -p ${dir}
 mkdir -p ${exp_dir}
 
 if [ $stage -le 0 ]; then 
      # stage 0: data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
-     python3 local/data_prepare.py --data-dir ${dir} --augment
+     python3 local/data_prepare.py \
+     --data-dir ${dir} --augment --vox2-base-path ${vox2_base_path}
 fi 
 
 if [ $stage -le 1 ]; then
diff --git a/paddleaudio/paddleaudio/datasets/__init__.py b/paddleaudio/paddleaudio/datasets/__init__.py
index cbf9b3ae..6f44e977 100644
--- a/paddleaudio/paddleaudio/datasets/__init__.py
+++ b/paddleaudio/paddleaudio/datasets/__init__.py
@@ -15,5 +15,5 @@ from .esc50 import ESC50
 from .gtzan import GTZAN
 from .tess import TESS
 from .urban_sound import UrbanSound8K
-from .voxceleb import VoxCeleb1
+from .voxceleb import VoxCeleb
 from .rirs_noises import OpenRIRNoise
diff --git a/paddleaudio/paddleaudio/datasets/voxceleb.py b/paddleaudio/paddleaudio/datasets/voxceleb.py
index 4989accb..f8d634f2 100644
--- a/paddleaudio/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/paddleaudio/datasets/voxceleb.py
@@ -25,10 +25,10 @@ from paddle.io import Dataset
 from pathos.multiprocessing import Pool
 from tqdm import tqdm
 
-from .dataset import feat_funcs
 from ..backends import load as load_audio
 from ..utils import DATA_HOME
 from ..utils import decompress
+from .dataset import feat_funcs
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.utils.download import download_and_decompress
 from utils.utility import download
@@ -36,10 +36,10 @@ from utils.utility import unpack
 
 logger = Log(__name__).getlog()
 
-__all__ = ['VoxCeleb1']
+__all__ = ['VoxCeleb']
 
 
-class VoxCeleb1(Dataset):
+class VoxCeleb(Dataset):
     source_url = 'https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a/'
     archieves_audio_dev = [
         {
@@ -94,8 +94,18 @@ class VoxCeleb1(Dataset):
             split_ratio: float=0.9,  # train split ratio
             seed: int=0,
             target_dir: str=None,
+            vox2_base_path=None,
             **kwargs):
-
+        """VoxCeleb data prepare and get the specific dataset audio info
+
+        Args:
+            subset (str, optional): dataset name, such as train, dev, enroll or test. Defaults to 'train'.
+            feat_type (str, optional): feat type, such raw, melspectrogram(fbank) or mfcc . Defaults to 'raw'.
+            random_chunk (bool, optional): random select a duration from audio. Defaults to True.
+            chunk_duration (float, optional): chunk duration if random_chunk flag is set. Defaults to 3.0.
+            target_dir (str, optional): data dir, audio info will be stored in this directory. Defaults to None.
+            vox2_base_path (_type_, optional): vox2 directory. vox2 data must be converted from m4a to wav. Defaults to None.
+        """
         assert subset in self.subsets, \
             'Dataset subset must be one in {}, but got {}'.format(self.subsets, subset)
 
@@ -106,19 +116,20 @@ class VoxCeleb1(Dataset):
         self.random_chunk = random_chunk
         self.chunk_duration = chunk_duration
         self.split_ratio = split_ratio
-        self.target_dir = target_dir if target_dir else VoxCeleb1.base_path
+        self.target_dir = target_dir if target_dir else VoxCeleb.base_path
+        self.vox2_base_path = vox2_base_path
 
         # if we set the target dir, we will change the vox data info data from base path to target dir
-        VoxCeleb1.csv_path = os.path.join(
-            target_dir, "voxceleb", 'csv') if target_dir else VoxCeleb1.csv_path
-        VoxCeleb1.meta_path = os.path.join(
+        VoxCeleb.csv_path = os.path.join(
+            target_dir, "voxceleb", 'csv') if target_dir else VoxCeleb.csv_path
+        VoxCeleb.meta_path = os.path.join(
             target_dir, "voxceleb",
-            'meta') if target_dir else VoxCeleb1.meta_path
-        VoxCeleb1.veri_test_file = os.path.join(VoxCeleb1.meta_path,
-                                                'veri_test2.txt')
+            'meta') if target_dir else VoxCeleb.meta_path
+        VoxCeleb.veri_test_file = os.path.join(VoxCeleb.meta_path,
+                                               'veri_test2.txt')
         # self._data = self._get_data()[:1000]  # KP: Small dataset test.
         self._data = self._get_data()
-        super(VoxCeleb1, self).__init__()
+        super(VoxCeleb, self).__init__()
 
         # Set up a seed to reproduce training or predicting result.
         # random.seed(seed)
@@ -300,7 +311,14 @@ class VoxCeleb1(Dataset):
         # get all the train and dev audios file path
         audio_files = []
         speakers = set()
-        for path in [self.wav_path]:
+        for path in [self.wav_path, self.vox2_base_path]:
+            # if vox2 directory is not set and vox2 is not a directory 
+            # we will not process this directory
+            if not path or not os.path.exists(path):
+                logger.warning(
+                    f"{path} is an invalid path, please check again, "
+                    "and we will ignore the vox2 base path")
+                continue
             for file in glob.glob(
                     os.path.join(path, "**", "*.wav"), recursive=True):
                 spk = file.split('/wav/')[1].split('/')[0]
diff --git a/paddlespeech/vector/exps/ecapa-tdnn/extract_speaker_embedding.py b/paddlespeech/vector/exps/ecapa_tdnn/extract_speaker_embedding.py
similarity index 99%
rename from paddlespeech/vector/exps/ecapa-tdnn/extract_speaker_embedding.py
rename to paddlespeech/vector/exps/ecapa_tdnn/extract_speaker_embedding.py
index 78498c61..44cbd204 100644
--- a/paddlespeech/vector/exps/ecapa-tdnn/extract_speaker_embedding.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/extract_speaker_embedding.py
@@ -28,6 +28,7 @@ from paddlespeech.vector.training.seeding import seed_everything
 
 logger = Log(__name__).getlog()
 
+
 def extract_audio_embedding(args, config):
     # stage 0: set the training device, cpu or gpu
     paddle.set_device(args.device)
@@ -83,7 +84,7 @@ if __name__ == "__main__":
                         choices=['cpu', 'gpu'],
                         default="gpu",
                         help="Select which device to train model, defaults to gpu.")
-    parser.add_argument("--config", 
+    parser.add_argument("--config",
                         default=None,
                         type=str,
                         help="configuration file")
diff --git a/paddlespeech/vector/exps/ecapa-tdnn/speaker_verification_cosine.py b/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
similarity index 96%
rename from paddlespeech/vector/exps/ecapa-tdnn/speaker_verification_cosine.py
rename to paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
index 4d85bd62..01a3506a 100644
--- a/paddlespeech/vector/exps/ecapa-tdnn/speaker_verification_cosine.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
@@ -17,15 +17,15 @@ import os
 
 import numpy as np
 import paddle
-from yacs.config import CfgNode
 import paddle.nn.functional as F
 from paddle.io import BatchSampler
 from paddle.io import DataLoader
 from tqdm import tqdm
+from yacs.config import CfgNode
 
-from paddleaudio.paddleaudio.datasets import VoxCeleb1
-from paddlespeech.s2t.utils.log import Log
+from paddleaudio.paddleaudio.datasets import VoxCeleb
 from paddleaudio.paddleaudio.metric import compute_eer
+from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.batch import batch_feature_normalize
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
 from paddlespeech.vector.modules.sid_model import SpeakerIdetification
@@ -33,6 +33,7 @@ from paddlespeech.vector.training.seeding import seed_everything
 
 logger = Log(__name__).getlog()
 
+
 def main(args, config):
     # stage0: set the training device, cpu or gpu
     paddle.set_device(args.device)
@@ -44,7 +45,7 @@ def main(args, config):
 
     # stage2: build the speaker verification eval instance with backbone model
     model = SpeakerIdetification(
-        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
+        backbone=ecapa_tdnn, num_class=VoxCeleb.num_speakers)
 
     # stage3: load the pre-trained model
     args.load_checkpoint = os.path.abspath(
@@ -57,7 +58,7 @@ def main(args, config):
     logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
 
     # stage4: construct the enroll and test dataloader
-    enroll_dataset = VoxCeleb1(
+    enroll_dataset = VoxCeleb(
         subset='enroll',
         target_dir=args.data_dir,
         feat_type='melspectrogram',
@@ -73,7 +74,7 @@ def main(args, config):
                     num_workers=config.num_workers,
                     return_list=True,)
 
-    test_dataset = VoxCeleb1(
+    test_dataset = VoxCeleb(
         subset='test',
         target_dir=args.data_dir,
         feat_type='melspectrogram',
@@ -145,7 +146,7 @@ def main(args, config):
     labels = []
     enrol_ids = []
     test_ids = []
-    with open(VoxCeleb1.veri_test_file, 'r') as f:
+    with open(VoxCeleb.veri_test_file, 'r') as f:
         for line in f.readlines():
             label, enrol_id, test_id = line.strip().split(' ')
             labels.append(int(label))
@@ -171,7 +172,7 @@ if __name__ == "__main__":
                         choices=['cpu', 'gpu'],
                         default="gpu",
                         help="Select which device to train model, defaults to gpu.")
-    parser.add_argument("--config", 
+    parser.add_argument("--config",
                         default=None,
                         type=str,
                         help="configuration file")
diff --git a/paddlespeech/vector/exps/ecapa-tdnn/train.py b/paddlespeech/vector/exps/ecapa_tdnn/train.py
similarity index 97%
rename from paddlespeech/vector/exps/ecapa-tdnn/train.py
rename to paddlespeech/vector/exps/ecapa_tdnn/train.py
index 08a4ac1c..6e6e5ab2 100644
--- a/paddlespeech/vector/exps/ecapa-tdnn/train.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/train.py
@@ -20,8 +20,9 @@ from paddle.io import BatchSampler
 from paddle.io import DataLoader
 from paddle.io import DistributedBatchSampler
 from yacs.config import CfgNode
+
 from paddleaudio.paddleaudio.compliance.librosa import melspectrogram
-from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb1
+from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.augment import build_augment_pipeline
 from paddlespeech.vector.io.augment import waveform_augment
@@ -30,13 +31,14 @@ from paddlespeech.vector.io.batch import waveform_collate_fn
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
 from paddlespeech.vector.modules.loss import AdditiveAngularMargin
 from paddlespeech.vector.modules.loss import LogSoftmaxWrapper
-from paddlespeech.vector.training.scheduler import CyclicLRScheduler
 from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+from paddlespeech.vector.training.scheduler import CyclicLRScheduler
 from paddlespeech.vector.training.seeding import seed_everything
 from paddlespeech.vector.utils.time import Timer
 
 logger = Log(__name__).getlog()
 
+
 def main(args, config):
     # stage0: set the training device, cpu or gpu
     paddle.set_device(args.device)
@@ -50,8 +52,8 @@ def main(args, config):
 
     # stage2: data prepare, such vox1 and vox2 data, and augment noise data and pipline
     # note: some cmd must do in rank==0, so wo will refactor the data prepare code
-    train_dataset = VoxCeleb1('train', target_dir=args.data_dir)
-    dev_dataset = VoxCeleb1('dev', target_dir=args.data_dir)
+    train_dataset = VoxCeleb('train', target_dir=args.data_dir)
+    dev_dataset = VoxCeleb('dev', target_dir=args.data_dir)
 
     if args.augment:
         augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
@@ -63,7 +65,7 @@ def main(args, config):
 
     # stage4: build the speaker verification train instance with backbone model
     model = SpeakerIdetification(
-        backbone=ecapa_tdnn, num_class=VoxCeleb1.num_speakers)
+        backbone=ecapa_tdnn, num_class=VoxCeleb.num_speakers)
 
     # stage5: build the optimizer, we now only construct the AdamW optimizer
     lr_schedule = CyclicLRScheduler(
@@ -263,7 +265,7 @@ if __name__ == "__main__":
                         choices=['cpu', 'gpu'],
                         default="cpu",
                         help="Select which device to train model, defaults to gpu.")
-    parser.add_argument("--config", 
+    parser.add_argument("--config",
                         default=None,
                         type=str,
                         help="configuration file")
diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
index 76312978..1b9d1fbd 100644
--- a/paddlespeech/vector/io/augment.py
+++ b/paddlespeech/vector/io/augment.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# this is modified from https://github.com/speechbrain/speechbrain/blob/085be635c07f16d42cd1295045bc46c407f1e15b/speechbrain/lobes/augment.py
 import math
 import os
 from typing import List
diff --git a/paddlespeech/vector/models/ecapa_tdnn.py b/paddlespeech/vector/models/ecapa_tdnn.py
index 4c960e11..0e7287cd 100644
--- a/paddlespeech/vector/models/ecapa_tdnn.py
+++ b/paddlespeech/vector/models/ecapa_tdnn.py
@@ -19,16 +19,6 @@ import paddle.nn.functional as F
 
 
 def length_to_mask(length, max_len=None, dtype=None):
-    """_summary_
-
-    Args:
-        length (_type_): _description_
-        max_len (_type_, optional): _description_. Defaults to None.
-        dtype (_type_, optional): _description_. Defaults to None.
-
-    Returns:
-        _type_: _description_
-    """
     assert len(length.shape) == 1
 
     if max_len is None:
@@ -60,15 +50,15 @@ class Conv1d(nn.Layer):
         """_summary_
 
         Args:
-            in_channels (_type_): _description_
-            out_channels (_type_): _description_
-            kernel_size (_type_): _description_
-            stride (int, optional): _description_. Defaults to 1.
-            padding (str, optional): _description_. Defaults to "same".
-            dilation (int, optional): _description_. Defaults to 1.
-            groups (int, optional): _description_. Defaults to 1.
-            bias (bool, optional): _description_. Defaults to True.
-            padding_mode (str, optional): _description_. Defaults to "reflect".
+            in_channels (int): intput channel or input data dimensions
+            out_channels (int): output channel or output data dimensions
+            kernel_size (int): kernel size of 1-d convolution
+            stride (int, optional): strid in 1-d convolution . Defaults to 1.
+            padding (str, optional): padding value. Defaults to "same".
+            dilation (int, optional): dilation in 1-d convolution. Defaults to 1.
+            groups (int, optional): groups in 1-d convolution. Defaults to 1.
+            bias (bool, optional): bias in 1-d convolution . Defaults to True.
+            padding_mode (str, optional): padding mode. Defaults to "reflect".
         """
         super().__init__()
 
@@ -89,17 +79,6 @@ class Conv1d(nn.Layer):
             bias_attr=bias, )
 
     def forward(self, x):
-        """_summary_
-
-        Args:
-            x (_type_): _description_
-
-        Raises:
-            ValueError: _description_
-
-        Returns:
-            _type_: _description_
-        """
         if self.padding == "same":
             x = self._manage_padding(x, self.kernel_size, self.dilation,
                                      self.stride)
@@ -109,17 +88,6 @@ class Conv1d(nn.Layer):
         return self.conv(x)
 
     def _manage_padding(self, x, kernel_size: int, dilation: int, stride: int):
-        """_summary_
-
-        Args:
-            x (_type_): _description_
-            kernel_size (int): _description_
-            dilation (int): _description_
-            stride (int): _description_
-
-        Returns:
-            _type_: _description_
-        """
         L_in = x.shape[-1]  # Detecting input shape
         padding = self._get_padding_elem(L_in, stride, kernel_size,
                                          dilation)  # Time padding
@@ -133,17 +101,6 @@ class Conv1d(nn.Layer):
                           stride: int,
                           kernel_size: int,
                           dilation: int):
-        """_summary_
-
-        Args:
-            L_in (int): _description_
-            stride (int): _description_
-            kernel_size (int): _description_
-            dilation (int): _description_
-
-        Returns:
-            _type_: _description_
-        """
         if stride > 1:
             n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1)
             L_out = stride * (n_steps - 1) + kernel_size * dilation
@@ -220,8 +177,8 @@ class Res2NetBlock(nn.Layer):
         Args:
             in_channels (int): input channels or input dimensions
             out_channels (int): output channels or output dimensions
-            scale (int, optional): _description_. Defaults to 8.
-            dilation (int, optional): _description_. Defaults to 1.
+            scale (int, optional): scale in res2net bolck. Defaults to 8.
+            dilation (int, optional): dilation of 1-d convolution in TDNN block. Defaults to 1.
         """
         super().__init__()
         assert in_channels % scale == 0
@@ -358,15 +315,16 @@ class SERes2NetBlock(nn.Layer):
             dilation=1,
             activation=nn.ReLU, ):
         """Implementation of Squeeze-Extraction Res2Blocks in ECAPA-TDNN network model
-
+           The paper is refered "Squeeze-and-Excitation Networks"
+           whose url is: https://arxiv.org/pdf/1709.01507.pdf
         Args:
             in_channels (int): input channels or input data dimensions
-            out_channels (_type_): _description_
-            res2net_scale (int, optional): _description_. Defaults to 8.
-            se_channels (int, optional): _description_. Defaults to 128.
-            kernel_size (int, optional): _description_. Defaults to 1.
-            dilation (int, optional): _description_. Defaults to 1.
-            activation (_type_, optional): _description_. Defaults to nn.ReLU.
+            out_channels (int): output channels or output data dimensions
+            res2net_scale (int, optional): scale in the res2net block. Defaults to 8.
+            se_channels (int, optional): embedding dimensions of res2net block. Defaults to 128.
+            kernel_size (int, optional): kernel size of 1-d convolution in TDNN block. Defaults to 1.
+            dilation (int, optional): dilation of 1-d convolution in TDNN block. Defaults to 1.
+            activation (paddle.nn.class, optional): activation function. Defaults to nn.ReLU.
         """
         super().__init__()
         self.out_channels = out_channels
@@ -419,7 +377,21 @@ class EcapaTdnn(nn.Layer):
             res2net_scale=8,
             se_channels=128,
             global_context=True, ):
-
+        """Implementation of ECAPA-TDNN backbone model network
+           The paper is refered as "ECAPA-TDNN: Emphasized Channel Attention, Propagation and Aggregation in TDNN Based Speaker Verification"
+           whose url is: https://arxiv.org/abs/2005.07143
+        Args:
+            input_size (_type_): input fature dimension
+            lin_neurons (int, optional): speaker embedding size. Defaults to 192.
+            activation (paddle.nn.class, optional): activation function. Defaults to nn.ReLU.
+            channels (list, optional): inter embedding dimension. Defaults to [512, 512, 512, 512, 1536].
+            kernel_sizes (list, optional): kernel size of 1-d convolution in TDNN block . Defaults to [5, 3, 3, 3, 1].
+            dilations (list, optional): dilations of 1-d convolution in TDNN block. Defaults to [1, 2, 3, 4, 1].
+            attention_channels (int, optional): attention dimensions. Defaults to 128.
+            res2net_scale (int, optional): scale value in res2net. Defaults to 8.
+            se_channels (int, optional): dimensions of squeeze-excitation block. Defaults to 128.
+            global_context (bool, optional): global context flag. Defaults to True.
+        """
         super().__init__()
         assert len(channels) == len(kernel_sizes)
         assert len(channels) == len(dilations)

From 311fa87a1193e784913434d502df3d5942e50b1f Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sun, 13 Mar 2022 22:56:26 +0800
Subject: [PATCH 027/126] add some comments to the code

---
 examples/voxceleb/sv0/conf/ecapa_tdnn.yaml    | 20 ++++--
 examples/voxceleb/sv0/run.sh                  |  3 +-
 paddleaudio/paddleaudio/metric/__init__.py    |  3 +-
 paddleaudio/paddleaudio/metric/eer.py         | 66 +++++++++++++++++++
 ...ct_speaker_embedding.py => extract_emb.py} |  0
 .../ecapa_tdnn/speaker_verification_cosine.py | 14 ++--
 paddlespeech/vector/exps/ecapa_tdnn/train.py  |  6 +-
 paddlespeech/vector/io/augment.py             |  3 +-
 paddlespeech/vector/io/batch.py               |  3 +
 9 files changed, 99 insertions(+), 19 deletions(-)
 rename paddlespeech/vector/exps/ecapa_tdnn/{extract_speaker_embedding.py => extract_emb.py} (100%)

diff --git a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
index 33304054..720326f8 100644
--- a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
+++ b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
@@ -1,3 +1,12 @@
+###########################################
+#                Data                 #
+###########################################
+batch_size: 32
+num_workers: 2
+num_speakers: 7205 # 1211 vox1, 5994 vox2, 7205 vox1+2, test speakers: 41
+shuffle: True
+random_chunk: True
+
 ###########################################################
 #                FEATURE EXTRACTION SETTING               #
 ###########################################################
@@ -7,7 +16,6 @@ feature:
   window_size: 400     #25ms, sample rate 16000, 25 * 16000 / 1000 = 400 
   hop_length: 160     #10ms, sample rate 16000, 10 * 16000 / 1000 = 160
 
-
 ###########################################################
 #                       MODEL SETTING                     #
 ###########################################################
@@ -15,9 +23,8 @@ feature:
 # if we want use another model, please choose another configuration yaml file
 model:
   input_size: 80
-  ##"channels": [1024, 1024, 1024, 1024, 3072],
   # "channels": [512, 512, 512, 512, 1536],
-  channels: [512, 512, 512, 512, 1536]
+  channels: [1024, 1024, 1024, 1024, 3072]
   kernel_sizes: [5, 3, 3, 3, 1]
   dilations: [1, 2, 3, 4, 1]
   attention_channels: 128
@@ -26,10 +33,9 @@ model:
 ###########################################
 #                Training                 #
 ###########################################
-seed: 0
+seed: 1986 # according from speechbrain configuration
 epochs: 10
-batch_size: 32
-num_workers: 2
 save_freq: 10
-log_freq: 10
+log_interval: 10
 learning_rate: 1e-8
+
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index 769332eb..c5dc3dd2 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -47,7 +47,8 @@ mkdir -p ${exp_dir}
 if [ $stage -le 0 ]; then 
      # stage 0: data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
      python3 local/data_prepare.py \
-     --data-dir ${dir} --augment --vox2-base-path ${vox2_base_path}
+     --data-dir ${dir} --augment --vox2-base-path ${vox2_base_path} \
+     --config conf/ecapa_tdnn.yaml
 fi 
 
 if [ $stage -le 1 ]; then
diff --git a/paddleaudio/paddleaudio/metric/__init__.py b/paddleaudio/paddleaudio/metric/__init__.py
index b435571d..8e5ca9f7 100644
--- a/paddleaudio/paddleaudio/metric/__init__.py
+++ b/paddleaudio/paddleaudio/metric/__init__.py
@@ -12,5 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from .dtw import dtw_distance
-from .mcd import mcd_distance
 from .eer import compute_eer
+from .eer import compute_minDCF
+from .mcd import mcd_distance
diff --git a/paddleaudio/paddleaudio/metric/eer.py b/paddleaudio/paddleaudio/metric/eer.py
index 65dc7a3c..7738987e 100644
--- a/paddleaudio/paddleaudio/metric/eer.py
+++ b/paddleaudio/paddleaudio/metric/eer.py
@@ -14,6 +14,7 @@
 from typing import List
 
 import numpy as np
+import paddle
 from sklearn.metrics import roc_curve
 
 
@@ -26,3 +27,68 @@ def compute_eer(labels: np.ndarray, scores: np.ndarray) -> List[float]:
     eer_threshold = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
     eer = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
     return eer, eer_threshold
+
+
+def compute_minDCF(positive_scores,
+                   negative_scores,
+                   c_miss=1.0,
+                   c_fa=1.0,
+                   p_target=0.01):
+    """
+    This is modified from SpeechBrain
+    https://github.com/speechbrain/speechbrain/blob/085be635c07f16d42cd1295045bc46c407f1e15b/speechbrain/utils/metric_stats.py#L509
+    Computes the minDCF metric normally used to evaluate speaker verification
+    systems. The min_DCF is the minimum of the following C_det function computed
+    within the defined threshold range:
+
+    C_det =  c_miss * p_miss * p_target + c_fa * p_fa * (1 -p_target)
+
+    where p_miss is the missing probability and p_fa is the probability of having
+    a false alarm.
+
+    Args:
+        positive_scores (Paddle.Tensor): The scores from entries of the same class.
+        negative_scores (Paddle.Tensor): The scores from entries of different classes.
+        c_miss (float, optional): Cost assigned to a missing error (default 1.0).
+        c_fa (float, optional): Cost assigned to a false alarm (default 1.0).
+        p_target (float, optional): Prior probability of having a target (default 0.01).
+
+    Returns:
+        _type_: min dcf 
+    """
+    # Computing candidate thresholds
+    if len(positive_scores.shape) > 1:
+        positive_scores = positive_scores.squeeze()
+
+    if len(negative_scores.shape) > 1:
+        negative_scores = negative_scores.squeeze()
+
+    thresholds = paddle.sort(paddle.concat([positive_scores, negative_scores]))
+    thresholds = paddle.unique(thresholds)
+
+    # Adding intermediate thresholds
+    interm_thresholds = (thresholds[0:-1] + thresholds[1:]) / 2
+    thresholds = paddle.sort(paddle.concat([thresholds, interm_thresholds]))
+
+    # Computing False Rejection Rate (miss detection)
+    positive_scores = paddle.concat(
+        len(thresholds) * [positive_scores.unsqueeze(0)])
+    pos_scores_threshold = positive_scores.transpose(perm=[1, 0]) <= thresholds
+    p_miss = (pos_scores_threshold.sum(0)
+              ).astype("float32") / positive_scores.shape[1]
+    del positive_scores
+    del pos_scores_threshold
+
+    # Computing False Acceptance Rate (false alarm)
+    negative_scores = paddle.concat(
+        len(thresholds) * [negative_scores.unsqueeze(0)])
+    neg_scores_threshold = negative_scores.transpose(perm=[1, 0]) > thresholds
+    p_fa = (neg_scores_threshold.sum(0)
+            ).astype("float32") / negative_scores.shape[1]
+    del negative_scores
+    del neg_scores_threshold
+
+    c_det = c_miss * p_miss * p_target + c_fa * p_fa * (1 - p_target)
+    c_min = paddle.min(c_det, axis=0)
+    min_index = paddle.argmin(c_det, axis=0)
+    return float(c_min), float(thresholds[min_index])
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/extract_speaker_embedding.py b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
similarity index 100%
rename from paddlespeech/vector/exps/ecapa_tdnn/extract_speaker_embedding.py
rename to paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py b/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
index 01a3506a..781bf2a5 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
@@ -45,7 +45,7 @@ def main(args, config):
 
     # stage2: build the speaker verification eval instance with backbone model
     model = SpeakerIdetification(
-        backbone=ecapa_tdnn, num_class=VoxCeleb.num_speakers)
+        backbone=ecapa_tdnn, num_class=config.num_speakers)
 
     # stage3: load the pre-trained model
     args.load_checkpoint = os.path.abspath(
@@ -93,6 +93,7 @@ def main(args, config):
     model.eval()
 
     # stage7: global embedding norm to imporve the performance
+    print("global embedding norm: {}".format(args.global_embedding_norm))
     if args.global_embedding_norm:
         global_embedding_mean = None
         global_embedding_std = None
@@ -118,6 +119,8 @@ def main(args, config):
                         -1).numpy()  # (N, emb_size, 1) -> (N, emb_size)
 
                     # Global embedding normalization.
+                    # if we use the global embedding norm
+                    # eer can reduece about relative 10%
                     if args.global_embedding_norm:
                         batch_count += 1
                         current_mean = embeddings.mean(
@@ -150,8 +153,8 @@ def main(args, config):
         for line in f.readlines():
             label, enrol_id, test_id = line.strip().split(' ')
             labels.append(int(label))
-            enrol_ids.append(enrol_id.split('.')[0].replace('/', '-'))
-            test_ids.append(test_id.split('.')[0].replace('/', '-'))
+            enrol_ids.append(enrol_id.split('.')[0].replace('/', '--'))
+            test_ids.append(test_id.split('.')[0].replace('/', '--'))
 
     cos_sim_func = paddle.nn.CosineSimilarity(axis=1)
     enrol_embeddings, test_embeddings = map(lambda ids: paddle.to_tensor(
@@ -185,11 +188,10 @@ if __name__ == "__main__":
                         default='',
                         help="Directory to load model checkpoint to contiune trainning.")
     parser.add_argument("--global-embedding-norm",
-                        type=bool,
-                        default=True,
+                        default=False,
+                        action="store_true",
                         help="Apply global normalization on speaker embeddings.")
     parser.add_argument("--embedding-mean-norm",
-                        type=bool,
                         default=True,
                         help="Apply mean normalization on speaker embeddings.")
     parser.add_argument("--embedding-std-norm",
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/train.py b/paddlespeech/vector/exps/ecapa_tdnn/train.py
index 6e6e5ab2..cb20ef16 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/train.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/train.py
@@ -178,9 +178,9 @@ def main(args, config):
             timer.count()  # step plus one in timer
 
             # stage 9-10: print the log information only on 0-rank per log-freq batchs
-            if (batch_idx + 1) % config.log_freq == 0 and local_rank == 0:
+            if (batch_idx + 1) % config.log_interval == 0 and local_rank == 0:
                 lr = optimizer.get_lr()
-                avg_loss /= config.log_freq
+                avg_loss /= config.log_interval
                 avg_acc = num_corrects / num_samples
 
                 print_msg = 'Train Epoch={}/{}, Step={}/{}'.format(
@@ -196,7 +196,7 @@ def main(args, config):
                 num_samples = 0
 
         # stage 9-11: save the model parameters only on 0-rank per save-freq batchs
-        if epoch % config.save_freq == 0 and batch_idx + 1 == steps_per_epoch:
+        if epoch % config.save_interval == 0 and batch_idx + 1 == steps_per_epoch:
             if local_rank != 0:
                 paddle.distributed.barrier(
                 )  # Wait for valid step in main process
diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
index 1b9d1fbd..f40ce41b 100644
--- a/paddlespeech/vector/io/augment.py
+++ b/paddlespeech/vector/io/augment.py
@@ -11,7 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# this is modified from https://github.com/speechbrain/speechbrain/blob/085be635c07f16d42cd1295045bc46c407f1e15b/speechbrain/lobes/augment.py
+# this is modified from SpeechBrain
+# https://github.com/speechbrain/speechbrain/blob/085be635c07f16d42cd1295045bc46c407f1e15b/speechbrain/lobes/augment.py
 import math
 import os
 from typing import List
diff --git a/paddlespeech/vector/io/batch.py b/paddlespeech/vector/io/batch.py
index 811775e2..85f2ab8b 100644
--- a/paddlespeech/vector/io/batch.py
+++ b/paddlespeech/vector/io/batch.py
@@ -75,6 +75,9 @@ def batch_feature_normalize(batch, mean_norm: bool=True, std_norm: bool=True):
             i]:].sum() == 0  # Padding valus should all be 0.
 
     # Converts into ratios.
+    # the utterance of the max length doesn't need to padding
+    # the remaining utterances need to padding and all of them will be padded to max length
+    # we convert the original length of each utterance to the ratio of the max length
     lengths = (lengths / lengths.max()).astype(np.float32)
 
     return {'ids': ids, 'feats': feats, 'lengths': lengths}
\ No newline at end of file

From 7eb8fa72a1e50fd3a0338ec2ec5a9ac7b3bb56d2 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sun, 13 Mar 2022 22:59:33 +0800
Subject: [PATCH 028/126] convert save_freq to save_interval, test=doc

---
 examples/voxceleb/sv0/conf/ecapa_tdnn.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
index 720326f8..d7f66380 100644
--- a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
+++ b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
@@ -35,7 +35,6 @@ model:
 ###########################################
 seed: 1986 # according from speechbrain configuration
 epochs: 10
-save_freq: 10
+save_interval: 10
 log_interval: 10
 learning_rate: 1e-8
-

From 506d26a9578f39808de010867217edcc48273fee Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 14 Mar 2022 21:18:29 +0800
Subject: [PATCH 029/126] change the code style to s2t code style, test=doc

---
 examples/voxceleb/sv0/conf/ecapa_tdnn.yaml    | 26 +++++--
 examples/voxceleb/sv0/local/data.sh           | 18 +++++
 examples/voxceleb/sv0/local/data_prepare.py   | 39 +++++-----
 examples/voxceleb/sv0/local/emb.sh            | 13 ++++
 examples/voxceleb/sv0/local/test.sh           |  8 +++
 examples/voxceleb/sv0/local/train.sh          | 22 ++++++
 examples/voxceleb/sv0/run.sh                  | 50 ++++++-------
 .../paddleaudio/datasets/rirs_noises.py       | 15 ++--
 paddleaudio/paddleaudio/datasets/voxceleb.py  | 30 ++++----
 paddleaudio/paddleaudio/utils/download.py     |  8 ++-
 .../vector/exps/ecapa_tdnn/extract_emb.py     | 24 ++++---
 ...speaker_verification_cosine.py => test.py} | 58 +++++++--------
 paddlespeech/vector/exps/ecapa_tdnn/train.py  | 25 +++++--
 paddlespeech/vector/io/augment.py             | 16 +++--
 paddlespeech/vector/utils/download.py         | 72 -------------------
 15 files changed, 216 insertions(+), 208 deletions(-)
 create mode 100755 examples/voxceleb/sv0/local/data.sh
 create mode 100755 examples/voxceleb/sv0/local/emb.sh
 create mode 100644 examples/voxceleb/sv0/local/test.sh
 create mode 100755 examples/voxceleb/sv0/local/train.sh
 rename paddlespeech/vector/exps/ecapa_tdnn/{speaker_verification_cosine.py => test.py} (82%)
 delete mode 100644 paddlespeech/vector/utils/download.py

diff --git a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
index d7f66380..0f4bf189 100644
--- a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
+++ b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
@@ -1,7 +1,10 @@
 ###########################################
 #                Data                 #
 ###########################################
-batch_size: 32
+# we should explicitly specify the wav path of vox2 audio data converted from m4a
+vox2_base_path: 
+augment: True
+batch_size: 16
 num_workers: 2
 num_speakers: 7205 # 1211 vox1, 5994 vox2, 7205 vox1+2, test speakers: 41
 shuffle: True
@@ -11,10 +14,10 @@ random_chunk: True
 #                FEATURE EXTRACTION SETTING               #
 ###########################################################
 # currently, we only support fbank
-feature:
-  n_mels: 80
-  window_size: 400     #25ms, sample rate 16000, 25 * 16000 / 1000 = 400 
-  hop_length: 160     #10ms, sample rate 16000, 10 * 16000 / 1000 = 160
+sample_rate: 16000
+n_mels: 80
+window_size: 400     #25ms, sample rate 16000, 25 * 16000 / 1000 = 400 
+hop_length: 160     #10ms, sample rate 16000, 10 * 16000 / 1000 = 160
 
 ###########################################################
 #                       MODEL SETTING                     #
@@ -35,6 +38,15 @@ model:
 ###########################################
 seed: 1986 # according from speechbrain configuration
 epochs: 10
-save_interval: 10
-log_interval: 10
+save_interval: 1
+log_interval: 1
 learning_rate: 1e-8
+
+
+###########################################
+#                Testing                  #
+###########################################
+global_embedding_norm: True
+embedding_mean_norm: True
+embedding_std_norm: False
+
diff --git a/examples/voxceleb/sv0/local/data.sh b/examples/voxceleb/sv0/local/data.sh
new file mode 100755
index 00000000..ec9c4c58
--- /dev/null
+++ b/examples/voxceleb/sv0/local/data.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+stage=-1
+stop_stage=100
+
+. ${MAIN_ROOT}/utils/parse_options.sh || exit -1;
+
+dir=$1
+conf_path=$2
+mkdir -p ${dir}
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+    # data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
+    # we should use the local/convert.sh convert m4a to wav
+    python3 local/data_prepare.py \
+                        --data-dir ${dir} \
+                        --config ${conf_path}
+fi 
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/local/data_prepare.py b/examples/voxceleb/sv0/local/data_prepare.py
index b906b5da..19ba41b8 100644
--- a/examples/voxceleb/sv0/local/data_prepare.py
+++ b/examples/voxceleb/sv0/local/data_prepare.py
@@ -14,10 +14,10 @@
 import argparse
 import os
 
-import numpy as np
 import paddle
+from yacs.config import CfgNode
 
-from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb
+from paddleaudio.datasets.voxceleb import VoxCeleb
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.augment import build_augment_pipeline
 from paddlespeech.vector.training.seeding import seed_everything
@@ -25,46 +25,47 @@ from paddlespeech.vector.training.seeding import seed_everything
 logger = Log(__name__).getlog()
 
 
-def main(args):
+def main(args, config):
 
     # stage0: set the cpu device, all data prepare process will be done in cpu mode
     paddle.set_device("cpu")
     # set the random seed, it is a must for multiprocess training
-    seed_everything(args.seed)
+    seed_everything(config.seed)
 
     # stage 1: generate the voxceleb csv file
     # Note: this may occurs c++ execption, but the program will execute fine
     # so we ignore the execption 
     # we explicitly pass the vox2 base path to data prepare and generate the audio info
+    logger.info("start to generate the voxceleb dataset info")
     train_dataset = VoxCeleb(
-        'train', target_dir=args.data_dir, vox2_base_path=args.vox2_base_path)
-    dev_dataset = VoxCeleb(
-        'dev', target_dir=args.data_dir, vox2_base_path=args.vox2_base_path)
+        'train', target_dir=args.data_dir, vox2_base_path=config.vox2_base_path)
 
     # stage 2: generate the augment noise csv file
-    if args.augment:
+    if config.augment:
+        logger.info("start to generate the augment dataset info")
         augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
 
 
 if __name__ == "__main__":
     # yapf: disable
     parser = argparse.ArgumentParser(__doc__)
-    parser.add_argument("--seed",
-                        default=0,
-                        type=int,
-                        help="random seed for paddle, numpy and python random package")
     parser.add_argument("--data-dir",
                         default="./data/",
                         type=str,
                         help="data directory")
-    parser.add_argument("--vox2-base-path",
+    parser.add_argument("--config",
                         default=None,
                         type=str,
-                        help="vox2 base path, where is store the wav audio")
-    parser.add_argument("--augment",
-                        action="store_true",
-                        default=False,
-                        help="Apply audio augments.")
+                        help="configuration file")
     args = parser.parse_args()
     # yapf: enable
-    main(args)
+
+    # https://yaml.org/type/float.html
+    config = CfgNode(new_allowed=True)
+    if args.config:
+        config.merge_from_file(args.config)
+
+    config.freeze()
+    print(config)
+
+    main(args, config)
diff --git a/examples/voxceleb/sv0/local/emb.sh b/examples/voxceleb/sv0/local/emb.sh
new file mode 100755
index 00000000..482e658e
--- /dev/null
+++ b/examples/voxceleb/sv0/local/emb.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+. ./path.sh
+
+exp_dir=exp/ecapa-tdnn-vox12-big//epoch_10/            # experiment directory
+conf_path=conf/ecapa_tdnn.yaml
+audio_path="demo/voxceleb/00001.wav"
+
+source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
+
+# extract the audio embedding
+python3 ${BIN_DIR}/extract_emb.py --device "gpu" \
+          --config ${conf_path} \
+          --audio-path ${audio_path} --load-checkpoint ${exp_dir}
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/local/test.sh b/examples/voxceleb/sv0/local/test.sh
new file mode 100644
index 00000000..d8a1a0ba
--- /dev/null
+++ b/examples/voxceleb/sv0/local/test.sh
@@ -0,0 +1,8 @@
+dir=$1
+exp_dir=$2
+conf_path=$3
+
+python3 ${BIN_DIR}/test.py \
+        --config ${conf_path} \
+        --data-dir ${dir} \
+        --load-checkpoint ${exp_dir}
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/local/train.sh b/examples/voxceleb/sv0/local/train.sh
new file mode 100755
index 00000000..385e8caa
--- /dev/null
+++ b/examples/voxceleb/sv0/local/train.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+dir=$1
+exp_dir=$2
+conf_path=$3
+
+ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+echo "using $ngpu gpus..."
+
+# train the speaker identification task with voxceleb data
+# Note: we will store the log file in exp/log directory
+python3 -m paddle.distributed.launch --gpus=$CUDA_VISIBLE_DEVICES \
+    ${BIN_DIR}/train.py --device "gpu" --checkpoint-dir ${exp_dir} --augment \
+    --data-dir ${dir} --config ${conf_path}
+
+
+if [ $? -ne 0 ]; then
+    echo "Failed in training!"
+    exit 1
+fi
+
+exit 0
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index c5dc3dd2..e38027e9 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -18,7 +18,7 @@ set -e
 
 #######################################################################
 # stage 0: data prepare, including voxceleb1 download and generate {train,dev,enroll,test}.csv
-#          voxceleb2 data is m4a format, so we need user to convert the m4a to wav yourselves as described in Readme.md
+#          voxceleb2 data is m4a format, so we need user to convert the m4a to wav yourselves as described in Readme.md with the script local/convert.sh
 # stage 1: train the speaker identification model
 # stage 2: test speaker identification 
 # stage 3: extract the training embeding to train the LDA and PLDA
@@ -30,49 +30,39 @@ set -e
 # and put all of them to ${PPAUDIO_HOME}/datasets/vox2
 # we will find the wav from ${PPAUDIO_HOME}/datasets/vox1/wav and ${PPAUDIO_HOME}/datasets/vox2/wav
 # export PPAUDIO_HOME=
-
 stage=0
+stop_stage=50
+
 # data directory
 # if we set the variable ${dir}, we will store the wav info to this directory
 # otherwise, we will store the wav info to vox1 and vox2 directory respectively
-dir=data/                          
-exp_dir=exp/ecapa-tdnn/            # experiment directory
-
 # vox2 wav path, we must convert the m4a format to wav format 
-# and store them in the ${PPAUDIO_HOME}/datasets/vox2/wav/ directory
-vox2_base_path=${PPAUDIO_HOME}/datasets/vox2/wav/
-mkdir -p ${dir}
+# dir=data-demo/                          # data info directory    
+dir=demo/                          # data info directory   
+
+exp_dir=exp/ecapa-tdnn-vox12-big//            # experiment directory
+conf_path=conf/ecapa_tdnn.yaml          
+gpus=0,1,2,3
+
+source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
+
 mkdir -p ${exp_dir}
 
-if [ $stage -le 0 ]; then 
+if [ $stage -le 0 ] && [ ${stop_stage} -ge 0 ]; then 
      # stage 0: data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
-     python3 local/data_prepare.py \
-     --data-dir ${dir} --augment --vox2-base-path ${vox2_base_path} \
-     --config conf/ecapa_tdnn.yaml
+     # and we should specifiy the vox2 data in the data.sh
+     bash ./local/data.sh ${dir} ${conf_path}|| exit -1;
 fi 
 
-if [ $stage -le 1 ]; then
+if [ $stage -le 1 ] && [ ${stop_stage} -ge 1 ]; then
      # stage 1: train the speaker identification model
-     python3 \
-          -m paddle.distributed.launch --gpus=0,1,2,3 \
-          ${BIN_DIR}/train.py --device "gpu" --checkpoint-dir ${exp_dir} --augment \
-          --data-dir ${dir} --config conf/ecapa_tdnn.yaml
+     CUDA_VISIBLE_DEVICES=${gpus} bash ./local/train.sh ${dir} ${exp_dir} ${conf_path} 
 fi
 
 if [ $stage -le 2 ]; then
-     # stage 1: get the speaker verification scores with cosine function
-     python3 \
-          ${BIN_DIR}/speaker_verification_cosine.py\
-          --config conf/ecapa_tdnn.yaml \
-          --data-dir ${dir} --load-checkpoint ${exp_dir}/epoch_10/
-fi
-
-if [ $stage -le 3 ]; then
-     # stage 3: extract the audio embedding
-     python3 \
-          ${BIN_DIR}/extract_speaker_embedding.py\
-          --config conf/ecapa_tdnn.yaml \
-          --audio-path "demo/csv/00001.wav" --load-checkpoint ${exp_dir}/epoch_60/
+     # stage 2: get the speaker verification scores with cosine function
+     #          now we only support use cosine to get the scores
+     CUDA_VISIBLE_DEVICES=0 bash ./local/test.sh ${dir} ${exp_dir} ${conf_path}
 fi
 
 # if [ $stage -le 3 ]; then
diff --git a/paddleaudio/paddleaudio/datasets/rirs_noises.py b/paddleaudio/paddleaudio/datasets/rirs_noises.py
index df5dec61..80bb2d74 100644
--- a/paddleaudio/paddleaudio/datasets/rirs_noises.py
+++ b/paddleaudio/paddleaudio/datasets/rirs_noises.py
@@ -25,13 +25,10 @@ from tqdm import tqdm
 
 from ..backends import load as load_audio
 from ..backends import save as save_wav
-from .dataset import feat_funcs
 from ..utils import DATA_HOME
 from ..utils import decompress
-from paddlespeech.s2t.utils.log import Log
-from paddlespeech.vector.utils.download import download_and_decompress
-
-logger = Log(__name__).getlog()
+from ..utils.download import download_and_decompress
+from .dataset import feat_funcs
 
 __all__ = ['OpenRIRNoise']
 
@@ -80,17 +77,17 @@ class OpenRIRNoise(Dataset):
 
     def _get_data(self):
         # Download audio files.
-        logger.info(f"rirs noises base path: {self.base_path}")
+        print(f"rirs noises base path: {self.base_path}")
         if not os.path.isdir(self.base_path):
             download_and_decompress(
                 self.archieves, self.base_path, decompress=True)
         else:
-            logger.info(
+            print(
                 f"{self.base_path} already exists, we will not download and decompress again"
             )
 
         # Data preparation.
-        logger.info(f"prepare the csv to {self.csv_path}")
+        print(f"prepare the csv to {self.csv_path}")
         if not os.path.isdir(self.csv_path):
             os.makedirs(self.csv_path)
             self.prepare_data()
@@ -161,7 +158,7 @@ class OpenRIRNoise(Dataset):
                      wav_files: List[str],
                      output_file: str,
                      split_chunks: bool=True):
-        logger.info(f'Generating csv: {output_file}')
+        print(f'Generating csv: {output_file}')
         header = ["id", "duration", "wav"]
 
         infos = list(
diff --git a/paddleaudio/paddleaudio/datasets/voxceleb.py b/paddleaudio/paddleaudio/datasets/voxceleb.py
index f8d634f2..b9b8c271 100644
--- a/paddleaudio/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/paddleaudio/datasets/voxceleb.py
@@ -28,13 +28,8 @@ from tqdm import tqdm
 from ..backends import load as load_audio
 from ..utils import DATA_HOME
 from ..utils import decompress
+from ..utils.download import download_and_decompress
 from .dataset import feat_funcs
-from paddlespeech.s2t.utils.log import Log
-from paddlespeech.vector.utils.download import download_and_decompress
-from utils.utility import download
-from utils.utility import unpack
-
-logger = Log(__name__).getlog()
 
 __all__ = ['VoxCeleb']
 
@@ -138,9 +133,9 @@ class VoxCeleb(Dataset):
         # Download audio files.
         # We need the users to decompress all vox1/dev/wav and vox1/test/wav/ to vox1/wav/ dir
         # so, we check the vox1/wav dir status
-        logger.info(f"wav base path: {self.wav_path}")
+        print(f"wav base path: {self.wav_path}")
         if not os.path.isdir(self.wav_path):
-            logger.info(f"start to download the voxceleb1 dataset")
+            print(f"start to download the voxceleb1 dataset")
             download_and_decompress(  # multi-zip parts concatenate to vox1_dev_wav.zip
                 self.archieves_audio_dev,
                 self.base_path,
@@ -152,7 +147,7 @@ class VoxCeleb(Dataset):
 
             # Download all parts and concatenate the files into one zip file.
             dev_zipfile = os.path.join(self.base_path, 'vox1_dev_wav.zip')
-            logger.info(f'Concatenating all parts to: {dev_zipfile}')
+            print(f'Concatenating all parts to: {dev_zipfile}')
             os.system(
                 f'cat {os.path.join(self.base_path, "vox1_dev_wav_parta*")} > {dev_zipfile}'
             )
@@ -162,6 +157,7 @@ class VoxCeleb(Dataset):
 
         # Download meta files.
         if not os.path.isdir(self.meta_path):
+            print("prepare the meta data")
             download_and_decompress(
                 self.archieves_meta, self.meta_path, decompress=False)
 
@@ -171,7 +167,7 @@ class VoxCeleb(Dataset):
             self.prepare_data()
 
         data = []
-        logger.info(
+        print(
             f"read the {self.subset} from {os.path.join(self.csv_path, f'{self.subset}.csv')}"
         )
         with open(os.path.join(self.csv_path, f'{self.subset}.csv'), 'r') as rf:
@@ -266,8 +262,8 @@ class VoxCeleb(Dataset):
                      wav_files: List[str],
                      output_file: str,
                      split_chunks: bool=True):
-        logger.info(f'Generating csv: {output_file}')
-        header = ["id", "duration", "wav", "start", "stop", "spk_id"]
+        print(f'Generating csv: {output_file}')
+        header = ["ID", "duration", "wav", "start", "stop", "spk_id"]
         # Note: this may occurs c++ execption, but the program will execute fine
         # so we can ignore the execption 
         with Pool(cpu_count()) as p:
@@ -290,7 +286,7 @@ class VoxCeleb(Dataset):
 
     def prepare_data(self):
         # Audio of speakers in veri_test_file should not be included in training set.
-        logger.info("start to prepare the data csv file")
+        print("start to prepare the data csv file")
         enroll_files = set()
         test_files = set()
         # get the enroll and test audio file path
@@ -311,13 +307,13 @@ class VoxCeleb(Dataset):
         # get all the train and dev audios file path
         audio_files = []
         speakers = set()
+        print("Getting file list...")
         for path in [self.wav_path, self.vox2_base_path]:
             # if vox2 directory is not set and vox2 is not a directory 
             # we will not process this directory
             if not path or not os.path.exists(path):
-                logger.warning(
-                    f"{path} is an invalid path, please check again, "
-                    "and we will ignore the vox2 base path")
+                print(f"{path} is an invalid path, please check again, "
+                      "and we will ignore the vox2 base path")
                 continue
             for file in glob.glob(
                     os.path.join(path, "**", "*.wav"), recursive=True):
@@ -327,7 +323,7 @@ class VoxCeleb(Dataset):
                 speakers.add(spk)
                 audio_files.append(file)
 
-        logger.info(
+        print(
             f"start to generate the {os.path.join(self.meta_path, 'spk_id2label.txt')}"
         )
         # encode the train and dev speakers label to spk_id2label.txt
diff --git a/paddleaudio/paddleaudio/utils/download.py b/paddleaudio/paddleaudio/utils/download.py
index 4658352f..07d5eea8 100644
--- a/paddleaudio/paddleaudio/utils/download.py
+++ b/paddleaudio/paddleaudio/utils/download.py
@@ -37,7 +37,9 @@ def decompress(file: str):
     download._decompress(file)
 
 
-def download_and_decompress(archives: List[Dict[str, str]], path: str):
+def download_and_decompress(archives: List[Dict[str, str]],
+                            path: str,
+                            decompress: bool=True):
     """
     Download archieves and decompress to specific path.
     """
@@ -47,8 +49,8 @@ def download_and_decompress(archives: List[Dict[str, str]], path: str):
     for archive in archives:
         assert 'url' in archive and 'md5' in archive, \
             'Dictionary keys of "url" and "md5" are required in the archive, but got: {list(archieve.keys())}'
-
-        download.get_path_from_url(archive['url'], path, archive['md5'])
+        download.get_path_from_url(
+            archive['url'], path, archive['md5'], decompress=decompress)
 
 
 def load_state_dict_from_url(url: str, path: str, md5: str=None):
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
index 44cbd204..0d09d211 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
@@ -14,12 +14,13 @@
 import argparse
 import os
 
+import time
 import numpy as np
 import paddle
 from yacs.config import CfgNode
 
-from paddleaudio.paddleaudio.backends import load as load_audio
-from paddleaudio.paddleaudio.compliance.librosa import melspectrogram
+from paddleaudio.backends import load as load_audio
+from paddleaudio.compliance.librosa import melspectrogram
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.batch import feature_normalize
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
@@ -39,7 +40,7 @@ def extract_audio_embedding(args, config):
     ecapa_tdnn = EcapaTdnn(**config.model)
 
     # stage4: build the speaker verification train instance with backbone model
-    model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=1211)
+    model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=config.num_speakers)
     # stage 2: load the pre-trained model
     args.load_checkpoint = os.path.abspath(
         os.path.expanduser(args.load_checkpoint))
@@ -60,7 +61,12 @@ def extract_audio_embedding(args, config):
     # feat type is numpy array, whose shape is [dim, time]
     # we need convert the audio feat to one-batch shape [batch, dim, time], where the batch is one
     # so the final shape is [1, dim, time]
-    feat = melspectrogram(x=waveform, **config.feature)
+    start_time = time.time()
+    feat = melspectrogram(x=waveform, 
+                          sr=config.sample_rate,
+                          n_mels=config.n_mels,
+                          window_size=config.window_size,
+                          hop_length=config.hop_length)
     feat = paddle.to_tensor(feat).unsqueeze(0)
 
     # in inference period, the lengths is all one without padding
@@ -71,9 +77,13 @@ def extract_audio_embedding(args, config):
     # model backbone network forward the feats and get the embedding
     embedding = model.backbone(
         feat, lengths).squeeze().numpy()  # (1, emb_size, 1) -> (emb_size)
+    elapsed_time = time.time() - start_time
+    audio_length = waveform.shape[0] / sr
+
 
     # stage 5: do global norm with external mean and std
-    # todo
+    rtf = elapsed_time / audio_length
+    logger.info(f"{args.device} rft={rtf}")
     return embedding
 
 
@@ -92,10 +102,6 @@ if __name__ == "__main__":
                         type=str,
                         default='',
                         help="Directory to load model checkpoint to contiune trainning.")
-    parser.add_argument("--global-embedding-norm",
-                        type=str,
-                        default=None,
-                        help="Apply global normalization on speaker embeddings.")
     parser.add_argument("--audio-path",
                         default="./data/demo.wav",
                         type=str,
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py b/paddlespeech/vector/exps/ecapa_tdnn/test.py
similarity index 82%
rename from paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
rename to paddlespeech/vector/exps/ecapa_tdnn/test.py
index 781bf2a5..03757033 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/speaker_verification_cosine.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/test.py
@@ -23,8 +23,8 @@ from paddle.io import DataLoader
 from tqdm import tqdm
 from yacs.config import CfgNode
 
-from paddleaudio.paddleaudio.datasets import VoxCeleb
-from paddleaudio.paddleaudio.metric import compute_eer
+from paddleaudio.datasets import VoxCeleb
+from paddleaudio.metric import compute_eer
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.batch import batch_feature_normalize
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
@@ -48,6 +48,9 @@ def main(args, config):
         backbone=ecapa_tdnn, num_class=config.num_speakers)
 
     # stage3: load the pre-trained model
+    #         we get the last model from the epoch and save_interval
+    last_save_epoch = (config.epochs // config.save_interval) * config.save_interval
+    args.load_checkpoint = os.path.join(args.load_checkpoint, "epoch_" + str(last_save_epoch))
     args.load_checkpoint = os.path.abspath(
         os.path.expanduser(args.load_checkpoint))
 
@@ -63,7 +66,9 @@ def main(args, config):
         target_dir=args.data_dir,
         feat_type='melspectrogram',
         random_chunk=False,
-        **config.feature)
+        n_mels=config.n_mels,
+        window_size=config.window_size,
+        hop_length=config.hop_length)
     enroll_sampler = BatchSampler(
         enroll_dataset, batch_size=config.batch_size,
         shuffle=True)  # Shuffle to make embedding normalization more robust.
@@ -73,13 +78,14 @@ def main(args, config):
                             x, mean_norm=True, std_norm=False),
                     num_workers=config.num_workers,
                     return_list=True,)
-
     test_dataset = VoxCeleb(
         subset='test',
         target_dir=args.data_dir,
         feat_type='melspectrogram',
         random_chunk=False,
-        **config.feature)
+        n_mels=config.n_mels,
+        window_size=config.window_size,
+        hop_length=config.hop_length)
 
     test_sampler = BatchSampler(
         test_dataset, batch_size=config.batch_size, shuffle=True)
@@ -89,19 +95,19 @@ def main(args, config):
                                 x, mean_norm=True, std_norm=False),
                             num_workers=config.num_workers,
                             return_list=True,)
-    # stage6: we must set the model to eval mode
+    # stage5: we must set the model to eval mode
     model.eval()
 
-    # stage7: global embedding norm to imporve the performance
-    print("global embedding norm: {}".format(args.global_embedding_norm))
-    if args.global_embedding_norm:
+    # stage6: global embedding norm to imporve the performance
+    logger.info(f"global embedding norm: {config.global_embedding_norm}")
+    if config.global_embedding_norm:
         global_embedding_mean = None
         global_embedding_std = None
-        mean_norm_flag = args.embedding_mean_norm
-        std_norm_flag = args.embedding_std_norm
+        mean_norm_flag = config.embedding_mean_norm
+        std_norm_flag = config.embedding_std_norm
         batch_count = 0
 
-    # stage8: Compute embeddings of audios in enrol and test dataset from model.
+    # stage7: Compute embeddings of audios in enrol and test dataset from model.
     id2embedding = {}
     # Run multi times to make embedding normalization more stable.
     for i in range(2):
@@ -121,7 +127,7 @@ def main(args, config):
                     # Global embedding normalization.
                     # if we use the global embedding norm
                     # eer can reduece about relative 10%
-                    if args.global_embedding_norm:
+                    if config.global_embedding_norm:
                         batch_count += 1
                         current_mean = embeddings.mean(
                             axis=0) if mean_norm_flag else 0
@@ -145,21 +151,22 @@ def main(args, config):
                     # Update embedding dict.
                     id2embedding.update(dict(zip(ids, embeddings)))
 
-    # stage 9: Compute cosine scores.
+    # stage 8: Compute cosine scores.
     labels = []
-    enrol_ids = []
+    enroll_ids = []
     test_ids = []
+    logger.info(f"read the trial from {VoxCeleb.veri_test_file}")
     with open(VoxCeleb.veri_test_file, 'r') as f:
         for line in f.readlines():
-            label, enrol_id, test_id = line.strip().split(' ')
+            label, enroll_id, test_id = line.strip().split(' ')
             labels.append(int(label))
-            enrol_ids.append(enrol_id.split('.')[0].replace('/', '--'))
-            test_ids.append(test_id.split('.')[0].replace('/', '--'))
+            enroll_ids.append(enroll_id.split('.')[0].replace('/', '-'))
+            test_ids.append(test_id.split('.')[0].replace('/', '-'))
 
     cos_sim_func = paddle.nn.CosineSimilarity(axis=1)
     enrol_embeddings, test_embeddings = map(lambda ids: paddle.to_tensor(
-        np.asarray([id2embedding[id] for id in ids], dtype='float32')),
-                                            [enrol_ids, test_ids
+        np.asarray([id2embedding[uttid] for uttid in ids], dtype='float32')),
+                                            [enroll_ids, test_ids
                                              ])  # (N, emb_size)
     scores = cos_sim_func(enrol_embeddings, test_embeddings)
     EER, threshold = compute_eer(np.asarray(labels), scores.numpy())
@@ -187,17 +194,6 @@ if __name__ == "__main__":
                         type=str,
                         default='',
                         help="Directory to load model checkpoint to contiune trainning.")
-    parser.add_argument("--global-embedding-norm",
-                        default=False,
-                        action="store_true",
-                        help="Apply global normalization on speaker embeddings.")
-    parser.add_argument("--embedding-mean-norm",
-                        default=True,
-                        help="Apply mean normalization on speaker embeddings.")
-    parser.add_argument("--embedding-std-norm",
-                        type=bool,
-                        default=False,
-                        help="Apply std normalization on speaker embeddings.")
     args = parser.parse_args()
     # yapf: enable
     # https://yaml.org/type/float.html
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/train.py b/paddlespeech/vector/exps/ecapa_tdnn/train.py
index cb20ef16..0d62c69d 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/train.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/train.py
@@ -21,8 +21,8 @@ from paddle.io import DataLoader
 from paddle.io import DistributedBatchSampler
 from yacs.config import CfgNode
 
-from paddleaudio.paddleaudio.compliance.librosa import melspectrogram
-from paddleaudio.paddleaudio.datasets.voxceleb import VoxCeleb
+from paddleaudio.compliance.librosa import melspectrogram
+from paddleaudio.datasets.voxceleb import VoxCeleb
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.augment import build_augment_pipeline
 from paddlespeech.vector.io.augment import waveform_augment
@@ -68,6 +68,8 @@ def main(args, config):
         backbone=ecapa_tdnn, num_class=VoxCeleb.num_speakers)
 
     # stage5: build the optimizer, we now only construct the AdamW optimizer
+    #         140000 is single gpu steps
+    #         so, in multi-gpu mode, wo reduce the step_size to 140000//nranks to enable CyclicLRScheduler
     lr_schedule = CyclicLRScheduler(
         base_lr=config.learning_rate, max_lr=1e-3, step_size=140000 // nranks)
     optimizer = paddle.optimizer.AdamW(
@@ -138,6 +140,10 @@ def main(args, config):
             waveforms, labels = batch['waveforms'], batch['labels']
 
             # stage 9-2: audio sample augment method, which is done on the audio sample point
+            #            the original wavefrom and the augmented waveform is concatented in a batch
+            #            eg. five augment method in the augment pipeline
+            #                the final data nums is batch_size * [five + one] 
+            #                -> five augmented waveform batch plus one original batch waveform
             if len(augment_pipeline) != 0:
                 waveforms = waveform_augment(waveforms, augment_pipeline)
                 labels = paddle.concat(
@@ -146,7 +152,11 @@ def main(args, config):
             # stage 9-3: extract the audio feats,such fbank, mfcc, spectrogram
             feats = []
             for waveform in waveforms.numpy():
-                feat = melspectrogram(x=waveform, **config.feature)
+                feat = melspectrogram(x=waveform, 
+                                      sr=config.sample_rate, 
+                                      n_mels=config.n_mels, 
+                                      window_size=config.window_size, 
+                                      hop_length=config.hop_length)
                 feats.append(feat)
             feats = paddle.to_tensor(np.asarray(feats))
 
@@ -205,7 +215,7 @@ def main(args, config):
             # stage 9-12: construct the valid dataset dataloader
             dev_sampler = BatchSampler(
                 dev_dataset,
-                batch_size=config.batch_size // 4,
+                batch_size=config.batch_size,
                 shuffle=False,
                 drop_last=False)
             dev_loader = DataLoader(
@@ -228,8 +238,11 @@ def main(args, config):
 
                     feats = []
                     for waveform in waveforms.numpy():
-                        # feat = melspectrogram(x=waveform, **cpu_feat_conf)
-                        feat = melspectrogram(x=waveform, **config.feature)
+                        feat = melspectrogram(x=waveform, 
+                                              sr=config.sample_rate,
+                                              n_mels=config.n_mels,
+                                              window_size=config.window_size,
+                                              hop_length=config.hop_length)
                         feats.append(feat)
 
                     feats = paddle.to_tensor(np.asarray(feats))
diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
index f40ce41b..6e508c37 100644
--- a/paddlespeech/vector/io/augment.py
+++ b/paddlespeech/vector/io/augment.py
@@ -22,8 +22,8 @@ import paddle
 import paddle.nn as nn
 import paddle.nn.functional as F
 
-from paddleaudio.paddleaudio import load as load_audio
-from paddleaudio.paddleaudio.datasets.rirs_noises import OpenRIRNoise
+from paddleaudio import load as load_audio
+from paddleaudio.datasets.rirs_noises import OpenRIRNoise
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.signal_processing import compute_amplitude
 from paddlespeech.vector.io.signal_processing import convolve1d
@@ -879,14 +879,18 @@ def waveform_augment(waveforms: paddle.Tensor,
     """process the augment pipeline and return all the waveforms
 
     Args:
-        waveforms (paddle.Tensor): _description_
-        augment_pipeline (List[paddle.nn.Layer]): _description_
+        waveforms (paddle.Tensor): original batch waveform
+        augment_pipeline (List[paddle.nn.Layer]): agument pipeline process
 
     Returns:
-        paddle.Tensor: _description_
+        paddle.Tensor: all the audio waveform including the original waveform and augmented waveform
     """
+    # stage 0: store the original waveforms
     waveforms_aug_list = [waveforms]
+
+    # augment the original batch waveform
     for aug in augment_pipeline:
+        # stage 1: augment the data
         waveforms_aug = aug(waveforms)  # (N, L)
         if waveforms_aug.shape[1] >= waveforms.shape[1]:
             # Trunc
@@ -897,6 +901,8 @@ def waveform_augment(waveforms: paddle.Tensor,
             waveforms_aug = F.pad(
                 waveforms_aug.unsqueeze(-1), [0, lengths_to_pad],
                 data_format='NLC').squeeze(-1)
+        # stage 2: append the augmented waveform into the list
         waveforms_aug_list.append(waveforms_aug)
 
+    # get the all the waveforms
     return paddle.concat(waveforms_aug_list, axis=0)
diff --git a/paddlespeech/vector/utils/download.py b/paddlespeech/vector/utils/download.py
deleted file mode 100644
index 476bfea7..00000000
--- a/paddlespeech/vector/utils/download.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright (c) 2021  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-from typing import Dict
-from typing import List
-
-from paddle.framework import load as load_state_dict
-from paddle.utils import download
-
-__all__ = [
-    'decompress',
-    'download_and_decompress',
-    'load_state_dict_from_url',
-]
-
-
-def decompress(file: str, path: str=os.PathLike):
-    """
-    Extracts all files from a compressed file to specific path.
-    """
-    assert os.path.isfile(file), "File: {} not exists.".format(file)
-
-    if path is None:
-        print("decompress the data: {}".format(file))
-        download._decompress(file)
-    else:
-        print("decompress the data: {} to {}".format(file, path))
-        if not os.path.isdir(path):
-            os.makedirs(path)
-
-        tmp_file = os.path.join(path, os.path.basename(file))
-        os.rename(file, tmp_file)
-        download._decompress(tmp_file)
-        os.rename(tmp_file, file)
-
-
-def download_and_decompress(archives: List[Dict[str, str]],
-                            path: str,
-                            decompress: bool=True):
-    """
-    Download archieves and decompress to specific path.
-    """
-    if not os.path.isdir(path):
-        os.makedirs(path)
-
-    for archive in archives:
-        assert 'url' in archive and 'md5' in archive, \
-            'Dictionary keys of "url" and "md5" are required in the archive, but got: {list(archieve.keys())}'
-        download.get_path_from_url(
-            archive['url'], path, archive['md5'], decompress=decompress)
-
-
-def load_state_dict_from_url(url: str, path: str, md5: str=None):
-    """
-    Download and load a state dict from url
-    """
-    if not os.path.isdir(path):
-        os.makedirs(path)
-
-    download.get_path_from_url(url, path, md5)
-    return load_state_dict(os.path.join(path, os.path.basename(url)))

From ab16d8ce3c4487574b7ef157b07be092fe41c5d2 Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Thu, 17 Mar 2022 03:22:33 +0000
Subject: [PATCH 030/126] change default initializer to kaiming_uniform,
 test=asr

---
 examples/aishell/asr1/conf/conformer.yaml     |   1 +
 paddlespeech/s2t/__init__.py                  |   6 +
 paddlespeech/s2t/models/u2/u2.py              |   9 +-
 paddlespeech/s2t/modules/attention.py         |   2 +-
 .../s2t/modules/conformer_convolution.py      |  18 +-
 paddlespeech/s2t/modules/decoder.py           |  15 +-
 paddlespeech/s2t/modules/decoder_layer.py     |  24 +-
 paddlespeech/s2t/modules/encoder.py           |   9 +-
 paddlespeech/s2t/modules/encoder_layer.py     |  42 ++-
 paddlespeech/s2t/modules/initializer.py       | 272 ++++++++++++++++++
 paddlespeech/s2t/modules/nets_utils.py        |  44 +++
 11 files changed, 423 insertions(+), 19 deletions(-)
 create mode 100644 paddlespeech/s2t/modules/initializer.py
 create mode 100644 paddlespeech/s2t/modules/nets_utils.py

diff --git a/examples/aishell/asr1/conf/conformer.yaml b/examples/aishell/asr1/conf/conformer.yaml
index 775a4527..679a5bf6 100644
--- a/examples/aishell/asr1/conf/conformer.yaml
+++ b/examples/aishell/asr1/conf/conformer.yaml
@@ -37,6 +37,7 @@ model_conf:
     ctc_weight: 0.3
     lsm_weight: 0.1     # label smoothing option
     length_normalized_loss: false
+    init_type: 'kaiming_uniform' 
 
 ###########################################
 #                   Data                  #
diff --git a/paddlespeech/s2t/__init__.py b/paddlespeech/s2t/__init__.py
index 855ceef9..26f59dfb 100644
--- a/paddlespeech/s2t/__init__.py
+++ b/paddlespeech/s2t/__init__.py
@@ -21,6 +21,7 @@ from paddle import nn
 from paddle.fluid import core
 from paddle.nn import functional as F
 
+from paddlespeech.s2t.modules import initializer
 from paddlespeech.s2t.utils.log import Log
 
 #TODO(Hui Zhang): remove  fluid import
@@ -505,3 +506,8 @@ if not hasattr(paddle.nn, 'LayerDict'):
     logger.debug(
         "register user LayerDict to paddle.nn, remove this when fixed!")
     setattr(paddle.nn, 'LayerDict', LayerDict)
+
+"""
+    hack KaiminigUniform: change limit from np.sqrt(6.0 / float(fan_in)) to np.sqrt(1.0 / float(fan_in))
+"""
+paddle.nn.initializer.KaimingUniform = initializer.KaimingUniform
diff --git a/paddlespeech/s2t/models/u2/u2.py b/paddlespeech/s2t/models/u2/u2.py
index 91079812..67ec5924 100644
--- a/paddlespeech/s2t/models/u2/u2.py
+++ b/paddlespeech/s2t/models/u2/u2.py
@@ -41,6 +41,7 @@ from paddlespeech.s2t.modules.mask import make_pad_mask
 from paddlespeech.s2t.modules.mask import mask_finished_preds
 from paddlespeech.s2t.modules.mask import mask_finished_scores
 from paddlespeech.s2t.modules.mask import subsequent_mask
+from paddlespeech.s2t.modules.nets_utils import initialize
 from paddlespeech.s2t.utils import checkpoint
 from paddlespeech.s2t.utils import layer_tools
 from paddlespeech.s2t.utils.ctc_utils import remove_duplicates_and_blank
@@ -72,6 +73,7 @@ class U2BaseModel(ASRInterface, nn.Layer):
         assert 0.0 <= ctc_weight <= 1.0, ctc_weight
 
         nn.Layer.__init__(self)
+
         # note that eos is the same as sos (equivalent ID)
         self.sos = vocab_size - 1
         self.eos = vocab_size - 1
@@ -780,9 +782,14 @@ class U2DecodeModel(U2BaseModel):
 
 class U2Model(U2DecodeModel):
     def __init__(self, configs: dict):
+        model_conf = configs.get('model_conf', dict())
+        init_type = model_conf.get("init_type", None)
+        if init_type is not None:
+            logger.info(f"Use {init_type} initializer as default initializer")
+        initialize(self, init_type)
         vocab_size, encoder, decoder, ctc = U2Model._init_from_config(configs)
+        nn.initializer.set_global_initializer(None)
 
-        model_conf = configs.get('model_conf', dict())
         super().__init__(
             vocab_size=vocab_size,
             encoder=encoder,
diff --git a/paddlespeech/s2t/modules/attention.py b/paddlespeech/s2t/modules/attention.py
index 3d5f8cd1..c2f5e503 100644
--- a/paddlespeech/s2t/modules/attention.py
+++ b/paddlespeech/s2t/modules/attention.py
@@ -95,7 +95,7 @@ class MultiHeadedAttention(nn.Layer):
             mask (paddle.Tensor): Mask, size (#batch, 1, time2) or
                 (#batch, time1, time2).
         Returns:
-            paddle.Tensor: Transformed value weighted 
+            paddle.Tensor: Transformed value weighted
                 by the attention score, (#batch, time1, d_model).
         """
         n_batch = value.shape[0]
diff --git a/paddlespeech/s2t/modules/conformer_convolution.py b/paddlespeech/s2t/modules/conformer_convolution.py
index 7ec92554..256d187c 100644
--- a/paddlespeech/s2t/modules/conformer_convolution.py
+++ b/paddlespeech/s2t/modules/conformer_convolution.py
@@ -60,8 +60,8 @@ class ConvolutionModule(nn.Layer):
         )
 
         # self.lorder is used to distinguish if it's a causal convolution,
-        # if self.lorder > 0: 
-        #    it's a causal convolution, the input will be padded with 
+        # if self.lorder > 0:
+        #    it's a causal convolution, the input will be padded with
         #    `self.lorder` frames on the left in forward (causal conv impl).
         # else: it's a symmetrical convolution
         if causal:
@@ -87,10 +87,20 @@ class ConvolutionModule(nn.Layer):
         assert norm in ['batch_norm', 'layer_norm']
         if norm == "batch_norm":
             self.use_layer_norm = False
-            self.norm = nn.BatchNorm1D(channels)
+            self.norm = nn.BatchNorm1D(
+                channels,
+                weight_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(1.0)),
+                bias_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(0.0)))
         else:
             self.use_layer_norm = True
-            self.norm = nn.LayerNorm(channels)
+            self.norm = nn.LayerNorm(
+                channels,
+                weight_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(1.0)),
+                bias_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(0.0)))
 
         self.pointwise_conv2 = nn.Conv1D(
             channels,
diff --git a/paddlespeech/s2t/modules/decoder.py b/paddlespeech/s2t/modules/decoder.py
index 6b4d9591..b0ae27e5 100644
--- a/paddlespeech/s2t/modules/decoder.py
+++ b/paddlespeech/s2t/modules/decoder.py
@@ -76,19 +76,30 @@ class TransformerDecoder(BatchScorerInterface, nn.Layer):
             concat_after: bool=False, ):
 
         assert check_argument_types()
+
         nn.Layer.__init__(self)
         self.selfattention_layer_type = 'selfattn'
         attention_dim = encoder_output_size
 
         if input_layer == "embed":
             self.embed = nn.Sequential(
-                nn.Embedding(vocab_size, attention_dim),
+                nn.Embedding(
+                    vocab_size,
+                    attention_dim,
+                    weight_attr=paddle.ParamAttr(
+                        initializer=nn.initializer.Normal())),
                 PositionalEncoding(attention_dim, positional_dropout_rate), )
         else:
             raise ValueError(f"only 'embed' is supported: {input_layer}")
 
         self.normalize_before = normalize_before
-        self.after_norm = nn.LayerNorm(attention_dim, epsilon=1e-12)
+        self.after_norm = nn.LayerNorm(
+            attention_dim,
+            epsilon=1e-12,
+            weight_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0)),
+            bias_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0)))
         self.use_output_layer = use_output_layer
         self.output_layer = nn.Linear(attention_dim, vocab_size)
 
diff --git a/paddlespeech/s2t/modules/decoder_layer.py b/paddlespeech/s2t/modules/decoder_layer.py
index 520b18de..8eee5ceb 100644
--- a/paddlespeech/s2t/modules/decoder_layer.py
+++ b/paddlespeech/s2t/modules/decoder_layer.py
@@ -62,9 +62,27 @@ class DecoderLayer(nn.Layer):
         self.self_attn = self_attn
         self.src_attn = src_attn
         self.feed_forward = feed_forward
-        self.norm1 = nn.LayerNorm(size, epsilon=1e-12)
-        self.norm2 = nn.LayerNorm(size, epsilon=1e-12)
-        self.norm3 = nn.LayerNorm(size, epsilon=1e-12)
+        self.norm1 = nn.LayerNorm(
+            size,
+            epsilon=1e-12,
+            weight_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0)),
+            bias_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0)))
+        self.norm2 = nn.LayerNorm(
+            size,
+            epsilon=1e-12,
+            weight_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0)),
+            bias_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0)))
+        self.norm3 = nn.LayerNorm(
+            size,
+            epsilon=1e-12,
+            weight_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0)),
+            bias_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0)))
         self.dropout = nn.Dropout(dropout_rate)
         self.normalize_before = normalize_before
         self.concat_after = concat_after
diff --git a/paddlespeech/s2t/modules/encoder.py b/paddlespeech/s2t/modules/encoder.py
index 5c8ba081..5f7b8e99 100644
--- a/paddlespeech/s2t/modules/encoder.py
+++ b/paddlespeech/s2t/modules/encoder.py
@@ -129,7 +129,13 @@ class BaseEncoder(nn.Layer):
                 d_model=output_size, dropout_rate=positional_dropout_rate), )
 
         self.normalize_before = normalize_before
-        self.after_norm = nn.LayerNorm(output_size, epsilon=1e-12)
+        self.after_norm = nn.LayerNorm(
+            output_size,
+            epsilon=1e-12,
+            weight_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0)),
+            bias_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0)))
         self.static_chunk_size = static_chunk_size
         self.use_dynamic_chunk = use_dynamic_chunk
         self.use_dynamic_left_chunk = use_dynamic_left_chunk
@@ -457,6 +463,7 @@ class ConformerEncoder(BaseEncoder):
             cnn_module_norm (str): cnn conv norm type, Optional['batch_norm','layer_norm']
         """
         assert check_argument_types()
+
         super().__init__(input_size, output_size, attention_heads, linear_units,
                          num_blocks, dropout_rate, positional_dropout_rate,
                          attention_dropout_rate, input_layer,
diff --git a/paddlespeech/s2t/modules/encoder_layer.py b/paddlespeech/s2t/modules/encoder_layer.py
index d39c0695..69a3f67b 100644
--- a/paddlespeech/s2t/modules/encoder_layer.py
+++ b/paddlespeech/s2t/modules/encoder_layer.py
@@ -39,7 +39,7 @@ class TransformerEncoderLayer(nn.Layer):
             normalize_before: bool=True,
             concat_after: bool=False, ):
         """Construct an EncoderLayer object.
-        
+
         Args:
             size (int): Input dimension.
             self_attn (nn.Layer): Self-attention module instance.
@@ -147,7 +147,7 @@ class ConformerEncoderLayer(nn.Layer):
             normalize_before: bool=True,
             concat_after: bool=False, ):
         """Construct an EncoderLayer object.
-        
+
         Args:
             size (int): Input dimension.
             self_attn (nn.Layer): Self-attention module instance.
@@ -174,18 +174,46 @@ class ConformerEncoderLayer(nn.Layer):
         self.feed_forward = feed_forward
         self.feed_forward_macaron = feed_forward_macaron
         self.conv_module = conv_module
-        self.norm_ff = nn.LayerNorm(size, epsilon=1e-12)  # for the FNN module
-        self.norm_mha = nn.LayerNorm(size, epsilon=1e-12)  # for the MHA module
+        self.norm_ff = nn.LayerNorm(
+            size,
+            epsilon=1e-12,
+            weight_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0)),
+            bias_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0)))  # for the FNN module
+        self.norm_mha = nn.LayerNorm(
+            size,
+            epsilon=1e-12,
+            weight_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0)),
+            bias_attr=paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0)))  # for the MHA module
         if feed_forward_macaron is not None:
-            self.norm_ff_macaron = nn.LayerNorm(size, epsilon=1e-12)
+            self.norm_ff_macaron = nn.LayerNorm(
+                size,
+                epsilon=1e-12,
+                weight_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(1.0)),
+                bias_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(0.0)))
             self.ff_scale = 0.5
         else:
             self.ff_scale = 1.0
         if self.conv_module is not None:
             self.norm_conv = nn.LayerNorm(
-                size, epsilon=1e-12)  # for the CNN module
+                size,
+                epsilon=1e-12,
+                weight_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(1.0)),
+                bias_attr=paddle.ParamAttr(initializer=nn.initializer.Constant(
+                    0.0)))  # for the CNN module
             self.norm_final = nn.LayerNorm(
-                size, epsilon=1e-12)  # for the final output of the block
+                size,
+                epsilon=1e-12,
+                weight_attr=paddle.ParamAttr(
+                    initializer=nn.initializer.Constant(1.0)),
+                bias_attr=paddle.ParamAttr(initializer=nn.initializer.Constant(
+                    0.0)))  # for the final output of the block
         self.dropout = nn.Dropout(dropout_rate)
         self.size = size
         self.normalize_before = normalize_before
diff --git a/paddlespeech/s2t/modules/initializer.py b/paddlespeech/s2t/modules/initializer.py
new file mode 100644
index 00000000..c91ab231
--- /dev/null
+++ b/paddlespeech/s2t/modules/initializer.py
@@ -0,0 +1,272 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+from paddle.fluid import framework
+from paddle.fluid.framework import in_dygraph_mode, default_main_program
+import numpy as np
+from paddle.fluid.core import VarDesc
+from paddle.fluid import unique_name
+
+__all__ = [
+    'MSRAInitializer'
+]
+
+
+class Initializer(object):
+    """Base class for variable initializers
+
+    Defines the common interface of variable initializers.
+    They add operations to the init program that are used
+    to initialize variables. Users should not use this class
+    directly, but need to use one of its implementations.
+    """
+
+    def __init__(self):
+        pass
+
+    def __call__(self, param, block=None):
+        """Add corresponding initialization operations to the network
+        """
+        raise NotImplementedError()
+
+    def _check_block(self, block):
+        if block is None:
+            block = default_main_program().global_block()
+
+        return block
+
+    def _compute_fans(self, var):
+        """Compute the fan_in and the fan_out for layers
+
+        This method computes the fan_in and the fan_out
+        for neural network layers, if not specified. It is
+        not possible to perfectly estimate fan_in and fan_out.
+        This method will estimate it correctly for matrix multiply and
+        convolutions.
+
+        Args:
+            var: variable for which fan_in and fan_out have to be computed
+
+        Returns:
+            tuple of two integers (fan_in, fan_out)
+        """
+        shape = var.shape
+        if not shape or len(shape) == 0:
+            fan_in = fan_out = 1
+        elif len(shape) == 1:
+            fan_in = fan_out = shape[0]
+        elif len(shape) == 2:
+            # This is the case for simple matrix multiply
+            fan_in = shape[0]
+            fan_out = shape[1]
+        else:
+            # Assume this to be a convolutional kernel
+            # In PaddlePaddle, the shape of the kernel is like:
+            # [num_filters, num_filter_channels, ...] where the remaining
+            # dimensions are the filter_size
+            receptive_field_size = np.prod(shape[2:])
+            fan_in = shape[1] * receptive_field_size
+            fan_out = shape[0] * receptive_field_size
+
+        return (fan_in, fan_out)
+
+
+
+class MSRAInitializer(Initializer):
+    r"""Implements the MSRA initializer a.k.a. Kaiming Initializer
+
+    This class implements the weight initialization from the paper
+    `Delving Deep into Rectifiers: Surpassing Human-Level Performance on
+    ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
+    by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
+    robust initialization method that particularly considers the rectifier
+    nonlinearities. In case of Uniform distribution, the range is [-x, x], where
+
+    .. math::
+
+        x = \sqrt{\\frac{6.0}{fan\_in}}
+
+    In case of Normal distribution, the mean is 0 and the standard deviation
+    is
+
+    .. math::
+
+        \sqrt{\\frac{2.0}{fan\_in}}
+
+    Args:
+        uniform (bool): whether to use uniform or normal distribution
+        fan_in (float32|None): fan_in for MSRAInitializer. If None, it is\
+        inferred from the variable. default is None.
+        seed (int32): random seed
+
+    Note:
+        It is recommended to set fan_in to None for most cases.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.fluid as fluid
+            paddle.enable_static()
+            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
+            fc = fluid.layers.fc(input=x, size=10,
+                param_attr=fluid.initializer.MSRA(uniform=False))
+
+    """
+
+    def __init__(self, uniform=True, fan_in=None, seed=0):
+        """Constructor for MSRAInitializer
+        """
+        assert uniform is not None
+        assert seed is not None
+        super(MSRAInitializer, self).__init__()
+        self._uniform = uniform
+        self._fan_in = fan_in
+        self._seed = seed
+
+    def __call__(self, var, block=None):
+        """Initialize the input tensor with MSRA initialization.
+
+        Args:
+            var(Tensor): Tensor that needs to be initialized.
+            block(Block, optional): The block in which initialization ops
+                   should be added. Used in static graph only, default None.
+
+        Returns:
+            The initialization op
+        """
+        block = self._check_block(block)
+
+        assert isinstance(var, framework.Variable)
+        assert isinstance(block, framework.Block)
+        f_in, f_out = self._compute_fans(var)
+
+        # If fan_in is passed, use it
+        fan_in = f_in if self._fan_in is None else self._fan_in
+
+        if self._seed == 0:
+            self._seed = block.program.random_seed
+
+        # to be compatible of fp16 initalizers
+        if var.dtype == VarDesc.VarType.FP16 or (
+                var.dtype == VarDesc.VarType.BF16 and not self._uniform):
+            out_dtype = VarDesc.VarType.FP32
+            out_var = block.create_var(
+                name=unique_name.generate(".".join(
+                    ['masra_init', var.name, 'tmp'])),
+                shape=var.shape,
+                dtype=out_dtype,
+                type=VarDesc.VarType.LOD_TENSOR,
+                persistable=False)
+        else:
+            out_dtype = var.dtype
+            out_var = var
+
+        if self._uniform:
+            limit = np.sqrt(1.0 / float(fan_in))
+            op = block.append_op(
+                type="uniform_random",
+                inputs={},
+                outputs={"Out": out_var},
+                attrs={
+                    "shape": out_var.shape,
+                    "dtype": int(out_dtype),
+                    "min": -limit,
+                    "max": limit,
+                    "seed": self._seed
+                },
+                stop_gradient=True)
+
+        else:
+            std = np.sqrt(2.0 / float(fan_in))
+            op = block.append_op(
+                type="gaussian_random",
+                outputs={"Out": out_var},
+                attrs={
+                    "shape": out_var.shape,
+                    "dtype": int(out_dtype),
+                    "mean": 0.0,
+                    "std": std,
+                    "seed": self._seed
+                },
+                stop_gradient=True)
+
+        if var.dtype == VarDesc.VarType.FP16 or (
+                var.dtype == VarDesc.VarType.BF16 and not self._uniform):
+            block.append_op(
+                type="cast",
+                inputs={"X": out_var},
+                outputs={"Out": var},
+                attrs={"in_dtype": out_var.dtype,
+                       "out_dtype": var.dtype})
+
+        if not framework.in_dygraph_mode():
+            var.op = op
+        return op
+
+class KaimingUniform(MSRAInitializer):
+    r"""Implements the Kaiming Uniform initializer
+
+    This class implements the weight initialization from the paper
+    `Delving Deep into Rectifiers: Surpassing Human-Level Performance on
+    ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
+    by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
+    robust initialization method that particularly considers the rectifier
+    nonlinearities.
+
+    In case of Uniform distribution, the range is [-x, x], where
+
+    .. math::
+
+        x = \sqrt{\frac{6.0}{fan\_in}}
+
+    Args:
+        fan_in (float32|None): fan_in for Kaiming uniform Initializer. If None, it is\
+        inferred from the variable. default is None.
+
+    Note:
+        It is recommended to set fan_in to None for most cases.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+            import paddle.nn as nn
+
+            linear = nn.Linear(2,
+                               4,
+                               weight_attr=nn.initializer.KaimingUniform())
+            data = paddle.rand([30, 10, 2], dtype='float32')
+            res = linear(data)
+
+    """
+
+    def __init__(self, fan_in=None):
+        super(KaimingUniform, self).__init__(
+            uniform=True, fan_in=fan_in, seed=0)
+
+
+
+# We short the class name, since users will use the initializer with the package
+# name. The sample code:
+#
+# import paddle.fluid as fluid
+#
+# hidden = fluid.layers.fc(...,
+#                          param_attr=ParamAttr(fluid.initializer.Xavier()))
+#
+# It is no need to add an `Initializer` as the class suffix
+MSRA = MSRAInitializer
diff --git a/paddlespeech/s2t/modules/nets_utils.py b/paddlespeech/s2t/modules/nets_utils.py
new file mode 100644
index 00000000..10915c8c
--- /dev/null
+++ b/paddlespeech/s2t/modules/nets_utils.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Modified from espnet(https://github.com/espnet/espnet)
+from paddle import nn
+from typeguard import check_argument_types
+
+def initialize(model: nn.Layer, init: str):
+    """Initialize weights of a neural network module.
+
+    Parameters are initialized using the given method or distribution.
+
+    Custom initialization routines can be implemented into submodules
+
+    Args:
+        model (nn.Layer): Target.
+        init (str): Method of initialization.
+    """
+    assert check_argument_types()
+
+    if init == "xavier_uniform":
+        nn.initializer.set_global_initializer(nn.initializer.XavierUniform(),
+                                              nn.initializer.Constant())
+    elif init == "xavier_normal":
+        nn.initializer.set_global_initializer(nn.initializer.XavierNormal(),
+                                              nn.initializer.Constant())
+    elif init == "kaiming_uniform":
+        nn.initializer.set_global_initializer(nn.initializer.KaimingUniform(),
+                                              nn.initializer.KaimingUniform())
+    elif init == "kaiming_normal":
+        nn.initializer.set_global_initializer(nn.initializer.KaimingNormal(),
+                                              nn.initializer.Constant())
+    else:
+        raise ValueError("Unknown initialization: " + init)

From d28ccfa96b7195068e335bddd53941eacb9203f1 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 21 Mar 2022 00:06:16 +0800
Subject: [PATCH 031/126] add vector cli component, test=doc

---
 examples/voxceleb/sv0/conf/ecapa_tdnn.yaml    |   4 +-
 examples/voxceleb/sv0/local/data.sh           |  27 +-
 examples/voxceleb/sv0/local/emb.sh            |  50 ++-
 examples/voxceleb/sv0/local/test.sh           |  42 ++-
 examples/voxceleb/sv0/local/train.sh          |  49 ++-
 examples/voxceleb/sv0/run.sh                  |  12 +-
 paddleaudio/paddleaudio/metric/eer.py         |  14 +-
 paddlespeech/cli/__init__.py                  |   1 +
 paddlespeech/cli/vector/__init__.py           |  14 +
 paddlespeech/cli/vector/infer.py              | 345 ++++++++++++++++++
 .../vector/exps/ecapa_tdnn/extract_emb.py     |   6 +-
 paddlespeech/vector/exps/ecapa_tdnn/test.py   |   7 +-
 paddlespeech/vector/exps/ecapa_tdnn/train.py  |  49 ++-
 paddlespeech/vector/io/batch.py               |  92 ++++-
 paddlespeech/vector/modules/loss.py           |  24 ++
 paddlespeech/vector/modules/sid_model.py      |  26 ++
 16 files changed, 712 insertions(+), 50 deletions(-)
 create mode 100644 paddlespeech/cli/vector/__init__.py
 create mode 100644 paddlespeech/cli/vector/infer.py

diff --git a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
index 0f4bf189..e58dca82 100644
--- a/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
+++ b/examples/voxceleb/sv0/conf/ecapa_tdnn.yaml
@@ -14,10 +14,10 @@ random_chunk: True
 #                FEATURE EXTRACTION SETTING               #
 ###########################################################
 # currently, we only support fbank
-sample_rate: 16000
+sr: 16000           # sample rate
 n_mels: 80
 window_size: 400     #25ms, sample rate 16000, 25 * 16000 / 1000 = 400 
-hop_length: 160     #10ms, sample rate 16000, 10 * 16000 / 1000 = 160
+hop_size: 160        #10ms, sample rate 16000, 10 * 16000 / 1000 = 160
 
 ###########################################################
 #                       MODEL SETTING                     #
diff --git a/examples/voxceleb/sv0/local/data.sh b/examples/voxceleb/sv0/local/data.sh
index ec9c4c58..42629c69 100755
--- a/examples/voxceleb/sv0/local/data.sh
+++ b/examples/voxceleb/sv0/local/data.sh
@@ -1,15 +1,36 @@
 #!/bin/bash
-
-stage=-1
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+stage=0
 stop_stage=100
 
 . ${MAIN_ROOT}/utils/parse_options.sh || exit -1;
 
+if [ $# -ne 2 ] ; then
+   echo "Usage: $0 [options] <data-dir> <conf-path>";
+   echo "e.g.: $0 ./data/ conf/ecapa_tdnn.yaml"
+   echo "Options: "
+   echo "  --stage <stage|-1>               # Used to run a partially-completed data process from somewhere in the middle."
+   echo "  --stop-stage <stop-stage|100>    # Used to run a partially-completed data process stop stage in the middle"
+   exit 1;
+fi
+
 dir=$1
 conf_path=$2
 mkdir -p ${dir}
 
-if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
     # data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
     # we should use the local/convert.sh convert m4a to wav
     python3 local/data_prepare.py \
diff --git a/examples/voxceleb/sv0/local/emb.sh b/examples/voxceleb/sv0/local/emb.sh
index 482e658e..31d79e52 100755
--- a/examples/voxceleb/sv0/local/emb.sh
+++ b/examples/voxceleb/sv0/local/emb.sh
@@ -1,13 +1,51 @@
 #!/bin/bash
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 . ./path.sh
 
-exp_dir=exp/ecapa-tdnn-vox12-big//epoch_10/            # experiment directory
+stage=0
+stop_stage=100
+exp_dir=exp/ecapa-tdnn-vox12-big/            # experiment directory
 conf_path=conf/ecapa_tdnn.yaml
 audio_path="demo/voxceleb/00001.wav"
+use_gpu=true
+
+. ${MAIN_ROOT}/utils/parse_options.sh || exit -1;
+
+if [ $# -ne 0 ] ; then
+   echo "Usage: $0 [options]";
+   echo "e.g.: $0 ./data/ exp/voxceleb12/ conf/ecapa_tdnn.yaml"
+   echo "Options: "
+   echo "  --use-gpu <true,false|true>      # specify is gpu is to be used for training"
+   echo "  --stage <stage|-1>               # Used to run a partially-completed data process from somewhere in the middle."
+   echo "  --stop-stage <stop-stage|100>    # Used to run a partially-completed data process stop stage in the middle"
+   echo "  --exp-dir                        # experiment directorh, where is has the model.pdparams"
+   echo "  --conf-path                      # configuration file for extracting the embedding"
+   echo "  --audio-path                     # audio-path, which will be processed to extract the embedding"
+   exit 1;
+fi
 
-source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
+# set the test device
+device="cpu"
+if ${use_gpu}; then
+    device="gpu"
+fi
 
-# extract the audio embedding
-python3 ${BIN_DIR}/extract_emb.py --device "gpu" \
-          --config ${conf_path} \
-          --audio-path ${audio_path} --load-checkpoint ${exp_dir}
\ No newline at end of file
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    # extract the audio embedding
+    python3 ${BIN_DIR}/extract_emb.py --device ${device} \
+            --config ${conf_path} \
+            --audio-path ${audio_path} --load-checkpoint ${exp_dir}
+fi
\ No newline at end of file
diff --git a/examples/voxceleb/sv0/local/test.sh b/examples/voxceleb/sv0/local/test.sh
index d8a1a0ba..4460a165 100644
--- a/examples/voxceleb/sv0/local/test.sh
+++ b/examples/voxceleb/sv0/local/test.sh
@@ -1,8 +1,42 @@
+#!/bin/bash
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stage=1
+stop_stage=100
+use_gpu=true    # if true, we run on GPU.
+
+. ${MAIN_ROOT}/utils/parse_options.sh || exit -1;
+
+if [ $# -ne 3 ] ; then
+   echo "Usage: $0 [options] <data-dir> <exp-dir> <conf-path>";
+   echo "e.g.: $0 ./data/ exp/voxceleb12/ conf/ecapa_tdnn.yaml"
+   echo "Options: "
+   echo "  --use-gpu <true,false|true>      # specify is gpu is to be used for training"
+   echo "  --stage <stage|-1>               # Used to run a partially-completed data process from somewhere in the middle."
+   echo "  --stop-stage <stop-stage|100>    # Used to run a partially-completed data process stop stage in the middle"
+   exit 1;
+fi
+
 dir=$1
 exp_dir=$2
 conf_path=$3
 
-python3 ${BIN_DIR}/test.py \
-        --config ${conf_path} \
-        --data-dir ${dir} \
-        --load-checkpoint ${exp_dir}
\ No newline at end of file
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+   # test the model and compute the eer metrics
+   python3 ${BIN_DIR}/test.py \
+         --data-dir ${dir} \
+         --load-checkpoint ${exp_dir} \
+         --config ${conf_path}
+fi
diff --git a/examples/voxceleb/sv0/local/train.sh b/examples/voxceleb/sv0/local/train.sh
index 385e8caa..5477d0a3 100755
--- a/examples/voxceleb/sv0/local/train.sh
+++ b/examples/voxceleb/sv0/local/train.sh
@@ -1,18 +1,57 @@
 #!/bin/bash
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stage=0
+stop_stage=100
+use_gpu=true    # if true, we run on GPU.
+
+. ${MAIN_ROOT}/utils/parse_options.sh || exit -1;
+
+if [ $# -ne 3 ] ; then
+   echo "Usage: $0 [options] <data-dir> <exp-dir> <conf-path>";
+   echo "e.g.: $0 ./data/ exp/voxceleb12/ conf/ecapa_tdnn.yaml"
+   echo "Options: "
+   echo "  --use-gpu <true,false|true>      # specify is gpu is to be used for training"
+   echo "  --stage <stage|-1>               # Used to run a partially-completed data process from somewhere in the middle."
+   echo "  --stop-stage <stop-stage|100>    # Used to run a partially-completed data process stop stage in the middle"
+   exit 1;
+fi
 
 dir=$1
 exp_dir=$2
 conf_path=$3
 
+# get the gpu nums for training
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-# train the speaker identification task with voxceleb data
-# Note: we will store the log file in exp/log directory
-python3 -m paddle.distributed.launch --gpus=$CUDA_VISIBLE_DEVICES \
-    ${BIN_DIR}/train.py --device "gpu" --checkpoint-dir ${exp_dir} --augment \
-    --data-dir ${dir} --config ${conf_path}
+# setting training device
+device="cpu"
+if ${use_gpu}; then
+    device="gpu"
+fi
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    # train the speaker identification task with voxceleb data
+    # and we will create the trained model parameters in ${exp_dir}/model.pdparams as the soft link
+    # Note: we will store the log file in exp/log directory
+    python3 -m paddle.distributed.launch --gpus=$CUDA_VISIBLE_DEVICES \
+        ${BIN_DIR}/train.py --device ${device} --checkpoint-dir ${exp_dir} \
+        --data-dir ${dir} --config ${conf_path}
 
+fi 
 
 if [ $? -ne 0 ]; then
     echo "Failed in training!"
diff --git a/examples/voxceleb/sv0/run.sh b/examples/voxceleb/sv0/run.sh
index e38027e9..bbc9e3db 100755
--- a/examples/voxceleb/sv0/run.sh
+++ b/examples/voxceleb/sv0/run.sh
@@ -36,11 +36,10 @@ stop_stage=50
 # data directory
 # if we set the variable ${dir}, we will store the wav info to this directory
 # otherwise, we will store the wav info to vox1 and vox2 directory respectively
-# vox2 wav path, we must convert the m4a format to wav format 
-# dir=data-demo/                          # data info directory    
-dir=demo/                          # data info directory   
+# vox2 wav path, we must convert the m4a format to wav format    
+dir=data/                                 # data info directory   
 
-exp_dir=exp/ecapa-tdnn-vox12-big//            # experiment directory
+exp_dir=exp/ecapa-tdnn-vox12-big/            # experiment directory
 conf_path=conf/ecapa_tdnn.yaml          
 gpus=0,1,2,3
 
@@ -50,16 +49,15 @@ mkdir -p ${exp_dir}
 
 if [ $stage -le 0 ] && [ ${stop_stage} -ge 0 ]; then 
      # stage 0: data prepare for vox1 and vox2, vox2 must be converted from m4a to wav
-     # and we should specifiy the vox2 data in the data.sh
      bash ./local/data.sh ${dir} ${conf_path}|| exit -1;
-fi 
+fi
 
 if [ $stage -le 1 ] && [ ${stop_stage} -ge 1 ]; then
      # stage 1: train the speaker identification model
      CUDA_VISIBLE_DEVICES=${gpus} bash ./local/train.sh ${dir} ${exp_dir} ${conf_path} 
 fi
 
-if [ $stage -le 2 ]; then
+if [ $stage -le 2 ] && [ ${stop_stage} -ge 2 ]; then
      # stage 2: get the speaker verification scores with cosine function
      #          now we only support use cosine to get the scores
      CUDA_VISIBLE_DEVICES=0 bash ./local/test.sh ${dir} ${exp_dir} ${conf_path}
diff --git a/paddleaudio/paddleaudio/metric/eer.py b/paddleaudio/paddleaudio/metric/eer.py
index 7738987e..a1166d3f 100644
--- a/paddleaudio/paddleaudio/metric/eer.py
+++ b/paddleaudio/paddleaudio/metric/eer.py
@@ -19,9 +19,15 @@ from sklearn.metrics import roc_curve
 
 
 def compute_eer(labels: np.ndarray, scores: np.ndarray) -> List[float]:
-    '''
-    Compute EER and return score threshold.
-    '''
+    """Compute EER and return score threshold.
+
+    Args:
+        labels (np.ndarray): the trial label, shape: [N], one-dimention, N refer to the samples num
+        scores (np.ndarray): the trial scores, shape: [N], one-dimention, N refer to the samples num
+
+    Returns:
+        List[float]: eer and the specific threshold
+    """
     fpr, tpr, threshold = roc_curve(y_true=labels, y_score=scores)
     fnr = 1 - tpr
     eer_threshold = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
@@ -54,7 +60,7 @@ def compute_minDCF(positive_scores,
         p_target (float, optional): Prior probability of having a target (default 0.01).
 
     Returns:
-        _type_: min dcf 
+        List[float]: min dcf and the specific threshold
     """
     # Computing candidate thresholds
     if len(positive_scores.shape) > 1:
diff --git a/paddlespeech/cli/__init__.py b/paddlespeech/cli/__init__.py
index b526a384..ddf0359b 100644
--- a/paddlespeech/cli/__init__.py
+++ b/paddlespeech/cli/__init__.py
@@ -21,5 +21,6 @@ from .st import STExecutor
 from .stats import StatsExecutor
 from .text import TextExecutor
 from .tts import TTSExecutor
+from .vector import VectorExecutor
 
 _locale._getdefaultlocale = (lambda *args: ['en_US', 'utf8'])
diff --git a/paddlespeech/cli/vector/__init__.py b/paddlespeech/cli/vector/__init__.py
new file mode 100644
index 00000000..038596af
--- /dev/null
+++ b/paddlespeech/cli/vector/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .infer import VectorExecutor
diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
new file mode 100644
index 00000000..de4d6621
--- /dev/null
+++ b/paddlespeech/cli/vector/infer.py
@@ -0,0 +1,345 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import os
+import sys
+from collections import OrderedDict
+from typing import List
+from typing import Optional
+from typing import Union
+
+import librosa
+import numpy as np
+import paddle
+import soundfile
+from yacs.config import CfgNode
+
+from paddleaudio.backends import load as load_audio
+from paddleaudio.compliance.librosa import melspectrogram
+from ..download import get_path_from_url
+from ..executor import BaseExecutor
+from ..log import logger
+from ..utils import cli_register
+from ..utils import download_and_decompress
+from ..utils import MODEL_HOME
+from ..utils import stats_wrapper
+from paddlespeech.vector.io.batch import feature_normalize
+from paddlespeech.s2t.frontend.featurizer.text_featurizer import TextFeaturizer
+from paddlespeech.s2t.transform.transformation import Transformation
+from paddlespeech.s2t.utils.dynamic_import import dynamic_import
+from paddlespeech.s2t.utils.utility import UpdateConfig
+from paddlespeech.vector.modules.sid_model import SpeakerIdetification
+
+pretrained_models = {
+    # The tags for pretrained_models should be "{model_name}[_{dataset}][-{lang}][-...]".
+    # e.g. "conformer_wenetspeech-zh-16k" and "panns_cnn6-32k".
+    # Command line and python api use "{model_name}[_{dataset}]" as --model, usage:
+    # "paddlespeech asr --model conformer_wenetspeech --lang zh --sr 16000 --input ./input.wav"
+    "ecapa_tdnn-16k": {
+        'url':
+        'https://paddlespeech.bj.bcebos.com/s2t/wenetspeech/asr1_conformer_wenetspeech_ckpt_0.1.1.model.tar.gz',
+        'md5':
+        '76cb19ed857e6623856b7cd7ebbfeda4',
+        'cfg_path':
+        'model.yaml',
+        'ckpt_path':
+        'exp/conformer/checkpoints/wenetspeech',
+    },
+}
+
+model_alias = {
+    "ecapa_tdnn": "paddlespeech.vector.models.ecapa_tdnn:EcapaTdnn",
+}
+
+
+@cli_register(
+    name="paddlespeech.vector",
+    description="Speech to vector embedding infer command.")
+class VectorExecutor(BaseExecutor):
+    def __init__(self):
+        super(VectorExecutor, self).__init__()
+
+        self.parser = argparse.ArgumentParser(
+            prog="paddlespeech.vector", add_help=True)
+        self.parser.add_argument(
+            "--model",
+            type=str,
+            default="ecapa_tdnn-voxceleb12",
+            choices=["ecapa_tdnn"],
+            help="Choose model type of asr task.")
+        self.parser.add_argument(
+            "--task",
+            type=str,
+            default="spk",
+            choices=["spk"],
+            help="task type in vector domain")
+        self.parser.add_argument(
+            "--input", type=str, default=None, help="Audio file to recognize.")
+        self.parser.add_argument(
+            "--sample_rate",
+            type=int,
+            default=16000,
+            choices=[16000, 8000],
+            help="Choose the audio sample rate of the model. 8000 or 16000")
+        self.parser.add_argument(
+            "--ckpt_path",
+            type=str,
+            default=None,
+            help="Checkpoint file of model.")
+        self.parser.add_argument(
+            '--config',
+            type=str,
+            default=None,
+            help='Config of asr task. Use deault config when it is None.')
+        self.parser.add_argument(
+            "--device",
+            type=str,
+            default=paddle.get_device(),
+            help="Choose device to execute model inference.")
+        self.parser.add_argument(
+            '-d',
+            '--job_dump_result',
+            action='store_true',
+            help='Save job result into file.')
+
+        self.parser.add_argument(
+            '-v',
+            '--verbose',
+            action='store_true',
+            help='Increase logger verbosity of current task.')
+
+    def execute(self, argv: List[str]) -> bool:
+        """Command line entry for vector model
+
+        Args:
+            argv (List[str]): command line args list
+
+        Returns:
+            bool: 
+                 False: some audio occurs error
+                 True: all audio process success
+        """
+        # stage 0: parse the args and get the required args
+        parser_args = self.parser.parse_args(argv)
+        model = parser_args.model
+        sample_rate = parser_args.sample_rate
+        config = parser_args.config
+        ckpt_path = parser_args.ckpt_path
+        device = parser_args.device
+
+        # stage 1: configurate the verbose flag
+        if not parser_args.verbose:
+            self.disable_task_loggers()
+
+        # stage 2: read the input data and store them as a list
+        task_source = self.get_task_source(parser_args.input)
+        logger.info(f"task source: {task_source}")
+
+        # stage 3: process the audio one by one
+        task_result = OrderedDict()
+        has_exceptions = False
+        for id_, input_ in task_source.items():
+            try:
+                res = self(input_, model, sample_rate, config, ckpt_path,
+                           device)
+                task_result[id_] = res
+            except Exception as e:
+                has_exceptions = True
+                task_result[id_] = f'{e.__class__.__name__}: {e}'
+
+        logger.info("task result as follows: ")
+        logger.info(f"{task_result}")
+
+        # stage 4: process the all the task results
+        self.process_task_results(parser_args.input, task_result,
+                                  parser_args.job_dump_result)
+
+        # stage 5: return the exception flag
+        #          if return False, somen audio process occurs error
+        if has_exceptions:
+            return False
+        else:
+            return True
+
+    @stats_wrapper
+    def __call__(self,
+                 audio_file: os.PathLike,
+                 model: str='ecapa_tdnn-voxceleb12',
+                 sample_rate: int=16000,
+                 config: os.PathLike=None,
+                 ckpt_path: os.PathLike=None,
+                 force_yes: bool=False,
+                 device=paddle.get_device()):
+        audio_file = os.path.abspath(audio_file)
+        if not self._check(audio_file, sample_rate):
+            sys.exit(-1)
+
+        logger.info(f"device type: {device}")
+        paddle.device.set_device(device)
+        self._init_from_path(model, sample_rate, config, ckpt_path)
+        self.preprocess(model, audio_file)
+        self.infer(model)
+        res = self.postprocess()
+
+        return res
+
+    def _get_pretrained_path(self, tag: str) -> os.PathLike:
+        support_models = list(pretrained_models.keys())
+        assert tag in pretrained_models, \
+            'The model "{}" you want to use has not been supported, \
+            please choose other models.\n \
+            The support models includes \n\t\t{}'.format(tag, "\n\t\t".join(support_models))
+
+        res_path = os.path.join(MODEL_HOME, tag)
+
+    def _init_from_path(self,
+                        model_type: str='ecapa_tdnn-voxceleb12',
+                        sample_rate: int=16000,
+                        cfg_path: Optional[os.PathLike]=None,
+                        ckpt_path: Optional[os.PathLike]=None):
+        if hasattr(self, "model"):
+            logger.info("Model has been initialized")
+            return
+
+        # stage 1: get the model and config path
+        if cfg_path is None or ckpt_path is None:
+            sample_rate_str = "16k" if sample_rate == 16000 else "8k"
+            tag = model_type + "-" + sample_rate_str
+            res_path = self._get_pretrained_path(tag)
+        else:
+            self.cfg_path = os.path.abspath(cfg_path)
+            self.ckpt_path = os.path.abspath(ckpt_path + ".pdparams")
+            self.res_path = os.path.dirname(
+                os.path.dirname(os.path.abspath(self.cfg_path)))
+
+        logger.info(f"start to read the ckpt from {self.ckpt_path}")
+        logger.info(f"read the config from {self.cfg_path}")
+        logger.info(f"get the res path {self.res_path}")
+
+        # stage 2: read and config and init the model body
+        self.config = CfgNode(new_allowed=True)
+        self.config.merge_from_file(self.cfg_path)
+
+        # stage 3: get the model name to instance the model network with dynamic_import
+        # Noet: we use the '-' to get the model name instead of '_'
+        logger.info("start to dynamic import the model class")
+        model_name = model_type[:model_type.rindex('-')]
+        logger.info(f"model name {model_name}")
+        model_class = dynamic_import(model_name, model_alias)
+        model_conf = self.config.model
+        backbone = model_class(**model_conf)
+        model = SpeakerIdetification(
+            backbone=backbone, num_class=self.config.num_speakers)
+        self.model = model
+        self.model.eval()
+
+        # stage 4: load the model parameters
+        logger.info("start to set the model parameters to model")
+        model_dict = paddle.load(self.ckpt_path)
+        self.model.set_state_dict(model_dict)
+
+        logger.info("create the model instance success")
+
+    @paddle.no_grad()
+    def infer(self, model_type: str):
+
+        feats = self._inputs["feats"]
+        lengths = self._inputs["lengths"]
+        logger.info(f"start to do backbone network model forward")
+        logger.info(
+            f"feats shape:{feats.shape}, lengths shape: {lengths.shape}")
+        # embedding from (1, emb_size, 1) -> (emb_size)
+        embedding = self.model.backbone(feats, lengths).squeeze().numpy()
+        logger.info(f"embedding size: {embedding.shape}")
+
+        self._outputs["embedding"] = embedding
+
+    def postprocess(self) -> Union[str, os.PathLike]:
+        return self._outputs["embedding"]
+
+    def preprocess(self, model_type: str, input_file: Union[str, os.PathLike]):
+        audio_file = input_file
+        if isinstance(audio_file, (str, os.PathLike)):
+            logger.info(f"Preprocess audio file: {audio_file}")
+
+        # stage 1: load the audio
+        waveform, sr = load_audio(audio_file)
+        logger.info(f"load the audio sample points, shape is: {waveform.shape}")
+
+        # stage 2: get the audio feat
+        try:
+            feat = melspectrogram(
+                x=waveform,
+                sr=self.config.sr,
+                n_mels=self.config.n_mels,
+                window_size=self.config.window_size,
+                hop_length=self.config.hop_size)
+            logger.info(f"extract the audio feat, shape is: {feat.shape}")
+        except Exception as e:
+            logger.info(f"feat occurs exception {e}")
+            sys.exit(-1)
+
+        feat = paddle.to_tensor(feat).unsqueeze(0)
+        # in inference period, the lengths is all one without padding
+        lengths = paddle.ones([1])
+        feat = feature_normalize(feat, mean_norm=True, std_norm=False)
+
+        logger.info(f"feats shape: {feat.shape}")
+        self._inputs["feats"] = feat
+        self._inputs["lengths"] = lengths
+
+        logger.info("audio extract the feat success")
+
+    def _check(self, audio_file: str, sample_rate: int):
+        self.sample_rate = sample_rate
+        if self.sample_rate != 16000 and self.sample_rate != 8000:
+            logger.error(
+                "invalid sample rate, please input --sr 8000 or --sr 16000")
+            return False
+
+        if isinstance(audio_file, (str, os.PathLike)):
+            if not os.path.isfile(audio_file):
+                logger.error("Please input the right audio file path")
+                return False
+
+        logger.info("checking the aduio file format......")
+        try:
+            audio, audio_sample_rate = soundfile.read(
+                audio_file, dtype="float32", always_2d=True)
+        except Exception as e:
+            logger.exception(e)
+            logger.error(
+                "can not open the audio file, please check the audio file format is 'wav'. \n \
+                 you can try to use sox to change the file format.\n \
+                 For example: \n \
+                 sample rate: 16k \n \
+                 sox input_audio.xx --rate 16k --bits 16 --channels 1 output_audio.wav \n \
+                 sample rate: 8k \n \
+                 sox input_audio.xx --rate 8k --bits 16 --channels 1 output_audio.wav \n \
+                 ")
+            return False
+
+        logger.info(f"The sample rate is {audio_sample_rate}")
+
+        if audio_sample_rate != self.sample_rate:
+            logger.error("The sample rate of the input file is not {}.\n \
+                            The program will resample the wav file to {}.\n \
+                            If the result does not meet your expectations，\n \
+                            Please input the 16k 16 bit 1 channel wav file. \
+                        ".format(self.sample_rate, self.sample_rate))
+            sys.exit(-1)
+        else:
+            logger.info("The audio file format is right")
+
+        return True
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
index 0d09d211..6dfcf06d 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
@@ -63,16 +63,16 @@ def extract_audio_embedding(args, config):
     # so the final shape is [1, dim, time]
     start_time = time.time()
     feat = melspectrogram(x=waveform, 
-                          sr=config.sample_rate,
+                          sr=config.sr,
                           n_mels=config.n_mels,
                           window_size=config.window_size,
-                          hop_length=config.hop_length)
+                          hop_length=config.hop_size)
     feat = paddle.to_tensor(feat).unsqueeze(0)
 
     # in inference period, the lengths is all one without padding
     lengths = paddle.ones([1])
     feat = feature_normalize(
-        feat, mean_norm=True, std_norm=False, convert_to_numpy=True)
+        feat, mean_norm=True, std_norm=False)
 
     # model backbone network forward the feats and get the embedding
     embedding = model.backbone(
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/test.py b/paddlespeech/vector/exps/ecapa_tdnn/test.py
index 03757033..76832fd8 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/test.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/test.py
@@ -49,8 +49,6 @@ def main(args, config):
 
     # stage3: load the pre-trained model
     #         we get the last model from the epoch and save_interval
-    last_save_epoch = (config.epochs // config.save_interval) * config.save_interval
-    args.load_checkpoint = os.path.join(args.load_checkpoint, "epoch_" + str(last_save_epoch))
     args.load_checkpoint = os.path.abspath(
         os.path.expanduser(args.load_checkpoint))
 
@@ -61,6 +59,7 @@ def main(args, config):
     logger.info(f'Checkpoint loaded from {args.load_checkpoint}')
 
     # stage4: construct the enroll and test dataloader
+
     enroll_dataset = VoxCeleb(
         subset='enroll',
         target_dir=args.data_dir,
@@ -68,7 +67,7 @@ def main(args, config):
         random_chunk=False,
         n_mels=config.n_mels,
         window_size=config.window_size,
-        hop_length=config.hop_length)
+        hop_length=config.hop_size)
     enroll_sampler = BatchSampler(
         enroll_dataset, batch_size=config.batch_size,
         shuffle=True)  # Shuffle to make embedding normalization more robust.
@@ -85,7 +84,7 @@ def main(args, config):
         random_chunk=False,
         n_mels=config.n_mels,
         window_size=config.window_size,
-        hop_length=config.hop_length)
+        hop_length=config.hop_size)
 
     test_sampler = BatchSampler(
         test_dataset, batch_size=config.batch_size, shuffle=True)
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/train.py b/paddlespeech/vector/exps/ecapa_tdnn/train.py
index 0d62c69d..fb02d486 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/train.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/train.py
@@ -15,6 +15,7 @@ import argparse
 import os
 
 import numpy as np
+import time
 import paddle
 from paddle.io import BatchSampler
 from paddle.io import DataLoader
@@ -35,6 +36,7 @@ from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 from paddlespeech.vector.training.scheduler import CyclicLRScheduler
 from paddlespeech.vector.training.seeding import seed_everything
 from paddlespeech.vector.utils.time import Timer
+from paddlespeech.vector.io.batch import batch_pad_right
 
 logger = Log(__name__).getlog()
 
@@ -55,7 +57,7 @@ def main(args, config):
     train_dataset = VoxCeleb('train', target_dir=args.data_dir)
     dev_dataset = VoxCeleb('dev', target_dir=args.data_dir)
 
-    if args.augment:
+    if config.augment:
         augment_pipeline = build_augment_pipeline(target_dir=args.data_dir)
     else:
         augment_pipeline = []
@@ -126,6 +128,7 @@ def main(args, config):
     #         we will comment the training process
     steps_per_epoch = len(train_sampler)
     timer = Timer(steps_per_epoch * config.epochs)
+    last_saved_epoch = ""
     timer.start()
 
     for epoch in range(start_epoch + 1, config.epochs + 1):
@@ -135,9 +138,19 @@ def main(args, config):
         avg_loss = 0
         num_corrects = 0
         num_samples = 0
+        train_reader_cost = 0.0
+        train_feat_cost = 0.0
+        train_run_cost = 0.0
+
+        reader_start = time.time()
         for batch_idx, batch in enumerate(train_loader):
+            train_reader_cost += time.time() - reader_start
+            
             # stage 9-1: batch data is audio sample points and speaker id label
+            feat_start = time.time()
             waveforms, labels = batch['waveforms'], batch['labels']
+            waveforms, lengths = batch_pad_right(waveforms.numpy())
+            waveforms = paddle.to_tensor(waveforms)
 
             # stage 9-2: audio sample augment method, which is done on the audio sample point
             #            the original wavefrom and the augmented waveform is concatented in a batch
@@ -153,18 +166,20 @@ def main(args, config):
             feats = []
             for waveform in waveforms.numpy():
                 feat = melspectrogram(x=waveform, 
-                                      sr=config.sample_rate, 
+                                      sr=config.sr, 
                                       n_mels=config.n_mels, 
                                       window_size=config.window_size, 
-                                      hop_length=config.hop_length)
+                                      hop_length=config.hop_size)
                 feats.append(feat)
             feats = paddle.to_tensor(np.asarray(feats))
 
             # stage 9-4: feature normalize, which help converge and imporve the performance
             feats = feature_normalize(
                 feats, mean_norm=True, std_norm=False)  # Features normalization
+            train_feat_cost += time.time() - feat_start
 
             # stage 9-5: model forward, such ecapa-tdnn, x-vector
+            train_start = time.time()
             logits = model(feats)
 
             # stage 9-6: loss function criterion, such AngularMargin, AdditiveAngularMargin
@@ -177,6 +192,7 @@ def main(args, config):
                           paddle.optimizer.lr.LRScheduler):
                 optimizer._learning_rate.step()
             optimizer.clear_grad()
+            train_run_cost += time.time() - train_start
 
             # stage 9-8: Calculate average loss per batch
             avg_loss += loss.numpy()[0]
@@ -186,7 +202,7 @@ def main(args, config):
             num_corrects += (preds == labels).numpy().sum()
             num_samples += feats.shape[0]
             timer.count()  # step plus one in timer
-
+            
             # stage 9-10: print the log information only on 0-rank per log-freq batchs
             if (batch_idx + 1) % config.log_interval == 0 and local_rank == 0:
                 lr = optimizer.get_lr()
@@ -197,6 +213,9 @@ def main(args, config):
                     epoch, config.epochs, batch_idx + 1, steps_per_epoch)
                 print_msg += ' loss={:.4f}'.format(avg_loss)
                 print_msg += ' acc={:.4f}'.format(avg_acc)
+                print_msg += ' avg_reader_cost: {:.5f} sec,'.format(train_reader_cost / config.log_interval)
+                print_msg += ' avg_feat_cost: {:.5f} sec,'.format(train_feat_cost / config.log_interval)
+                print_msg += ' avg_train_cost: {:.5f} sec,'.format(train_run_cost / config.log_interval)
                 print_msg += ' lr={:.4E} step/sec={:.2f} | ETA {}'.format(
                     lr, timer.timing, timer.eta)
                 logger.info(print_msg)
@@ -204,6 +223,11 @@ def main(args, config):
                 avg_loss = 0
                 num_corrects = 0
                 num_samples = 0
+                train_reader_cost = 0.0
+                train_feat_cost = 0.0
+                train_run_cost = 0.0
+
+            reader_start = time.time()
 
         # stage 9-11: save the model parameters only on 0-rank per save-freq batchs
         if epoch % config.save_interval == 0 and batch_idx + 1 == steps_per_epoch:
@@ -239,10 +263,10 @@ def main(args, config):
                     feats = []
                     for waveform in waveforms.numpy():
                         feat = melspectrogram(x=waveform, 
-                                              sr=config.sample_rate,
+                                              sr=config.sr,
                                               n_mels=config.n_mels,
                                               window_size=config.window_size,
-                                              hop_length=config.hop_length)
+                                              hop_length=config.hop_size)
                         feats.append(feat)
 
                     feats = paddle.to_tensor(np.asarray(feats))
@@ -261,6 +285,7 @@ def main(args, config):
             # stage 9-14: Save model parameters
             save_dir = os.path.join(args.checkpoint_dir,
                                     'epoch_{}'.format(epoch))
+            last_saved_epoch = os.path.join('epoch_{}'.format(epoch), "model.pdparams")
             logger.info('Saving model checkpoint to {}'.format(save_dir))
             paddle.save(model.state_dict(),
                         os.path.join(save_dir, 'model.pdparams'))
@@ -270,6 +295,14 @@ def main(args, config):
             if nranks > 1:
                 paddle.distributed.barrier()  # Main process
 
+    # stage 10: create the final trained model.pdparams with soft link
+    if local_rank == 0:
+        final_model = os.path.join(args.checkpoint_dir, "model.pdparams")
+        logger.info(f"we will create the final model: {final_model}")
+        if os.path.islink(final_model):
+            logger.info(f"An {final_model} already exists, we will rm is and create it again")
+            os.unlink(final_model)
+        os.symlink(last_saved_epoch, final_model)
 
 if __name__ == "__main__":
     # yapf: disable
@@ -294,10 +327,6 @@ if __name__ == "__main__":
                         type=str,
                         default='./checkpoint',
                         help="Directory to save model checkpoints.")
-    parser.add_argument("--augment",
-                        action="store_true",
-                        default=False,
-                        help="Apply audio augments.")
 
     args = parser.parse_args()
     # yapf: enable
diff --git a/paddlespeech/vector/io/batch.py b/paddlespeech/vector/io/batch.py
index 85f2ab8b..25522ebb 100644
--- a/paddlespeech/vector/io/batch.py
+++ b/paddlespeech/vector/io/batch.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 import numpy as np
 import paddle
-
+import numpy
 
 def waveform_collate_fn(batch):
     waveforms = np.stack([item['feat'] for item in batch])
@@ -80,4 +80,92 @@ def batch_feature_normalize(batch, mean_norm: bool=True, std_norm: bool=True):
     # we convert the original length of each utterance to the ratio of the max length
     lengths = (lengths / lengths.max()).astype(np.float32)
 
-    return {'ids': ids, 'feats': feats, 'lengths': lengths}
\ No newline at end of file
+    return {'ids': ids, 'feats': feats, 'lengths': lengths}
+
+
+def pad_right_to(array, target_shape, mode="constant", value=0):
+    """
+    This function takes a numpy array of arbitrary shape and pads it to target
+    shape by appending values on the right.
+
+    Args:
+        array: input numpy array. Input array whose dimension we need to pad.
+    target_shape : (list, tuple). Target shape we want for the target array its len must be equal to array.ndim
+    mode : str. Pad mode, please refer to numpy.pad documentation.
+    value : float. Pad value, please refer to numpy.pad documentation.
+
+    Returns:
+        array: numpy.array. Padded array.
+        valid_vals : list. List containing proportion for each dimension of original, non-padded values.
+    """
+    assert len(target_shape) == array.ndim
+    pads = []  # this contains the abs length of the padding for each dimension.
+    valid_vals = []  # thic contains the relative lengths for each dimension.
+    i = 0 # iterating over target_shape ndims
+    while i < len(target_shape):
+        assert (
+            target_shape[i] >= array.shape[i]
+        ), "Target shape must be >= original shape for every dim"
+        pads.append([0, target_shape[i] - array.shape[i]])
+        valid_vals.append(array.shape[i] / target_shape[i])
+        i += 1
+
+    array = numpy.pad(array, pads, mode=mode, constant_values=value)
+
+    return array, valid_vals
+
+
+def batch_pad_right(arrays, mode="constant", value=0):
+    """Given a list of numpy arrays it batches them together by padding to the right
+    on each dimension in order to get same length for all.
+
+    Args:
+        arrays : list. List of array we wish to pad together.
+        mode : str. Padding mode see numpy.pad documentation.
+        value : float. Padding value see numpy.pad documentation.
+
+    Returns:
+        array : numpy.array. Padded array.
+        valid_vals : list. List containing proportion for each dimension of original, non-padded values.
+    """
+
+    if not len(arrays):
+        raise IndexError("arrays list must not be empty")
+
+    if len(arrays) == 1:
+        # if there is only one array in the batch we simply unsqueeze it.
+        return numpy.expand_dims(arrays[0], axis=0), numpy.array([1.0])
+
+    if not (
+        any(
+            [arrays[i].ndim == arrays[0].ndim for i in range(1, len(arrays))]
+        )
+    ):
+        raise IndexError("All arrays must have same number of dimensions")
+
+    # FIXME we limit the support here: we allow padding of only the last dimension
+    # need to remove this when feat extraction is updated to handle multichannel.
+    max_shape = []
+    for dim in range(arrays[0].ndim):
+        if dim != (arrays[0].ndim - 1):
+            if not all(
+                [x.shape[dim] == arrays[0].shape[dim] for x in arrays[1:]]
+            ):
+                raise EnvironmentError(
+                    "arrays should have same dimensions except for last one"
+                )
+        max_shape.append(max([x.shape[dim] for x in arrays]))
+
+    batched = []
+    valid = []
+    for t in arrays:
+        # for each array we apply pad_right_to
+        padded, valid_percent = pad_right_to(
+            t, max_shape, mode=mode, value=value
+        )
+        batched.append(padded)
+        valid.append(valid_percent[-1])
+
+    batched = numpy.stack(batched)
+
+    return batched, numpy.array(valid)
diff --git a/paddlespeech/vector/modules/loss.py b/paddlespeech/vector/modules/loss.py
index 1aa0599a..1c80dda4 100644
--- a/paddlespeech/vector/modules/loss.py
+++ b/paddlespeech/vector/modules/loss.py
@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# This is modified from SpeechBrain
+# https://github.com/speechbrain/speechbrain/blob/085be635c07f16d42cd1295045bc46c407f1e15b/speechbrain/nnet/losses.py
 import math
 
 import paddle
@@ -20,6 +22,14 @@ import paddle.nn.functional as F
 
 class AngularMargin(nn.Layer):
     def __init__(self, margin=0.0, scale=1.0):
+        """An implementation of Angular Margin (AM) proposed in the following
+           paper: '''Margin Matters: Towards More Discriminative Deep Neural Network
+           Embeddings for Speaker Recognition''' (https://arxiv.org/abs/1906.07317)
+
+        Args:
+            margin (float, optional): The margin for cosine similiarity. Defaults to 0.0.
+            scale (float, optional): The scale for cosine similiarity. Defaults to 1.0.
+        """
         super(AngularMargin, self).__init__()
         self.margin = margin
         self.scale = scale
@@ -31,6 +41,15 @@ class AngularMargin(nn.Layer):
 
 class AdditiveAngularMargin(AngularMargin):
     def __init__(self, margin=0.0, scale=1.0, easy_margin=False):
+        """The Implementation of Additive Angular Margin (AAM) proposed
+       in the following paper: '''Margin Matters: Towards More Discriminative Deep Neural Network Embeddings for Speaker Recognition'''
+       (https://arxiv.org/abs/1906.07317)
+
+        Args:
+            margin (float, optional): margin factor. Defaults to 0.0.
+            scale (float, optional): scale factor. Defaults to 1.0.
+            easy_margin (bool, optional): easy_margin flag. Defaults to False.
+        """
         super(AdditiveAngularMargin, self).__init__(margin, scale)
         self.easy_margin = easy_margin
 
@@ -53,6 +72,11 @@ class AdditiveAngularMargin(AngularMargin):
 
 class LogSoftmaxWrapper(nn.Layer):
     def __init__(self, loss_fn):
+        """Speaker identificatin loss function wrapper 
+           including all of compositions of the loss transformation
+        Args:
+            loss_fn (_type_): the loss value of a batch
+        """
         super(LogSoftmaxWrapper, self).__init__()
         self.loss_fn = loss_fn
         self.criterion = paddle.nn.KLDivLoss(reduction="sum")
diff --git a/paddlespeech/vector/modules/sid_model.py b/paddlespeech/vector/modules/sid_model.py
index 8a46c3cd..dc13b2e0 100644
--- a/paddlespeech/vector/modules/sid_model.py
+++ b/paddlespeech/vector/modules/sid_model.py
@@ -24,13 +24,25 @@ class SpeakerIdetification(nn.Layer):
             lin_blocks=0,
             lin_neurons=192,
             dropout=0.1, ):
+        """_summary_
 
+        Args:
+            backbone (Paddle.nn.Layer class): the speaker identification backbone network model
+            num_class (_type_): the speaker class num in the training dataset
+            lin_blocks (int, optional): the linear layer transform between the embedding and the final linear layer. Defaults to 0.
+            lin_neurons (int, optional): the output dimension of final linear layer. Defaults to 192.
+            dropout (float, optional): the dropout factor on the embedding. Defaults to 0.1.
+        """
         super(SpeakerIdetification, self).__init__()
+        # speaker idenfication backbone network model
+        # the output of the backbond network is the target embedding
         self.backbone = backbone
         if dropout > 0:
             self.dropout = nn.Dropout(dropout)
         else:
             self.dropout = None
+        
+        # construct the speaker classifer
         input_size = self.backbone.emb_size
         self.blocks = nn.LayerList()
         for i in range(lin_blocks):
@@ -40,12 +52,26 @@ class SpeakerIdetification(nn.Layer):
             ])
             input_size = lin_neurons
 
+        # the final layer
         self.weight = paddle.create_parameter(
             shape=(input_size, num_class),
             dtype='float32',
             attr=paddle.ParamAttr(initializer=nn.initializer.XavierUniform()), )
 
     def forward(self, x, lengths=None):
+        """Do the speaker identification model forwrd, 
+           including the speaker embedding model and the classifier model network
+
+        Args:
+            x (Paddle.Tensor): input audio feats, 
+                               shape=[batch, dimension, times]
+            lengths (_type_, optional): input audio length.
+                                        shape=[batch, times]
+                                        Defaults to None.
+
+        Returns:
+            _type_: _description_
+        """
         # x.shape: (N, C, L)
         x = self.backbone(x, lengths).squeeze(
             -1)  # (N, emb_size, 1) -> (N, emb_size)

From 9c6735f921d767f116e47f1671ec6290d72315dd Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 21 Mar 2022 17:40:40 +0800
Subject: [PATCH 032/126] add vector voxceleb12 base mode url, test=doc

---
 paddlespeech/cli/vector/infer.py | 33 ++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index de4d6621..205d61f9 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -42,19 +42,19 @@ from paddlespeech.s2t.utils.utility import UpdateConfig
 from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 
 pretrained_models = {
-    # The tags for pretrained_models should be "{model_name}[_{dataset}][-{lang}][-...]".
-    # e.g. "conformer_wenetspeech-zh-16k" and "panns_cnn6-32k".
-    # Command line and python api use "{model_name}[_{dataset}]" as --model, usage:
-    # "paddlespeech asr --model conformer_wenetspeech --lang zh --sr 16000 --input ./input.wav"
-    "ecapa_tdnn-16k": {
+    # The tags for pretrained_models should be "{model_name}[-{dataset}][-{sr}][-...]".
+    # e.g. "ecapa_tdnn-voxceleb12-16k".
+    # Command line and python api use "{model_name}[-{dataset}]" as --model, usage:
+    # "paddlespeech vector --task spk --model ecapa_tdnn-voxceleb12-16k --sr 16000 --input ./input.wav"
+    "ecapa_tdnn-voxceleb12-16k": {
         'url':
-        'https://paddlespeech.bj.bcebos.com/s2t/wenetspeech/asr1_conformer_wenetspeech_ckpt_0.1.1.model.tar.gz',
+        'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz',
         'md5':
-        '76cb19ed857e6623856b7cd7ebbfeda4',
+        '85ff08ce0ef406b8c6d7b5ffc5b2b48f',
         'cfg_path':
-        'model.yaml',
+        'conf/model.yaml',
         'ckpt_path':
-        'exp/conformer/checkpoints/wenetspeech',
+        'model/model',
     },
 }
 
@@ -202,6 +202,14 @@ class VectorExecutor(BaseExecutor):
             The support models includes \n\t\t{}'.format(tag, "\n\t\t".join(support_models))
 
         res_path = os.path.join(MODEL_HOME, tag)
+        decompressed_path = download_and_decompress(pretrained_models[tag],
+                                                    res_path)
+
+        decompressed_path = os.path.abspath(decompressed_path)
+        logger.info(
+            'Use pretrained model stored in: {}'.format(decompressed_path))
+
+        return decompressed_path
 
     def _init_from_path(self,
                         model_type: str='ecapa_tdnn-voxceleb12',
@@ -216,7 +224,12 @@ class VectorExecutor(BaseExecutor):
         if cfg_path is None or ckpt_path is None:
             sample_rate_str = "16k" if sample_rate == 16000 else "8k"
             tag = model_type + "-" + sample_rate_str
+            logger.info(f"load the pretrained model: {tag}")
             res_path = self._get_pretrained_path(tag)
+            self.res_path = res_path
+
+            self.cfg_path = os.path.join(res_path, pretrained_models[tag]['cfg_path'])
+            self.ckpt_path = os.path.join(res_path, pretrained_models[tag]['ckpt_path'] + '.pdparams')
         else:
             self.cfg_path = os.path.abspath(cfg_path)
             self.ckpt_path = os.path.abspath(ckpt_path + ".pdparams")
@@ -226,7 +239,7 @@ class VectorExecutor(BaseExecutor):
         logger.info(f"start to read the ckpt from {self.ckpt_path}")
         logger.info(f"read the config from {self.cfg_path}")
         logger.info(f"get the res path {self.res_path}")
-
+        
         # stage 2: read and config and init the model body
         self.config = CfgNode(new_allowed=True)
         self.config.merge_from_file(self.cfg_path)

From b9eafddd9494f6f62fbbffbd149b08e4cc36dccf Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 21 Mar 2022 17:49:39 +0800
Subject: [PATCH 033/126] change - to _ to distinguish field

---
 paddlespeech/cli/vector/infer.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index 205d61f9..c942c850 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -43,10 +43,10 @@ from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 
 pretrained_models = {
     # The tags for pretrained_models should be "{model_name}[-{dataset}][-{sr}][-...]".
-    # e.g. "ecapa_tdnn-voxceleb12-16k".
+    # e.g. "EcapaTdnn_voxceleb12-16k".
     # Command line and python api use "{model_name}[-{dataset}]" as --model, usage:
-    # "paddlespeech vector --task spk --model ecapa_tdnn-voxceleb12-16k --sr 16000 --input ./input.wav"
-    "ecapa_tdnn-voxceleb12-16k": {
+    # "paddlespeech vector --task spk --model EcapaTdnn_voxceleb12-voxceleb12-16k --sr 16000 --input ./input.wav"
+    "EcapaTdnn_voxceleb12-16k": {
         'url':
         'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz',
         'md5':
@@ -59,7 +59,7 @@ pretrained_models = {
 }
 
 model_alias = {
-    "ecapa_tdnn": "paddlespeech.vector.models.ecapa_tdnn:EcapaTdnn",
+    "EcapaTdnn": "paddlespeech.vector.models.ecapa_tdnn:EcapaTdnn",
 }
 
 
@@ -75,8 +75,8 @@ class VectorExecutor(BaseExecutor):
         self.parser.add_argument(
             "--model",
             type=str,
-            default="ecapa_tdnn-voxceleb12",
-            choices=["ecapa_tdnn"],
+            default="EcapaTdnn_voxceleb12",
+            choices=["EcapaTdnn_voxceleb12"],
             help="Choose model type of asr task.")
         self.parser.add_argument(
             "--task",
@@ -175,7 +175,7 @@ class VectorExecutor(BaseExecutor):
     @stats_wrapper
     def __call__(self,
                  audio_file: os.PathLike,
-                 model: str='ecapa_tdnn-voxceleb12',
+                 model: str='EcapaTdnn-voxceleb12',
                  sample_rate: int=16000,
                  config: os.PathLike=None,
                  ckpt_path: os.PathLike=None,
@@ -212,7 +212,7 @@ class VectorExecutor(BaseExecutor):
         return decompressed_path
 
     def _init_from_path(self,
-                        model_type: str='ecapa_tdnn-voxceleb12',
+                        model_type: str='EcapaTdnn_voxceleb12',
                         sample_rate: int=16000,
                         cfg_path: Optional[os.PathLike]=None,
                         ckpt_path: Optional[os.PathLike]=None):
@@ -247,7 +247,7 @@ class VectorExecutor(BaseExecutor):
         # stage 3: get the model name to instance the model network with dynamic_import
         # Noet: we use the '-' to get the model name instead of '_'
         logger.info("start to dynamic import the model class")
-        model_name = model_type[:model_type.rindex('-')]
+        model_name = model_type[:model_type.rindex('_')]
         logger.info(f"model name {model_name}")
         model_class = dynamic_import(model_name, model_alias)
         model_conf = self.config.model

From d53e1163a60ff992d4ee4790e4cc3f02793e0c7c Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Tue, 22 Mar 2022 05:12:21 +0000
Subject: [PATCH 034/126] update the code, test=asr

---
 paddlespeech/s2t/__init__.py                  |   6 -
 paddlespeech/s2t/exps/u2/model.py             |   4 +-
 paddlespeech/s2t/models/u2/u2.py              |  10 +-
 paddlespeech/s2t/modules/activation.py        |  13 +-
 paddlespeech/s2t/modules/align.py             |  74 +++++++
 paddlespeech/s2t/modules/attention.py         |  11 +-
 .../s2t/modules/conformer_convolution.py      |  25 +--
 paddlespeech/s2t/modules/ctc.py               |   3 +-
 paddlespeech/s2t/modules/decoder.py           |  19 +-
 paddlespeech/s2t/modules/decoder_layer.py     |  30 +--
 paddlespeech/s2t/modules/encoder.py           |  10 +-
 paddlespeech/s2t/modules/encoder_layer.py     |  52 ++---
 paddlespeech/s2t/modules/initializer.py       | 185 +++++-------------
 paddlespeech/s2t/modules/nets_utils.py        |  44 -----
 .../s2t/modules/positionwise_feed_forward.py  |   5 +-
 paddlespeech/s2t/modules/subsampling.py       |  29 +--
 16 files changed, 196 insertions(+), 324 deletions(-)
 create mode 100644 paddlespeech/s2t/modules/align.py
 delete mode 100644 paddlespeech/s2t/modules/nets_utils.py

diff --git a/paddlespeech/s2t/__init__.py b/paddlespeech/s2t/__init__.py
index 26f59dfb..855ceef9 100644
--- a/paddlespeech/s2t/__init__.py
+++ b/paddlespeech/s2t/__init__.py
@@ -21,7 +21,6 @@ from paddle import nn
 from paddle.fluid import core
 from paddle.nn import functional as F
 
-from paddlespeech.s2t.modules import initializer
 from paddlespeech.s2t.utils.log import Log
 
 #TODO(Hui Zhang): remove  fluid import
@@ -506,8 +505,3 @@ if not hasattr(paddle.nn, 'LayerDict'):
     logger.debug(
         "register user LayerDict to paddle.nn, remove this when fixed!")
     setattr(paddle.nn, 'LayerDict', LayerDict)
-
-"""
-    hack KaiminigUniform: change limit from np.sqrt(6.0 / float(fan_in)) to np.sqrt(1.0 / float(fan_in))
-"""
-paddle.nn.initializer.KaimingUniform = initializer.KaimingUniform
diff --git a/paddlespeech/s2t/exps/u2/model.py b/paddlespeech/s2t/exps/u2/model.py
index d7bee6d7..bcbc15d6 100644
--- a/paddlespeech/s2t/exps/u2/model.py
+++ b/paddlespeech/s2t/exps/u2/model.py
@@ -239,7 +239,7 @@ class U2Trainer(Trainer):
                 n_iter_processes=config.num_workers,
                 subsampling_factor=1,
                 num_encs=1,
-                dist_sampler=False,
+                dist_sampler=True,
                 shortest_first=False)
 
             self.valid_loader = BatchDataLoader(
@@ -260,7 +260,7 @@ class U2Trainer(Trainer):
                 n_iter_processes=config.num_workers,
                 subsampling_factor=1,
                 num_encs=1,
-                dist_sampler=False,
+                dist_sampler=True,
                 shortest_first=False)
             logger.info("Setup train/valid Dataloader!")
         else:
diff --git a/paddlespeech/s2t/models/u2/u2.py b/paddlespeech/s2t/models/u2/u2.py
index 67ec5924..e077cd5b 100644
--- a/paddlespeech/s2t/models/u2/u2.py
+++ b/paddlespeech/s2t/models/u2/u2.py
@@ -41,7 +41,6 @@ from paddlespeech.s2t.modules.mask import make_pad_mask
 from paddlespeech.s2t.modules.mask import mask_finished_preds
 from paddlespeech.s2t.modules.mask import mask_finished_scores
 from paddlespeech.s2t.modules.mask import subsequent_mask
-from paddlespeech.s2t.modules.nets_utils import initialize
 from paddlespeech.s2t.utils import checkpoint
 from paddlespeech.s2t.utils import layer_tools
 from paddlespeech.s2t.utils.ctc_utils import remove_duplicates_and_blank
@@ -51,6 +50,8 @@ from paddlespeech.s2t.utils.tensor_utils import pad_sequence
 from paddlespeech.s2t.utils.tensor_utils import th_accuracy
 from paddlespeech.s2t.utils.utility import log_add
 from paddlespeech.s2t.utils.utility import UpdateConfig
+from paddlespeech.s2t.modules.initializer import DefaultInitializerContext
+# from paddlespeech.s2t.modules.initializer import initialize
 
 __all__ = ["U2Model", "U2InferModel"]
 
@@ -784,11 +785,8 @@ class U2Model(U2DecodeModel):
     def __init__(self, configs: dict):
         model_conf = configs.get('model_conf', dict())
         init_type = model_conf.get("init_type", None)
-        if init_type is not None:
-            logger.info(f"Use {init_type} initializer as default initializer")
-        initialize(self, init_type)
-        vocab_size, encoder, decoder, ctc = U2Model._init_from_config(configs)
-        nn.initializer.set_global_initializer(None)
+        with DefaultInitializerContext(init_type):
+            vocab_size, encoder, decoder, ctc = U2Model._init_from_config(configs)
 
         super().__init__(
             vocab_size=vocab_size,
diff --git a/paddlespeech/s2t/modules/activation.py b/paddlespeech/s2t/modules/activation.py
index 4081f7f8..48c84fa6 100644
--- a/paddlespeech/s2t/modules/activation.py
+++ b/paddlespeech/s2t/modules/activation.py
@@ -16,7 +16,8 @@ from collections import OrderedDict
 import paddle
 from paddle import nn
 from paddle.nn import functional as F
-
+from paddlespeech.s2t.modules.align import Linear
+from paddlespeech.s2t.modules.align import Conv2D
 from paddlespeech.s2t.utils.log import Log
 
 logger = Log(__name__).getlog()
@@ -51,7 +52,7 @@ class LinearGLUBlock(nn.Layer):
             idim (int): input and output dimension
         """
         super().__init__()
-        self.fc = nn.Linear(idim, idim * 2)
+        self.fc = Linear(idim, idim * 2)
 
     def forward(self, xs):
         return glu(self.fc(xs), dim=-1)
@@ -75,7 +76,7 @@ class ConvGLUBlock(nn.Layer):
         self.conv_residual = None
         if in_ch != out_ch:
             self.conv_residual = nn.utils.weight_norm(
-                nn.Conv2D(
+                Conv2D(
                     in_channels=in_ch, out_channels=out_ch, kernel_size=(1, 1)),
                 name='weight',
                 dim=0)
@@ -86,7 +87,7 @@ class ConvGLUBlock(nn.Layer):
         layers = OrderedDict()
         if bottlececk_dim == 0:
             layers['conv'] = nn.utils.weight_norm(
-                nn.Conv2D(
+                Conv2D(
                     in_channels=in_ch,
                     out_channels=out_ch * 2,
                     kernel_size=(kernel_size, 1)),
@@ -106,7 +107,7 @@ class ConvGLUBlock(nn.Layer):
                 dim=0)
             layers['dropout_in'] = nn.Dropout(p=dropout)
             layers['conv_bottleneck'] = nn.utils.weight_norm(
-                nn.Conv2D(
+                Conv2D(
                     in_channels=bottlececk_dim,
                     out_channels=bottlececk_dim,
                     kernel_size=(kernel_size, 1)),
@@ -115,7 +116,7 @@ class ConvGLUBlock(nn.Layer):
             layers['dropout'] = nn.Dropout(p=dropout)
             layers['glu'] = GLU()
             layers['conv_out'] = nn.utils.weight_norm(
-                nn.Conv2D(
+                Conv2D(
                     in_channels=bottlececk_dim,
                     out_channels=out_ch * 2,
                     kernel_size=(1, 1)),
diff --git a/paddlespeech/s2t/modules/align.py b/paddlespeech/s2t/modules/align.py
new file mode 100644
index 00000000..575773d7
--- /dev/null
+++ b/paddlespeech/s2t/modules/align.py
@@ -0,0 +1,74 @@
+import paddle
+from paddle import nn
+from paddlespeech.s2t.modules.initializer import KaimingUniform
+
+"""
+    To align the initializer between paddle and torch, 
+    the API below are set defalut initializer with priority higger than global initializer.
+"""
+global_init_type = None
+
+
+class LayerNorm(nn.LayerNorm):
+    def __init__(self, normalized_shape, epsilon=1e-05, weight_attr=None, bias_attr=None, name=None):
+        if weight_attr is None:
+            weight_attr = paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0))
+        if bias_attr is None:
+            bias_attr = paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0))
+        super(LayerNorm, self).__init__(normalized_shape, epsilon, weight_attr, bias_attr, name)
+
+class BatchNorm1D(nn.BatchNorm1D): 
+    def __init__(self, num_features, momentum=0.9, epsilon=1e-05, weight_attr=None, bias_attr=None, data_format='NCL', name=None):
+        if weight_attr is None:
+            weight_attr = paddle.ParamAttr(
+                initializer=nn.initializer.Constant(1.0))
+        if bias_attr is None:
+            bias_attr = paddle.ParamAttr(
+                initializer=nn.initializer.Constant(0.0))
+        super(BatchNorm1D, self).__init__(num_features, momentum, epsilon, weight_attr, bias_attr, data_format, name)
+
+class Embedding(nn.Embedding):
+    def __init__(self, num_embeddings, embedding_dim, padding_idx=None, sparse=False, weight_attr=None, name=None):
+        if weight_attr is None:
+            weight_attr = paddle.ParamAttr(
+                initializer=nn.initializer.Normal())
+        super(Embedding, self).__init__(num_embeddings, embedding_dim, padding_idx, sparse, weight_attr, name)
+
+class Linear(nn.Linear):
+    def __init__(self, in_features, out_features, weight_attr=None, bias_attr=None, name=None):
+       if weight_attr is None:
+           if global_init_type == "kaiming_uniform":
+               weight_attr = paddle.ParamAttr(
+                   initializer=KaimingUniform())
+       if bias_attr is None:
+           if global_init_type == "kaiming_uniform":
+               bias_attr = paddle.ParamAttr(
+                   initializer=KaimingUniform())
+        super(Linear, self).__init__(in_features, out_features, weight_attr, bias_attr, name)
+
+class Conv1D(nn.Conv1D):
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, padding_mode='zeros', weight_attr=None, bias_attr=None, data_format='NCL'):
+        if weight_attr is None:
+            if global_init_type == "kaiming_uniform":
+                print("set kaiming_uniform")
+                weight_attr = paddle.ParamAttr(
+                    initializer=KaimingUniform())
+        if bias_attr is None:
+            if global_init_type == "kaiming_uniform":
+                bias_attr = paddle.ParamAttr(
+                    initializer=KaimingUniform())
+        super(Conv1D, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, padding_mode, weight_attr, bias_attr, data_format)
+    
+class Conv2D(nn.Conv2D):
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, padding_mode='zeros', weight_attr=None, bias_attr=None, data_format='NCHW'):
+       if weight_attr is None:
+           if global_init_type == "kaiming_uniform":
+               weight_attr = paddle.ParamAttr(
+                   initializer=KaimingUniform())
+       if bias_attr is None:
+           if global_init_type == "kaiming_uniform":
+               bias_attr = paddle.ParamAttr(
+                   initializer=KaimingUniform())
+        super(Conv2D, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, padding_mode, weight_attr, bias_attr, data_format)
diff --git a/paddlespeech/s2t/modules/attention.py b/paddlespeech/s2t/modules/attention.py
index c2f5e503..438efd2a 100644
--- a/paddlespeech/s2t/modules/attention.py
+++ b/paddlespeech/s2t/modules/attention.py
@@ -22,6 +22,7 @@ import paddle
 from paddle import nn
 from paddle.nn import initializer as I
 
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.utils.log import Log
 
 logger = Log(__name__).getlog()
@@ -48,10 +49,10 @@ class MultiHeadedAttention(nn.Layer):
         # We assume d_v always equals d_k
         self.d_k = n_feat // n_head
         self.h = n_head
-        self.linear_q = nn.Linear(n_feat, n_feat)
-        self.linear_k = nn.Linear(n_feat, n_feat)
-        self.linear_v = nn.Linear(n_feat, n_feat)
-        self.linear_out = nn.Linear(n_feat, n_feat)
+        self.linear_q = Linear(n_feat, n_feat)
+        self.linear_k = Linear(n_feat, n_feat)
+        self.linear_v = Linear(n_feat, n_feat)
+        self.linear_out = Linear(n_feat, n_feat)
         self.dropout = nn.Dropout(p=dropout_rate)
 
     def forward_qkv(self,
@@ -150,7 +151,7 @@ class RelPositionMultiHeadedAttention(MultiHeadedAttention):
         """
         super().__init__(n_head, n_feat, dropout_rate)
         # linear transformation for positional encoding
-        self.linear_pos = nn.Linear(n_feat, n_feat, bias_attr=False)
+        self.linear_pos = Linear(n_feat, n_feat, bias_attr=False)
         # these two learnable bias are used in matrix c and matrix d
         # as described in https://arxiv.org/abs/1901.02860 Section 3.3
         #self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k))
diff --git a/paddlespeech/s2t/modules/conformer_convolution.py b/paddlespeech/s2t/modules/conformer_convolution.py
index 256d187c..89e65268 100644
--- a/paddlespeech/s2t/modules/conformer_convolution.py
+++ b/paddlespeech/s2t/modules/conformer_convolution.py
@@ -21,6 +21,9 @@ import paddle
 from paddle import nn
 from typeguard import check_argument_types
 
+from paddlespeech.s2t.modules.align import BatchNorm1D
+from paddlespeech.s2t.modules.align import Conv1D
+from paddlespeech.s2t.modules.align import LayerNorm
 from paddlespeech.s2t.utils.log import Log
 
 logger = Log(__name__).getlog()
@@ -49,7 +52,7 @@ class ConvolutionModule(nn.Layer):
         """
         assert check_argument_types()
         super().__init__()
-        self.pointwise_conv1 = nn.Conv1D(
+        self.pointwise_conv1 = Conv1D(
             channels,
             2 * channels,
             kernel_size=1,
@@ -73,7 +76,7 @@ class ConvolutionModule(nn.Layer):
             padding = (kernel_size - 1) // 2
             self.lorder = 0
 
-        self.depthwise_conv = nn.Conv1D(
+        self.depthwise_conv = Conv1D(
             channels,
             channels,
             kernel_size,
@@ -87,22 +90,12 @@ class ConvolutionModule(nn.Layer):
         assert norm in ['batch_norm', 'layer_norm']
         if norm == "batch_norm":
             self.use_layer_norm = False
-            self.norm = nn.BatchNorm1D(
-                channels,
-                weight_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(1.0)),
-                bias_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(0.0)))
+            self.norm = BatchNorm1D(channels)
         else:
             self.use_layer_norm = True
-            self.norm = nn.LayerNorm(
-                channels,
-                weight_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(1.0)),
-                bias_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(0.0)))
-
-        self.pointwise_conv2 = nn.Conv1D(
+            self.norm = LayerNorm(channels)
+
+        self.pointwise_conv2 = Conv1D(
             channels,
             channels,
             kernel_size=1,
diff --git a/paddlespeech/s2t/modules/ctc.py b/paddlespeech/s2t/modules/ctc.py
index 2094182a..33ad472d 100644
--- a/paddlespeech/s2t/modules/ctc.py
+++ b/paddlespeech/s2t/modules/ctc.py
@@ -18,6 +18,7 @@ from paddle import nn
 from paddle.nn import functional as F
 from typeguard import check_argument_types
 
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.modules.loss import CTCLoss
 from paddlespeech.s2t.utils import ctc_utils
 from paddlespeech.s2t.utils.log import Log
@@ -69,7 +70,7 @@ class CTCDecoderBase(nn.Layer):
         self.blank_id = blank_id
         self.odim = odim
         self.dropout = nn.Dropout(dropout_rate)
-        self.ctc_lo = nn.Linear(enc_n_units, self.odim)
+        self.ctc_lo = Linear(enc_n_units, self.odim)
         reduction_type = "sum" if reduction else "none"
         self.criterion = CTCLoss(
             blank=self.blank_id,
diff --git a/paddlespeech/s2t/modules/decoder.py b/paddlespeech/s2t/modules/decoder.py
index b0ae27e5..3a851ec6 100644
--- a/paddlespeech/s2t/modules/decoder.py
+++ b/paddlespeech/s2t/modules/decoder.py
@@ -24,6 +24,9 @@ from paddle import nn
 from typeguard import check_argument_types
 
 from paddlespeech.s2t.decoders.scorers.scorer_interface import BatchScorerInterface
+from paddlespeech.s2t.modules.align import Embedding
+from paddlespeech.s2t.modules.align import LayerNorm
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.modules.attention import MultiHeadedAttention
 from paddlespeech.s2t.modules.decoder_layer import DecoderLayer
 from paddlespeech.s2t.modules.embedding import PositionalEncoding
@@ -83,25 +86,15 @@ class TransformerDecoder(BatchScorerInterface, nn.Layer):
 
         if input_layer == "embed":
             self.embed = nn.Sequential(
-                nn.Embedding(
-                    vocab_size,
-                    attention_dim,
-                    weight_attr=paddle.ParamAttr(
-                        initializer=nn.initializer.Normal())),
+                Embedding(vocab_size, attention_dim),
                 PositionalEncoding(attention_dim, positional_dropout_rate), )
         else:
             raise ValueError(f"only 'embed' is supported: {input_layer}")
 
         self.normalize_before = normalize_before
-        self.after_norm = nn.LayerNorm(
-            attention_dim,
-            epsilon=1e-12,
-            weight_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(1.0)),
-            bias_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(0.0)))
+        self.after_norm = LayerNorm(attention_dim, epsilon=1e-12)
         self.use_output_layer = use_output_layer
-        self.output_layer = nn.Linear(attention_dim, vocab_size)
+        self.output_layer = Linear(attention_dim, vocab_size)
 
         self.decoders = nn.LayerList([
             DecoderLayer(
diff --git a/paddlespeech/s2t/modules/decoder_layer.py b/paddlespeech/s2t/modules/decoder_layer.py
index 8eee5ceb..b7f8694c 100644
--- a/paddlespeech/s2t/modules/decoder_layer.py
+++ b/paddlespeech/s2t/modules/decoder_layer.py
@@ -20,6 +20,8 @@ from typing import Tuple
 import paddle
 from paddle import nn
 
+from paddlespeech.s2t.modules.align import LayerNorm
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.utils.log import Log
 
 logger = Log(__name__).getlog()
@@ -62,32 +64,14 @@ class DecoderLayer(nn.Layer):
         self.self_attn = self_attn
         self.src_attn = src_attn
         self.feed_forward = feed_forward
-        self.norm1 = nn.LayerNorm(
-            size,
-            epsilon=1e-12,
-            weight_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(1.0)),
-            bias_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(0.0)))
-        self.norm2 = nn.LayerNorm(
-            size,
-            epsilon=1e-12,
-            weight_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(1.0)),
-            bias_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(0.0)))
-        self.norm3 = nn.LayerNorm(
-            size,
-            epsilon=1e-12,
-            weight_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(1.0)),
-            bias_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(0.0)))
+        self.norm1 = LayerNorm(size, epsilon=1e-12)
+        self.norm2 = LayerNorm(size, epsilon=1e-12)
+        self.norm3 = LayerNorm(size, epsilon=1e-12)
         self.dropout = nn.Dropout(dropout_rate)
         self.normalize_before = normalize_before
         self.concat_after = concat_after
-        self.concat_linear1 = nn.Linear(size + size, size)
-        self.concat_linear2 = nn.Linear(size + size, size)
+        self.concat_linear1 = Linear(size + size, size)
+        self.concat_linear2 = Linear(size + size, size)
 
     def forward(
             self,
diff --git a/paddlespeech/s2t/modules/encoder.py b/paddlespeech/s2t/modules/encoder.py
index 5f7b8e99..71a2bad4 100644
--- a/paddlespeech/s2t/modules/encoder.py
+++ b/paddlespeech/s2t/modules/encoder.py
@@ -23,6 +23,8 @@ from paddle import nn
 from typeguard import check_argument_types
 
 from paddlespeech.s2t.modules.activation import get_activation
+from paddlespeech.s2t.modules.align import LayerNorm
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.modules.attention import MultiHeadedAttention
 from paddlespeech.s2t.modules.attention import RelPositionMultiHeadedAttention
 from paddlespeech.s2t.modules.conformer_convolution import ConvolutionModule
@@ -129,13 +131,7 @@ class BaseEncoder(nn.Layer):
                 d_model=output_size, dropout_rate=positional_dropout_rate), )
 
         self.normalize_before = normalize_before
-        self.after_norm = nn.LayerNorm(
-            output_size,
-            epsilon=1e-12,
-            weight_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(1.0)),
-            bias_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(0.0)))
+        self.after_norm = LayerNorm(output_size, epsilon=1e-12)
         self.static_chunk_size = static_chunk_size
         self.use_dynamic_chunk = use_dynamic_chunk
         self.use_dynamic_left_chunk = use_dynamic_left_chunk
diff --git a/paddlespeech/s2t/modules/encoder_layer.py b/paddlespeech/s2t/modules/encoder_layer.py
index 69a3f67b..e80a298d 100644
--- a/paddlespeech/s2t/modules/encoder_layer.py
+++ b/paddlespeech/s2t/modules/encoder_layer.py
@@ -20,6 +20,8 @@ from typing import Tuple
 import paddle
 from paddle import nn
 
+from paddlespeech.s2t.modules.align import LayerNorm
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.utils.log import Log
 
 logger = Log(__name__).getlog()
@@ -59,15 +61,15 @@ class TransformerEncoderLayer(nn.Layer):
         super().__init__()
         self.self_attn = self_attn
         self.feed_forward = feed_forward
-        self.norm1 = nn.LayerNorm(size, epsilon=1e-12)
-        self.norm2 = nn.LayerNorm(size, epsilon=1e-12)
+        self.norm1 = LayerNorm(size, epsilon=1e-12)
+        self.norm2 = LayerNorm(size, epsilon=1e-12)
         self.dropout = nn.Dropout(dropout_rate)
         self.size = size
         self.normalize_before = normalize_before
         self.concat_after = concat_after
         # concat_linear may be not used in forward fuction,
         # but will be saved in the *.pt
-        self.concat_linear = nn.Linear(size + size, size)
+        self.concat_linear = Linear(size + size, size)
 
     def forward(
             self,
@@ -174,51 +176,23 @@ class ConformerEncoderLayer(nn.Layer):
         self.feed_forward = feed_forward
         self.feed_forward_macaron = feed_forward_macaron
         self.conv_module = conv_module
-        self.norm_ff = nn.LayerNorm(
-            size,
-            epsilon=1e-12,
-            weight_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(1.0)),
-            bias_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(0.0)))  # for the FNN module
-        self.norm_mha = nn.LayerNorm(
-            size,
-            epsilon=1e-12,
-            weight_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(1.0)),
-            bias_attr=paddle.ParamAttr(
-                initializer=nn.initializer.Constant(0.0)))  # for the MHA module
+        self.norm_ff = LayerNorm(size, epsilon=1e-12)  # for the FNN module
+        self.norm_mha = LayerNorm(size, epsilon=1e-12)  # for the MHA module
         if feed_forward_macaron is not None:
-            self.norm_ff_macaron = nn.LayerNorm(
-                size,
-                epsilon=1e-12,
-                weight_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(1.0)),
-                bias_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(0.0)))
+            self.norm_ff_macaron = LayerNorm(size, epsilon=1e-12)
             self.ff_scale = 0.5
         else:
             self.ff_scale = 1.0
         if self.conv_module is not None:
-            self.norm_conv = nn.LayerNorm(
-                size,
-                epsilon=1e-12,
-                weight_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(1.0)),
-                bias_attr=paddle.ParamAttr(initializer=nn.initializer.Constant(
-                    0.0)))  # for the CNN module
-            self.norm_final = nn.LayerNorm(
-                size,
-                epsilon=1e-12,
-                weight_attr=paddle.ParamAttr(
-                    initializer=nn.initializer.Constant(1.0)),
-                bias_attr=paddle.ParamAttr(initializer=nn.initializer.Constant(
-                    0.0)))  # for the final output of the block
+            self.norm_conv = LayerNorm(
+                size, epsilon=1e-12)  # for the CNN module
+            self.norm_final = LayerNorm(
+                size, epsilon=1e-12)  # for the final output of the block
         self.dropout = nn.Dropout(dropout_rate)
         self.size = size
         self.normalize_before = normalize_before
         self.concat_after = concat_after
-        self.concat_linear = nn.Linear(size + size, size)
+        self.concat_linear = Linear(size + size, size)
 
     def forward(
             self,
diff --git a/paddlespeech/s2t/modules/initializer.py b/paddlespeech/s2t/modules/initializer.py
index c91ab231..3fbab285 100644
--- a/paddlespeech/s2t/modules/initializer.py
+++ b/paddlespeech/s2t/modules/initializer.py
@@ -11,93 +11,35 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from __future__ import print_function
-
-from paddle.fluid import framework
-from paddle.fluid.framework import in_dygraph_mode, default_main_program
 import numpy as np
-from paddle.fluid.core import VarDesc
+from paddle import nn
+from paddle.fluid import framework
 from paddle.fluid import unique_name
+from paddle.fluid.core import VarDesc
+from paddle.fluid.framework import default_main_program
+from paddle.fluid.framework import in_dygraph_mode
+from paddle.fluid.initializer import Initializer
+from paddle.fluid.initializer import MSRAInitializer
+from typeguard import check_argument_types
 
-__all__ = [
-    'MSRAInitializer'
-]
-
-
-class Initializer(object):
-    """Base class for variable initializers
-
-    Defines the common interface of variable initializers.
-    They add operations to the init program that are used
-    to initialize variables. Users should not use this class
-    directly, but need to use one of its implementations.
-    """
-
-    def __init__(self):
-        pass
-
-    def __call__(self, param, block=None):
-        """Add corresponding initialization operations to the network
-        """
-        raise NotImplementedError()
-
-    def _check_block(self, block):
-        if block is None:
-            block = default_main_program().global_block()
-
-        return block
-
-    def _compute_fans(self, var):
-        """Compute the fan_in and the fan_out for layers
-
-        This method computes the fan_in and the fan_out
-        for neural network layers, if not specified. It is
-        not possible to perfectly estimate fan_in and fan_out.
-        This method will estimate it correctly for matrix multiply and
-        convolutions.
-
-        Args:
-            var: variable for which fan_in and fan_out have to be computed
-
-        Returns:
-            tuple of two integers (fan_in, fan_out)
-        """
-        shape = var.shape
-        if not shape or len(shape) == 0:
-            fan_in = fan_out = 1
-        elif len(shape) == 1:
-            fan_in = fan_out = shape[0]
-        elif len(shape) == 2:
-            # This is the case for simple matrix multiply
-            fan_in = shape[0]
-            fan_out = shape[1]
-        else:
-            # Assume this to be a convolutional kernel
-            # In PaddlePaddle, the shape of the kernel is like:
-            # [num_filters, num_filter_channels, ...] where the remaining
-            # dimensions are the filter_size
-            receptive_field_size = np.prod(shape[2:])
-            fan_in = shape[1] * receptive_field_size
-            fan_out = shape[0] * receptive_field_size
-
-        return (fan_in, fan_out)
-
+__all__ = ['KaimingUniform']
 
 
-class MSRAInitializer(Initializer):
-    r"""Implements the MSRA initializer a.k.a. Kaiming Initializer
+class KaimingUniform(MSRAInitializer):
+    r"""Implements the Kaiming Uniform initializer
 
     This class implements the weight initialization from the paper
     `Delving Deep into Rectifiers: Surpassing Human-Level Performance on
     ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
     by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
     robust initialization method that particularly considers the rectifier
-    nonlinearities. In case of Uniform distribution, the range is [-x, x], where
+    nonlinearities.
+
+    In case of Uniform distribution, the range is [-x, x], where
 
     .. math::
 
-        x = \sqrt{\\frac{6.0}{fan\_in}}
+        x = \sqrt{\frac{1.0}{fan\_in}}
 
     In case of Normal distribution, the mean is 0 and the standard deviation
     is
@@ -107,10 +49,8 @@ class MSRAInitializer(Initializer):
         \sqrt{\\frac{2.0}{fan\_in}}
 
     Args:
-        uniform (bool): whether to use uniform or normal distribution
-        fan_in (float32|None): fan_in for MSRAInitializer. If None, it is\
+        fan_in (float32|None): fan_in for Kaiming uniform Initializer. If None, it is\
         inferred from the variable. default is None.
-        seed (int32): random seed
 
     Note:
         It is recommended to set fan_in to None for most cases.
@@ -119,23 +59,19 @@ class MSRAInitializer(Initializer):
         .. code-block:: python
 
             import paddle
-            import paddle.fluid as fluid
-            paddle.enable_static()
-            x = fluid.data(name="data", shape=[8, 32, 32], dtype="float32")
-            fc = fluid.layers.fc(input=x, size=10,
-                param_attr=fluid.initializer.MSRA(uniform=False))
+            import paddle.nn as nn
+
+            linear = nn.Linear(2,
+                               4,
+                               weight_attr=nn.initializer.KaimingUniform())
+            data = paddle.rand([30, 10, 2], dtype='float32')
+            res = linear(data)
 
     """
 
-    def __init__(self, uniform=True, fan_in=None, seed=0):
-        """Constructor for MSRAInitializer
-        """
-        assert uniform is not None
-        assert seed is not None
-        super(MSRAInitializer, self).__init__()
-        self._uniform = uniform
-        self._fan_in = fan_in
-        self._seed = seed
+    def __init__(self, fan_in=None):
+        super(KaimingUniform, self).__init__(
+            uniform=True, fan_in=fan_in, seed=0)
 
     def __call__(self, var, block=None):
         """Initialize the input tensor with MSRA initialization.
@@ -165,8 +101,8 @@ class MSRAInitializer(Initializer):
                 var.dtype == VarDesc.VarType.BF16 and not self._uniform):
             out_dtype = VarDesc.VarType.FP32
             out_var = block.create_var(
-                name=unique_name.generate(".".join(
-                    ['masra_init', var.name, 'tmp'])),
+                name=unique_name.generate(
+                    ".".join(['masra_init', var.name, 'tmp'])),
                 shape=var.shape,
                 dtype=out_dtype,
                 type=VarDesc.VarType.LOD_TENSOR,
@@ -217,56 +153,23 @@ class MSRAInitializer(Initializer):
             var.op = op
         return op
 
-class KaimingUniform(MSRAInitializer):
-    r"""Implements the Kaiming Uniform initializer
-
-    This class implements the weight initialization from the paper
-    `Delving Deep into Rectifiers: Surpassing Human-Level Performance on
-    ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
-    by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
-    robust initialization method that particularly considers the rectifier
-    nonlinearities.
-
-    In case of Uniform distribution, the range is [-x, x], where
-
-    .. math::
-
-        x = \sqrt{\frac{6.0}{fan\_in}}
-
-    Args:
-        fan_in (float32|None): fan_in for Kaiming uniform Initializer. If None, it is\
-        inferred from the variable. default is None.
-
-    Note:
-        It is recommended to set fan_in to None for most cases.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import paddle.nn as nn
-
-            linear = nn.Linear(2,
-                               4,
-                               weight_attr=nn.initializer.KaimingUniform())
-            data = paddle.rand([30, 10, 2], dtype='float32')
-            res = linear(data)
 
+class DefaultInitializerContext(object):
     """
-
-    def __init__(self, fan_in=None):
-        super(KaimingUniform, self).__init__(
-            uniform=True, fan_in=fan_in, seed=0)
-
+        egs:
+        with DefaultInitializerContext("kaiming_uniform"):
+            code for setup_model
+    """
+    def __init__(self, init_type=None):
+        self.init_type = init_type
+ 
+    def __enter__(self):
+        from paddlespeech.s2t.modules import align
+        align.global_init_type = self.init_type
+        return self
+ 
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        from paddlespeech.s2t.modules import align
+        align.global_init_type = None
 
 
-# We short the class name, since users will use the initializer with the package
-# name. The sample code:
-#
-# import paddle.fluid as fluid
-#
-# hidden = fluid.layers.fc(...,
-#                          param_attr=ParamAttr(fluid.initializer.Xavier()))
-#
-# It is no need to add an `Initializer` as the class suffix
-MSRA = MSRAInitializer
diff --git a/paddlespeech/s2t/modules/nets_utils.py b/paddlespeech/s2t/modules/nets_utils.py
deleted file mode 100644
index 10915c8c..00000000
--- a/paddlespeech/s2t/modules/nets_utils.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# Modified from espnet(https://github.com/espnet/espnet)
-from paddle import nn
-from typeguard import check_argument_types
-
-def initialize(model: nn.Layer, init: str):
-    """Initialize weights of a neural network module.
-
-    Parameters are initialized using the given method or distribution.
-
-    Custom initialization routines can be implemented into submodules
-
-    Args:
-        model (nn.Layer): Target.
-        init (str): Method of initialization.
-    """
-    assert check_argument_types()
-
-    if init == "xavier_uniform":
-        nn.initializer.set_global_initializer(nn.initializer.XavierUniform(),
-                                              nn.initializer.Constant())
-    elif init == "xavier_normal":
-        nn.initializer.set_global_initializer(nn.initializer.XavierNormal(),
-                                              nn.initializer.Constant())
-    elif init == "kaiming_uniform":
-        nn.initializer.set_global_initializer(nn.initializer.KaimingUniform(),
-                                              nn.initializer.KaimingUniform())
-    elif init == "kaiming_normal":
-        nn.initializer.set_global_initializer(nn.initializer.KaimingNormal(),
-                                              nn.initializer.Constant())
-    else:
-        raise ValueError("Unknown initialization: " + init)
diff --git a/paddlespeech/s2t/modules/positionwise_feed_forward.py b/paddlespeech/s2t/modules/positionwise_feed_forward.py
index e2619cd4..c2725dc5 100644
--- a/paddlespeech/s2t/modules/positionwise_feed_forward.py
+++ b/paddlespeech/s2t/modules/positionwise_feed_forward.py
@@ -17,6 +17,7 @@
 import paddle
 from paddle import nn
 
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.utils.log import Log
 
 logger = Log(__name__).getlog()
@@ -44,10 +45,10 @@ class PositionwiseFeedForward(nn.Layer):
             activation (paddle.nn.Layer): Activation function
         """
         super().__init__()
-        self.w_1 = nn.Linear(idim, hidden_units)
+        self.w_1 = Linear(idim, hidden_units)
         self.activation = activation
         self.dropout = nn.Dropout(dropout_rate)
-        self.w_2 = nn.Linear(hidden_units, idim)
+        self.w_2 = Linear(hidden_units, idim)
 
     def forward(self, xs: paddle.Tensor) -> paddle.Tensor:
         """Forward function.
diff --git a/paddlespeech/s2t/modules/subsampling.py b/paddlespeech/s2t/modules/subsampling.py
index 99a8300f..88451ddd 100644
--- a/paddlespeech/s2t/modules/subsampling.py
+++ b/paddlespeech/s2t/modules/subsampling.py
@@ -19,6 +19,9 @@ from typing import Tuple
 import paddle
 from paddle import nn
 
+from paddlespeech.s2t.modules.align import Conv2D
+from paddlespeech.s2t.modules.align import LayerNorm
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.modules.embedding import PositionalEncoding
 from paddlespeech.s2t.utils.log import Log
 
@@ -60,8 +63,8 @@ class LinearNoSubsampling(BaseSubsampling):
         """
         super().__init__(pos_enc_class)
         self.out = nn.Sequential(
-            nn.Linear(idim, odim),
-            nn.LayerNorm(odim, epsilon=1e-12),
+            Linear(idim, odim),
+            LayerNorm(odim, epsilon=1e-12),
             nn.Dropout(dropout_rate),
             nn.ReLU(), )
         self.right_context = 0
@@ -108,12 +111,12 @@ class Conv2dSubsampling4(Conv2dSubsampling):
         """
         super().__init__(pos_enc_class)
         self.conv = nn.Sequential(
-            nn.Conv2D(1, odim, 3, 2),
+            Conv2D(1, odim, 3, 2),
             nn.ReLU(),
-            nn.Conv2D(odim, odim, 3, 2),
+            Conv2D(odim, odim, 3, 2),
             nn.ReLU(), )
         self.out = nn.Sequential(
-            nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim))
+            Linear(odim * (((idim - 1) // 2 - 1) // 2), odim))
         self.subsampling_rate = 4
         # The right context for every conv layer is computed by:
         # (kernel_size - 1) * frame_rate_of_this_layer
@@ -160,13 +163,13 @@ class Conv2dSubsampling6(Conv2dSubsampling):
         """
         super().__init__(pos_enc_class)
         self.conv = nn.Sequential(
-            nn.Conv2D(1, odim, 3, 2),
+            Conv2D(1, odim, 3, 2),
             nn.ReLU(),
-            nn.Conv2D(odim, odim, 5, 3),
+            Conv2D(odim, odim, 5, 3),
             nn.ReLU(), )
         # O = (I - F + Pstart + Pend) // S + 1
         # when Padding == 0, O = (I - F - S) // S
-        self.linear = nn.Linear(odim * (((idim - 1) // 2 - 2) // 3), odim)
+        self.linear = Linear(odim * (((idim - 1) // 2 - 2) // 3), odim)
         # The right context for every conv layer is computed by:
         # (kernel_size - 1) * frame_rate_of_this_layer
         # 10 = (3 - 1) * 1 + (5 - 1) * 2
@@ -212,14 +215,14 @@ class Conv2dSubsampling8(Conv2dSubsampling):
         """
         super().__init__(pos_enc_class)
         self.conv = nn.Sequential(
-            nn.Conv2D(1, odim, 3, 2),
+            Conv2D(1, odim, 3, 2),
             nn.ReLU(),
-            nn.Conv2D(odim, odim, 3, 2),
+            Conv2D(odim, odim, 3, 2),
             nn.ReLU(),
-            nn.Conv2D(odim, odim, 3, 2),
+            Conv2D(odim, odim, 3, 2),
             nn.ReLU(), )
-        self.linear = nn.Linear(odim * ((((idim - 1) // 2 - 1) // 2 - 1) // 2),
-                                odim)
+        self.linear = Linear(odim * ((((idim - 1) // 2 - 1) // 2 - 1) // 2),
+                             odim)
         self.subsampling_rate = 8
         # The right context for every conv layer is computed by:
         # (kernel_size - 1) * frame_rate_of_this_layer

From 9874fb7d75029fd015d49f4d61ec55d6e625bf1a Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Tue, 22 Mar 2022 14:25:27 +0800
Subject: [PATCH 035/126] add some comments in code

---
 paddlespeech/cli/vector/infer.py         | 46 +++++++++++-------------
 paddlespeech/vector/io/batch.py          | 29 +++++++--------
 paddlespeech/vector/modules/sid_model.py | 11 +++---
 3 files changed, 39 insertions(+), 47 deletions(-)

diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index c942c850..f1a0e79c 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -19,34 +19,28 @@ from typing import List
 from typing import Optional
 from typing import Union
 
-import librosa
-import numpy as np
 import paddle
 import soundfile
 from yacs.config import CfgNode
 
-from paddleaudio.backends import load as load_audio
-from paddleaudio.compliance.librosa import melspectrogram
-from ..download import get_path_from_url
 from ..executor import BaseExecutor
 from ..log import logger
 from ..utils import cli_register
 from ..utils import download_and_decompress
 from ..utils import MODEL_HOME
 from ..utils import stats_wrapper
-from paddlespeech.vector.io.batch import feature_normalize
-from paddlespeech.s2t.frontend.featurizer.text_featurizer import TextFeaturizer
-from paddlespeech.s2t.transform.transformation import Transformation
+from paddleaudio.backends import load as load_audio
+from paddleaudio.compliance.librosa import melspectrogram
 from paddlespeech.s2t.utils.dynamic_import import dynamic_import
-from paddlespeech.s2t.utils.utility import UpdateConfig
+from paddlespeech.vector.io.batch import feature_normalize
 from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 
 pretrained_models = {
     # The tags for pretrained_models should be "{model_name}[-{dataset}][-{sr}][-...]".
-    # e.g. "EcapaTdnn_voxceleb12-16k".
+    # e.g. "ecapatdnn_voxceleb12-16k".
     # Command line and python api use "{model_name}[-{dataset}]" as --model, usage:
-    # "paddlespeech vector --task spk --model EcapaTdnn_voxceleb12-voxceleb12-16k --sr 16000 --input ./input.wav"
-    "EcapaTdnn_voxceleb12-16k": {
+    # "paddlespeech vector --task spk --model ecapatdnn_voxceleb12-voxceleb12-16k --sr 16000 --input ./input.wav"
+    "ecapatdnn_voxceleb12-16k": {
         'url':
         'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz',
         'md5':
@@ -59,7 +53,7 @@ pretrained_models = {
 }
 
 model_alias = {
-    "EcapaTdnn": "paddlespeech.vector.models.ecapa_tdnn:EcapaTdnn",
+    "ecapatdnn": "paddlespeech.vector.models.ecapa_tdnn:EcapaTdnn",
 }
 
 
@@ -75,8 +69,8 @@ class VectorExecutor(BaseExecutor):
         self.parser.add_argument(
             "--model",
             type=str,
-            default="EcapaTdnn_voxceleb12",
-            choices=["EcapaTdnn_voxceleb12"],
+            default="ecapatdnn_voxceleb12",
+            choices=["ecapatdnn_voxceleb12"],
             help="Choose model type of asr task.")
         self.parser.add_argument(
             "--task",
@@ -90,7 +84,7 @@ class VectorExecutor(BaseExecutor):
             "--sample_rate",
             type=int,
             default=16000,
-            choices=[16000, 8000],
+            choices=[16000],
             help="Choose the audio sample rate of the model. 8000 or 16000")
         self.parser.add_argument(
             "--ckpt_path",
@@ -175,7 +169,7 @@ class VectorExecutor(BaseExecutor):
     @stats_wrapper
     def __call__(self,
                  audio_file: os.PathLike,
-                 model: str='EcapaTdnn-voxceleb12',
+                 model: str='ecapatdnn-voxceleb12',
                  sample_rate: int=16000,
                  config: os.PathLike=None,
                  ckpt_path: os.PathLike=None,
@@ -197,9 +191,9 @@ class VectorExecutor(BaseExecutor):
     def _get_pretrained_path(self, tag: str) -> os.PathLike:
         support_models = list(pretrained_models.keys())
         assert tag in pretrained_models, \
-            'The model "{}" you want to use has not been supported, \
-            please choose other models.\n \
-            The support models includes \n\t\t{}'.format(tag, "\n\t\t".join(support_models))
+            'The model "{}" you want to use has not been supported,'\
+            'please choose other models.\n' \
+            'The support models includes\n\t\t{}'.format(tag, "\n\t\t".join(support_models))
 
         res_path = os.path.join(MODEL_HOME, tag)
         decompressed_path = download_and_decompress(pretrained_models[tag],
@@ -212,7 +206,7 @@ class VectorExecutor(BaseExecutor):
         return decompressed_path
 
     def _init_from_path(self,
-                        model_type: str='EcapaTdnn_voxceleb12',
+                        model_type: str='ecapatdnn_voxceleb12',
                         sample_rate: int=16000,
                         cfg_path: Optional[os.PathLike]=None,
                         ckpt_path: Optional[os.PathLike]=None):
@@ -228,8 +222,10 @@ class VectorExecutor(BaseExecutor):
             res_path = self._get_pretrained_path(tag)
             self.res_path = res_path
 
-            self.cfg_path = os.path.join(res_path, pretrained_models[tag]['cfg_path'])
-            self.ckpt_path = os.path.join(res_path, pretrained_models[tag]['ckpt_path'] + '.pdparams')
+            self.cfg_path = os.path.join(res_path,
+                                         pretrained_models[tag]['cfg_path'])
+            self.ckpt_path = os.path.join(
+                res_path, pretrained_models[tag]['ckpt_path'] + '.pdparams')
         else:
             self.cfg_path = os.path.abspath(cfg_path)
             self.ckpt_path = os.path.abspath(ckpt_path + ".pdparams")
@@ -239,7 +235,7 @@ class VectorExecutor(BaseExecutor):
         logger.info(f"start to read the ckpt from {self.ckpt_path}")
         logger.info(f"read the config from {self.cfg_path}")
         logger.info(f"get the res path {self.res_path}")
-        
+
         # stage 2: read and config and init the model body
         self.config = CfgNode(new_allowed=True)
         self.config.merge_from_file(self.cfg_path)
@@ -269,7 +265,7 @@ class VectorExecutor(BaseExecutor):
 
         feats = self._inputs["feats"]
         lengths = self._inputs["lengths"]
-        logger.info(f"start to do backbone network model forward")
+        logger.info("start to do backbone network model forward")
         logger.info(
             f"feats shape:{feats.shape}, lengths shape: {lengths.shape}")
         # embedding from (1, emb_size, 1) -> (emb_size)
diff --git a/paddlespeech/vector/io/batch.py b/paddlespeech/vector/io/batch.py
index 25522ebb..92ca990c 100644
--- a/paddlespeech/vector/io/batch.py
+++ b/paddlespeech/vector/io/batch.py
@@ -11,9 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import numpy
 import numpy as np
 import paddle
-import numpy
+
 
 def waveform_collate_fn(batch):
     waveforms = np.stack([item['feat'] for item in batch])
@@ -57,6 +58,7 @@ def pad_right_2d(x, target_length, axis=-1, mode='constant', **kwargs):
 
     return np.pad(x, pad_width, mode=mode, **kwargs)
 
+
 def batch_feature_normalize(batch, mean_norm: bool=True, std_norm: bool=True):
     ids = [item['id'] for item in batch]
     lengths = np.asarray([item['feat'].shape[1] for item in batch])
@@ -100,12 +102,11 @@ def pad_right_to(array, target_shape, mode="constant", value=0):
     """
     assert len(target_shape) == array.ndim
     pads = []  # this contains the abs length of the padding for each dimension.
-    valid_vals = []  # thic contains the relative lengths for each dimension.
-    i = 0 # iterating over target_shape ndims
+    valid_vals = []  # this contains the relative lengths for each dimension.
+    i = 0  # iterating over target_shape ndims
     while i < len(target_shape):
-        assert (
-            target_shape[i] >= array.shape[i]
-        ), "Target shape must be >= original shape for every dim"
+        assert (target_shape[i] >= array.shape[i]
+                ), "Target shape must be >= original shape for every dim"
         pads.append([0, target_shape[i] - array.shape[i]])
         valid_vals.append(array.shape[i] / target_shape[i])
         i += 1
@@ -136,11 +137,8 @@ def batch_pad_right(arrays, mode="constant", value=0):
         # if there is only one array in the batch we simply unsqueeze it.
         return numpy.expand_dims(arrays[0], axis=0), numpy.array([1.0])
 
-    if not (
-        any(
-            [arrays[i].ndim == arrays[0].ndim for i in range(1, len(arrays))]
-        )
-    ):
+    if not (any(
+        [arrays[i].ndim == arrays[0].ndim for i in range(1, len(arrays))])):
         raise IndexError("All arrays must have same number of dimensions")
 
     # FIXME we limit the support here: we allow padding of only the last dimension
@@ -149,11 +147,9 @@ def batch_pad_right(arrays, mode="constant", value=0):
     for dim in range(arrays[0].ndim):
         if dim != (arrays[0].ndim - 1):
             if not all(
-                [x.shape[dim] == arrays[0].shape[dim] for x in arrays[1:]]
-            ):
+                [x.shape[dim] == arrays[0].shape[dim] for x in arrays[1:]]):
                 raise EnvironmentError(
-                    "arrays should have same dimensions except for last one"
-                )
+                    "arrays should have same dimensions except for last one")
         max_shape.append(max([x.shape[dim] for x in arrays]))
 
     batched = []
@@ -161,8 +157,7 @@ def batch_pad_right(arrays, mode="constant", value=0):
     for t in arrays:
         # for each array we apply pad_right_to
         padded, valid_percent = pad_right_to(
-            t, max_shape, mode=mode, value=value
-        )
+            t, max_shape, mode=mode, value=value)
         batched.append(padded)
         valid.append(valid_percent[-1])
 
diff --git a/paddlespeech/vector/modules/sid_model.py b/paddlespeech/vector/modules/sid_model.py
index dc13b2e0..4045f75d 100644
--- a/paddlespeech/vector/modules/sid_model.py
+++ b/paddlespeech/vector/modules/sid_model.py
@@ -24,7 +24,8 @@ class SpeakerIdetification(nn.Layer):
             lin_blocks=0,
             lin_neurons=192,
             dropout=0.1, ):
-        """_summary_
+        """The speaker identification model, which includes the speaker backbone network 
+           and the a linear transform to speaker class num in training
 
         Args:
             backbone (Paddle.nn.Layer class): the speaker identification backbone network model
@@ -41,7 +42,7 @@ class SpeakerIdetification(nn.Layer):
             self.dropout = nn.Dropout(dropout)
         else:
             self.dropout = None
-        
+
         # construct the speaker classifer
         input_size = self.backbone.emb_size
         self.blocks = nn.LayerList()
@@ -63,14 +64,14 @@ class SpeakerIdetification(nn.Layer):
            including the speaker embedding model and the classifier model network
 
         Args:
-            x (Paddle.Tensor): input audio feats, 
+            x (paddle.Tensor): input audio feats, 
                                shape=[batch, dimension, times]
-            lengths (_type_, optional): input audio length.
+            lengths (paddle.Tensor, optional): input audio length.
                                         shape=[batch, times]
                                         Defaults to None.
 
         Returns:
-            _type_: _description_
+            paddle.Tensor: return the logits of the feats
         """
         # x.shape: (N, C, L)
         x = self.backbone(x, lengths).squeeze(

From d85d1deef53b72dcf1c822bc26f431021027c9d9 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Tue, 22 Mar 2022 14:29:45 +0800
Subject: [PATCH 036/126] exec pre-commit in paddlespeech vector, test=doc

---
 paddlespeech/vector/__init__.py               |  2 +-
 .../vector/exps/ecapa_tdnn/extract_emb.py     | 21 ++++-----
 paddlespeech/vector/exps/ecapa_tdnn/train.py  | 47 +++++++++++--------
 3 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/paddlespeech/vector/__init__.py b/paddlespeech/vector/__init__.py
index 61d5aa21..185a92b8 100644
--- a/paddlespeech/vector/__init__.py
+++ b/paddlespeech/vector/__init__.py
@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
index 6dfcf06d..e30a50e4 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
@@ -13,9 +13,8 @@
 # limitations under the License.
 import argparse
 import os
-
 import time
-import numpy as np
+
 import paddle
 from yacs.config import CfgNode
 
@@ -40,7 +39,8 @@ def extract_audio_embedding(args, config):
     ecapa_tdnn = EcapaTdnn(**config.model)
 
     # stage4: build the speaker verification train instance with backbone model
-    model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=config.num_speakers)
+    model = SpeakerIdetification(
+        backbone=ecapa_tdnn, num_class=config.num_speakers)
     # stage 2: load the pre-trained model
     args.load_checkpoint = os.path.abspath(
         os.path.expanduser(args.load_checkpoint))
@@ -62,17 +62,17 @@ def extract_audio_embedding(args, config):
     # we need convert the audio feat to one-batch shape [batch, dim, time], where the batch is one
     # so the final shape is [1, dim, time]
     start_time = time.time()
-    feat = melspectrogram(x=waveform, 
-                          sr=config.sr,
-                          n_mels=config.n_mels,
-                          window_size=config.window_size,
-                          hop_length=config.hop_size)
+    feat = melspectrogram(
+        x=waveform,
+        sr=config.sr,
+        n_mels=config.n_mels,
+        window_size=config.window_size,
+        hop_length=config.hop_size)
     feat = paddle.to_tensor(feat).unsqueeze(0)
 
     # in inference period, the lengths is all one without padding
     lengths = paddle.ones([1])
-    feat = feature_normalize(
-        feat, mean_norm=True, std_norm=False)
+    feat = feature_normalize(feat, mean_norm=True, std_norm=False)
 
     # model backbone network forward the feats and get the embedding
     embedding = model.backbone(
@@ -80,7 +80,6 @@ def extract_audio_embedding(args, config):
     elapsed_time = time.time() - start_time
     audio_length = waveform.shape[0] / sr
 
-
     # stage 5: do global norm with external mean and std
     rtf = elapsed_time / audio_length
     logger.info(f"{args.device} rft={rtf}")
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/train.py b/paddlespeech/vector/exps/ecapa_tdnn/train.py
index fb02d486..257b97ab 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/train.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/train.py
@@ -13,9 +13,9 @@
 # limitations under the License.
 import argparse
 import os
+import time
 
 import numpy as np
-import time
 import paddle
 from paddle.io import BatchSampler
 from paddle.io import DataLoader
@@ -27,6 +27,7 @@ from paddleaudio.datasets.voxceleb import VoxCeleb
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.augment import build_augment_pipeline
 from paddlespeech.vector.io.augment import waveform_augment
+from paddlespeech.vector.io.batch import batch_pad_right
 from paddlespeech.vector.io.batch import feature_normalize
 from paddlespeech.vector.io.batch import waveform_collate_fn
 from paddlespeech.vector.models.ecapa_tdnn import EcapaTdnn
@@ -36,7 +37,6 @@ from paddlespeech.vector.modules.sid_model import SpeakerIdetification
 from paddlespeech.vector.training.scheduler import CyclicLRScheduler
 from paddlespeech.vector.training.seeding import seed_everything
 from paddlespeech.vector.utils.time import Timer
-from paddlespeech.vector.io.batch import batch_pad_right
 
 logger = Log(__name__).getlog()
 
@@ -145,7 +145,7 @@ def main(args, config):
         reader_start = time.time()
         for batch_idx, batch in enumerate(train_loader):
             train_reader_cost += time.time() - reader_start
-            
+
             # stage 9-1: batch data is audio sample points and speaker id label
             feat_start = time.time()
             waveforms, labels = batch['waveforms'], batch['labels']
@@ -165,11 +165,12 @@ def main(args, config):
             # stage 9-3: extract the audio feats,such fbank, mfcc, spectrogram
             feats = []
             for waveform in waveforms.numpy():
-                feat = melspectrogram(x=waveform, 
-                                      sr=config.sr, 
-                                      n_mels=config.n_mels, 
-                                      window_size=config.window_size, 
-                                      hop_length=config.hop_size)
+                feat = melspectrogram(
+                    x=waveform,
+                    sr=config.sr,
+                    n_mels=config.n_mels,
+                    window_size=config.window_size,
+                    hop_length=config.hop_size)
                 feats.append(feat)
             feats = paddle.to_tensor(np.asarray(feats))
 
@@ -202,7 +203,7 @@ def main(args, config):
             num_corrects += (preds == labels).numpy().sum()
             num_samples += feats.shape[0]
             timer.count()  # step plus one in timer
-            
+
             # stage 9-10: print the log information only on 0-rank per log-freq batchs
             if (batch_idx + 1) % config.log_interval == 0 and local_rank == 0:
                 lr = optimizer.get_lr()
@@ -213,9 +214,12 @@ def main(args, config):
                     epoch, config.epochs, batch_idx + 1, steps_per_epoch)
                 print_msg += ' loss={:.4f}'.format(avg_loss)
                 print_msg += ' acc={:.4f}'.format(avg_acc)
-                print_msg += ' avg_reader_cost: {:.5f} sec,'.format(train_reader_cost / config.log_interval)
-                print_msg += ' avg_feat_cost: {:.5f} sec,'.format(train_feat_cost / config.log_interval)
-                print_msg += ' avg_train_cost: {:.5f} sec,'.format(train_run_cost / config.log_interval)
+                print_msg += ' avg_reader_cost: {:.5f} sec,'.format(
+                    train_reader_cost / config.log_interval)
+                print_msg += ' avg_feat_cost: {:.5f} sec,'.format(
+                    train_feat_cost / config.log_interval)
+                print_msg += ' avg_train_cost: {:.5f} sec,'.format(
+                    train_run_cost / config.log_interval)
                 print_msg += ' lr={:.4E} step/sec={:.2f} | ETA {}'.format(
                     lr, timer.timing, timer.eta)
                 logger.info(print_msg)
@@ -262,11 +266,12 @@ def main(args, config):
 
                     feats = []
                     for waveform in waveforms.numpy():
-                        feat = melspectrogram(x=waveform, 
-                                              sr=config.sr,
-                                              n_mels=config.n_mels,
-                                              window_size=config.window_size,
-                                              hop_length=config.hop_size)
+                        feat = melspectrogram(
+                            x=waveform,
+                            sr=config.sr,
+                            n_mels=config.n_mels,
+                            window_size=config.window_size,
+                            hop_length=config.hop_size)
                         feats.append(feat)
 
                     feats = paddle.to_tensor(np.asarray(feats))
@@ -285,7 +290,8 @@ def main(args, config):
             # stage 9-14: Save model parameters
             save_dir = os.path.join(args.checkpoint_dir,
                                     'epoch_{}'.format(epoch))
-            last_saved_epoch = os.path.join('epoch_{}'.format(epoch), "model.pdparams")
+            last_saved_epoch = os.path.join('epoch_{}'.format(epoch),
+                                            "model.pdparams")
             logger.info('Saving model checkpoint to {}'.format(save_dir))
             paddle.save(model.state_dict(),
                         os.path.join(save_dir, 'model.pdparams'))
@@ -300,10 +306,13 @@ def main(args, config):
         final_model = os.path.join(args.checkpoint_dir, "model.pdparams")
         logger.info(f"we will create the final model: {final_model}")
         if os.path.islink(final_model):
-            logger.info(f"An {final_model} already exists, we will rm is and create it again")
+            logger.info(
+                f"An {final_model} already exists, we will rm is and create it again"
+            )
             os.unlink(final_model)
         os.symlink(last_saved_epoch, final_model)
 
+
 if __name__ == "__main__":
     # yapf: disable
     parser = argparse.ArgumentParser(__doc__)

From a4f5a680742240a471c6264432106eb14ad678d1 Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Tue, 22 Mar 2022 07:14:40 +0000
Subject: [PATCH 037/126] fix some format, test=asr

---
 paddlespeech/s2t/models/u2/u2.py        |   5 +-
 paddlespeech/s2t/modules/activation.py  |   3 +-
 paddlespeech/s2t/modules/align.py       | 139 +++++++++++++++++-------
 paddlespeech/s2t/modules/encoder.py     |   1 -
 paddlespeech/s2t/modules/initializer.py |  12 +-
 5 files changed, 110 insertions(+), 50 deletions(-)

diff --git a/paddlespeech/s2t/models/u2/u2.py b/paddlespeech/s2t/models/u2/u2.py
index e077cd5b..e94a127d 100644
--- a/paddlespeech/s2t/models/u2/u2.py
+++ b/paddlespeech/s2t/models/u2/u2.py
@@ -36,6 +36,7 @@ from paddlespeech.s2t.modules.ctc import CTCDecoderBase
 from paddlespeech.s2t.modules.decoder import TransformerDecoder
 from paddlespeech.s2t.modules.encoder import ConformerEncoder
 from paddlespeech.s2t.modules.encoder import TransformerEncoder
+from paddlespeech.s2t.modules.initializer import DefaultInitializerContext
 from paddlespeech.s2t.modules.loss import LabelSmoothingLoss
 from paddlespeech.s2t.modules.mask import make_pad_mask
 from paddlespeech.s2t.modules.mask import mask_finished_preds
@@ -50,7 +51,6 @@ from paddlespeech.s2t.utils.tensor_utils import pad_sequence
 from paddlespeech.s2t.utils.tensor_utils import th_accuracy
 from paddlespeech.s2t.utils.utility import log_add
 from paddlespeech.s2t.utils.utility import UpdateConfig
-from paddlespeech.s2t.modules.initializer import DefaultInitializerContext
 # from paddlespeech.s2t.modules.initializer import initialize
 
 __all__ = ["U2Model", "U2InferModel"]
@@ -786,7 +786,8 @@ class U2Model(U2DecodeModel):
         model_conf = configs.get('model_conf', dict())
         init_type = model_conf.get("init_type", None)
         with DefaultInitializerContext(init_type):
-            vocab_size, encoder, decoder, ctc = U2Model._init_from_config(configs)
+            vocab_size, encoder, decoder, ctc = U2Model._init_from_config(
+                configs)
 
         super().__init__(
             vocab_size=vocab_size,
diff --git a/paddlespeech/s2t/modules/activation.py b/paddlespeech/s2t/modules/activation.py
index 48c84fa6..2f387b0d 100644
--- a/paddlespeech/s2t/modules/activation.py
+++ b/paddlespeech/s2t/modules/activation.py
@@ -16,8 +16,9 @@ from collections import OrderedDict
 import paddle
 from paddle import nn
 from paddle.nn import functional as F
-from paddlespeech.s2t.modules.align import Linear
+
 from paddlespeech.s2t.modules.align import Conv2D
+from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.utils.log import Log
 
 logger = Log(__name__).getlog()
diff --git a/paddlespeech/s2t/modules/align.py b/paddlespeech/s2t/modules/align.py
index 575773d7..f8891679 100644
--- a/paddlespeech/s2t/modules/align.py
+++ b/paddlespeech/s2t/modules/align.py
@@ -1,7 +1,20 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 import paddle
 from paddle import nn
-from paddlespeech.s2t.modules.initializer import KaimingUniform
 
+from paddlespeech.s2t.modules.initializer import KaimingUniform
 """
     To align the initializer between paddle and torch, 
     the API below are set defalut initializer with priority higger than global initializer.
@@ -10,65 +23,117 @@ global_init_type = None
 
 
 class LayerNorm(nn.LayerNorm):
-    def __init__(self, normalized_shape, epsilon=1e-05, weight_attr=None, bias_attr=None, name=None):
+    def __init__(self,
+                 normalized_shape,
+                 epsilon=1e-05,
+                 weight_attr=None,
+                 bias_attr=None,
+                 name=None):
         if weight_attr is None:
             weight_attr = paddle.ParamAttr(
                 initializer=nn.initializer.Constant(1.0))
         if bias_attr is None:
             bias_attr = paddle.ParamAttr(
                 initializer=nn.initializer.Constant(0.0))
-        super(LayerNorm, self).__init__(normalized_shape, epsilon, weight_attr, bias_attr, name)
+        super(LayerNorm, self).__init__(normalized_shape, epsilon, weight_attr,
+                                        bias_attr, name)
+
 
-class BatchNorm1D(nn.BatchNorm1D): 
-    def __init__(self, num_features, momentum=0.9, epsilon=1e-05, weight_attr=None, bias_attr=None, data_format='NCL', name=None):
+class BatchNorm1D(nn.BatchNorm1D):
+    def __init__(self,
+                 num_features,
+                 momentum=0.9,
+                 epsilon=1e-05,
+                 weight_attr=None,
+                 bias_attr=None,
+                 data_format='NCL',
+                 name=None):
         if weight_attr is None:
             weight_attr = paddle.ParamAttr(
                 initializer=nn.initializer.Constant(1.0))
         if bias_attr is None:
             bias_attr = paddle.ParamAttr(
                 initializer=nn.initializer.Constant(0.0))
-        super(BatchNorm1D, self).__init__(num_features, momentum, epsilon, weight_attr, bias_attr, data_format, name)
+        super(BatchNorm1D,
+              self).__init__(num_features, momentum, epsilon, weight_attr,
+                             bias_attr, data_format, name)
+
 
 class Embedding(nn.Embedding):
-    def __init__(self, num_embeddings, embedding_dim, padding_idx=None, sparse=False, weight_attr=None, name=None):
+    def __init__(self,
+                 num_embeddings,
+                 embedding_dim,
+                 padding_idx=None,
+                 sparse=False,
+                 weight_attr=None,
+                 name=None):
         if weight_attr is None:
-            weight_attr = paddle.ParamAttr(
-                initializer=nn.initializer.Normal())
-        super(Embedding, self).__init__(num_embeddings, embedding_dim, padding_idx, sparse, weight_attr, name)
+            weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal())
+        super(Embedding, self).__init__(num_embeddings, embedding_dim,
+                                        padding_idx, sparse, weight_attr, name)
+
 
 class Linear(nn.Linear):
-    def __init__(self, in_features, out_features, weight_attr=None, bias_attr=None, name=None):
-       if weight_attr is None:
-           if global_init_type == "kaiming_uniform":
-               weight_attr = paddle.ParamAttr(
-                   initializer=KaimingUniform())
-       if bias_attr is None:
-           if global_init_type == "kaiming_uniform":
-               bias_attr = paddle.ParamAttr(
-                   initializer=KaimingUniform())
-        super(Linear, self).__init__(in_features, out_features, weight_attr, bias_attr, name)
+    def __init__(self,
+                 in_features,
+                 out_features,
+                 weight_attr=None,
+                 bias_attr=None,
+                 name=None):
+        if weight_attr is None:
+            if global_init_type == "kaiming_uniform":
+                weight_attr = paddle.ParamAttr(initializer=KaimingUniform())
+        if bias_attr is None:
+            if global_init_type == "kaiming_uniform":
+                bias_attr = paddle.ParamAttr(initializer=KaimingUniform())
+        super(Linear, self).__init__(in_features, out_features, weight_attr,
+                                     bias_attr, name)
+
 
 class Conv1D(nn.Conv1D):
-    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, padding_mode='zeros', weight_attr=None, bias_attr=None, data_format='NCL'):
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size,
+                 stride=1,
+                 padding=0,
+                 dilation=1,
+                 groups=1,
+                 padding_mode='zeros',
+                 weight_attr=None,
+                 bias_attr=None,
+                 data_format='NCL'):
         if weight_attr is None:
             if global_init_type == "kaiming_uniform":
                 print("set kaiming_uniform")
-                weight_attr = paddle.ParamAttr(
-                    initializer=KaimingUniform())
+                weight_attr = paddle.ParamAttr(initializer=KaimingUniform())
         if bias_attr is None:
             if global_init_type == "kaiming_uniform":
-                bias_attr = paddle.ParamAttr(
-                    initializer=KaimingUniform())
-        super(Conv1D, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, padding_mode, weight_attr, bias_attr, data_format)
-    
+                bias_attr = paddle.ParamAttr(initializer=KaimingUniform())
+        super(Conv1D, self).__init__(
+            in_channels, out_channels, kernel_size, stride, padding, dilation,
+            groups, padding_mode, weight_attr, bias_attr, data_format)
+
+
 class Conv2D(nn.Conv2D):
-    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, padding_mode='zeros', weight_attr=None, bias_attr=None, data_format='NCHW'):
-       if weight_attr is None:
-           if global_init_type == "kaiming_uniform":
-               weight_attr = paddle.ParamAttr(
-                   initializer=KaimingUniform())
-       if bias_attr is None:
-           if global_init_type == "kaiming_uniform":
-               bias_attr = paddle.ParamAttr(
-                   initializer=KaimingUniform())
-        super(Conv2D, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, padding_mode, weight_attr, bias_attr, data_format)
+    def __init__(self,
+                 in_channels,
+                 out_channels,
+                 kernel_size,
+                 stride=1,
+                 padding=0,
+                 dilation=1,
+                 groups=1,
+                 padding_mode='zeros',
+                 weight_attr=None,
+                 bias_attr=None,
+                 data_format='NCHW'):
+        if weight_attr is None:
+            if global_init_type == "kaiming_uniform":
+                weight_attr = paddle.ParamAttr(initializer=KaimingUniform())
+        if bias_attr is None:
+            if global_init_type == "kaiming_uniform":
+                bias_attr = paddle.ParamAttr(initializer=KaimingUniform())
+        super(Conv2D, self).__init__(
+            in_channels, out_channels, kernel_size, stride, padding, dilation,
+            groups, padding_mode, weight_attr, bias_attr, data_format)
diff --git a/paddlespeech/s2t/modules/encoder.py b/paddlespeech/s2t/modules/encoder.py
index 71a2bad4..c843c0e2 100644
--- a/paddlespeech/s2t/modules/encoder.py
+++ b/paddlespeech/s2t/modules/encoder.py
@@ -24,7 +24,6 @@ from typeguard import check_argument_types
 
 from paddlespeech.s2t.modules.activation import get_activation
 from paddlespeech.s2t.modules.align import LayerNorm
-from paddlespeech.s2t.modules.align import Linear
 from paddlespeech.s2t.modules.attention import MultiHeadedAttention
 from paddlespeech.s2t.modules.attention import RelPositionMultiHeadedAttention
 from paddlespeech.s2t.modules.conformer_convolution import ConvolutionModule
diff --git a/paddlespeech/s2t/modules/initializer.py b/paddlespeech/s2t/modules/initializer.py
index 3fbab285..98466ebd 100644
--- a/paddlespeech/s2t/modules/initializer.py
+++ b/paddlespeech/s2t/modules/initializer.py
@@ -12,15 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import numpy as np
-from paddle import nn
 from paddle.fluid import framework
 from paddle.fluid import unique_name
 from paddle.fluid.core import VarDesc
-from paddle.fluid.framework import default_main_program
-from paddle.fluid.framework import in_dygraph_mode
-from paddle.fluid.initializer import Initializer
 from paddle.fluid.initializer import MSRAInitializer
-from typeguard import check_argument_types
 
 __all__ = ['KaimingUniform']
 
@@ -160,16 +155,15 @@ class DefaultInitializerContext(object):
         with DefaultInitializerContext("kaiming_uniform"):
             code for setup_model
     """
+
     def __init__(self, init_type=None):
         self.init_type = init_type
- 
+
     def __enter__(self):
         from paddlespeech.s2t.modules import align
         align.global_init_type = self.init_type
         return self
- 
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         from paddlespeech.s2t.modules import align
         align.global_init_type = None
-
-

From 6da8465f146afaeeaf253c9acd13c7397df50065 Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Tue, 22 Mar 2022 08:10:52 +0000
Subject: [PATCH 038/126] add dist_sampler args, test=asr

---
 examples/aishell/asr1/conf/chunk_conformer.yaml | 3 ++-
 examples/aishell/asr1/conf/conformer.yaml       | 3 ++-
 examples/aishell/asr1/conf/transformer.yaml     | 5 +++--
 paddlespeech/s2t/models/u2/u2.py                | 1 -
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/examples/aishell/asr1/conf/chunk_conformer.yaml b/examples/aishell/asr1/conf/chunk_conformer.yaml
index 68e852ba..1ad77f97 100644
--- a/examples/aishell/asr1/conf/chunk_conformer.yaml
+++ b/examples/aishell/asr1/conf/chunk_conformer.yaml
@@ -70,7 +70,7 @@ batch_bins: 0
 batch_frames_in: 0
 batch_frames_out: 0
 batch_frames_inout: 0
-num_workers: 0
+num_workers: 2
 subsampling_factor: 1
 num_encs: 1
 
@@ -80,6 +80,7 @@ num_encs: 1
 n_epoch: 240 
 accum_grad: 2
 global_grad_clip: 5.0
+dist_sampler: True
 optim: adam
 optim_conf:
   lr: 0.002
diff --git a/examples/aishell/asr1/conf/conformer.yaml b/examples/aishell/asr1/conf/conformer.yaml
index 679a5bf6..d5d883a0 100644
--- a/examples/aishell/asr1/conf/conformer.yaml
+++ b/examples/aishell/asr1/conf/conformer.yaml
@@ -76,6 +76,7 @@ num_encs: 1
 n_epoch: 240 
 accum_grad: 2
 global_grad_clip: 5.0
+dist_sampler: True
 optim: adam
 optim_conf:
   lr: 0.002
@@ -84,7 +85,7 @@ scheduler: warmuplr
 scheduler_conf:
   warmup_steps: 25000
   lr_decay: 1.0
-log_interval: 100
+log_interval: 1
 checkpoint:
   kbest_n: 50
   latest_n: 5
diff --git a/examples/aishell/asr1/conf/transformer.yaml b/examples/aishell/asr1/conf/transformer.yaml
index 9d294653..9e08ea0e 100644
--- a/examples/aishell/asr1/conf/transformer.yaml
+++ b/examples/aishell/asr1/conf/transformer.yaml
@@ -61,16 +61,17 @@ batch_frames_in: 0
 batch_frames_out: 0
 batch_frames_inout: 0
 preprocess_config: conf/preprocess.yaml 
-num_workers: 0
+num_workers: 2
 subsampling_factor: 1
 num_encs: 1
 
 ###########################################
 #                 Training                #
 ###########################################
-n_epoch: 240 
+n_epoch: 30
 accum_grad: 2
 global_grad_clip: 5.0
+dist_sampler: False
 optim: adam
 optim_conf:
   lr: 0.002
diff --git a/paddlespeech/s2t/models/u2/u2.py b/paddlespeech/s2t/models/u2/u2.py
index e94a127d..51388586 100644
--- a/paddlespeech/s2t/models/u2/u2.py
+++ b/paddlespeech/s2t/models/u2/u2.py
@@ -51,7 +51,6 @@ from paddlespeech.s2t.utils.tensor_utils import pad_sequence
 from paddlespeech.s2t.utils.tensor_utils import th_accuracy
 from paddlespeech.s2t.utils.utility import log_add
 from paddlespeech.s2t.utils.utility import UpdateConfig
-# from paddlespeech.s2t.modules.initializer import initialize
 
 __all__ = ["U2Model", "U2InferModel"]
 

From c36039ce322f45e38ed1a9094994b7ff6572975b Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Tue, 22 Mar 2022 08:45:22 +0000
Subject: [PATCH 039/126] update readme for ljspeech hifigan, test=tts

---
 examples/ljspeech/tts3/local/synthesize.sh    | 53 ++++++++++++-----
 .../ljspeech/tts3/local/synthesize_e2e.sh     | 58 ++++++++++++++-----
 2 files changed, 81 insertions(+), 30 deletions(-)

diff --git a/examples/ljspeech/tts3/local/synthesize.sh b/examples/ljspeech/tts3/local/synthesize.sh
index f150d158..6dc34274 100755
--- a/examples/ljspeech/tts3/local/synthesize.sh
+++ b/examples/ljspeech/tts3/local/synthesize.sh
@@ -4,17 +4,42 @@ config_path=$1
 train_output_path=$2
 ckpt_name=$3
 
-FLAGS_allocator_strategy=naive_best_fit \
-FLAGS_fraction_of_gpu_memory_to_use=0.01 \
-python3 ${BIN_DIR}/../synthesize.py \
-    --am=fastspeech2_ljspeech \
-    --am_config=${config_path} \
-    --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
-    --am_stat=dump/train/speech_stats.npy \
-    --voc=pwgan_ljspeech \
-    --voc_config=pwg_ljspeech_ckpt_0.5/pwg_default.yaml \
-    --voc_ckpt=pwg_ljspeech_ckpt_0.5/pwg_snapshot_iter_400000.pdz  \
-    --voc_stat=pwg_ljspeech_ckpt_0.5/pwg_stats.npy \
-    --test_metadata=dump/test/norm/metadata.jsonl \
-    --output_dir=${train_output_path}/test \
-    --phones_dict=dump/phone_id_map.txt
+stage=0
+stop_stage=0
+
+# pwgan
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize.py \
+        --am=fastspeech2_ljspeech \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=pwgan_ljspeech \
+        --voc_config=pwg_ljspeech_ckpt_0.5/pwg_default.yaml \
+        --voc_ckpt=pwg_ljspeech_ckpt_0.5/pwg_snapshot_iter_400000.pdz  \
+        --voc_stat=pwg_ljspeech_ckpt_0.5/pwg_stats.npy \
+        --test_metadata=dump/test/norm/metadata.jsonl \
+        --output_dir=${train_output_path}/test \
+        --phones_dict=dump/phone_id_map.txt
+fi
+
+# hifigan
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize.py \
+        --am=fastspeech2_ljspeech \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=hifigan_ljspeech \
+        --voc_config=hifigan_ljspeech_ckpt_0.2.0/default.yaml \
+        --voc_ckpt=hifigan_ljspeech_ckpt_0.2.0/snapshot_iter_2500000.pdz \
+        --voc_stat=hifigan_ljspeech_ckpt_0.2.0/feats_stats.npy \
+        --test_metadata=dump/test/norm/metadata.jsonl \
+        --output_dir=${train_output_path}/test \
+        --phones_dict=dump/phone_id_map.txt
+fi
+
diff --git a/examples/ljspeech/tts3/local/synthesize_e2e.sh b/examples/ljspeech/tts3/local/synthesize_e2e.sh
index 0b0cb574..36865f7f 100755
--- a/examples/ljspeech/tts3/local/synthesize_e2e.sh
+++ b/examples/ljspeech/tts3/local/synthesize_e2e.sh
@@ -4,19 +4,45 @@ config_path=$1
 train_output_path=$2
 ckpt_name=$3
 
-FLAGS_allocator_strategy=naive_best_fit \
-FLAGS_fraction_of_gpu_memory_to_use=0.01 \
-python3 ${BIN_DIR}/../synthesize_e2e.py \
-    --am=fastspeech2_ljspeech \
-    --am_config=${config_path} \
-    --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
-    --am_stat=dump/train/speech_stats.npy \
-    --voc=pwgan_ljspeech \
-    --voc_config=pwg_ljspeech_ckpt_0.5/pwg_default.yaml \
-    --voc_ckpt=pwg_ljspeech_ckpt_0.5/pwg_snapshot_iter_400000.pdz  \
-    --voc_stat=pwg_ljspeech_ckpt_0.5/pwg_stats.npy \
-    --lang=en \
-    --text=${BIN_DIR}/../sentences_en.txt \
-    --output_dir=${train_output_path}/test_e2e \
-    --inference_dir=${train_output_path}/inference \
-    --phones_dict=dump/phone_id_map.txt
\ No newline at end of file
+stage=0
+stop_stage=0
+
+# pwgan
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize_e2e.py \
+        --am=fastspeech2_ljspeech \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=pwgan_ljspeech \
+        --voc_config=pwg_ljspeech_ckpt_0.5/pwg_default.yaml \
+        --voc_ckpt=pwg_ljspeech_ckpt_0.5/pwg_snapshot_iter_400000.pdz  \
+        --voc_stat=pwg_ljspeech_ckpt_0.5/pwg_stats.npy \
+        --lang=en \
+        --text=${BIN_DIR}/../sentences_en.txt \
+        --output_dir=${train_output_path}/test_e2e \
+        --inference_dir=${train_output_path}/inference \
+        --phones_dict=dump/phone_id_map.txt
+fi
+
+# hifigan
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize_e2e.py \
+        --am=fastspeech2_ljspeech \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=hifigan_ljspeech \
+        --voc_config=hifigan_ljspeech_ckpt_0.2.0/default.yaml \
+        --voc_ckpt=hifigan_ljspeech_ckpt_0.2.0/snapshot_iter_2500000.pdz \
+        --voc_stat=hifigan_ljspeech_ckpt_0.2.0/feats_stats.npy \
+        --lang=en \
+        --text=${BIN_DIR}/../sentences_en.txt \
+        --output_dir=${train_output_path}/test_e2e \
+        --inference_dir=${train_output_path}/inference \
+        --phones_dict=dump/phone_id_map.txt
+fi

From e1b581b622608820b0c36971a2ad64cf2e0923c5 Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Tue, 22 Mar 2022 08:59:26 +0000
Subject: [PATCH 040/126] fix some bug, test=asr

---
 examples/aishell/asr1/conf/conformer.yaml | 2 +-
 paddlespeech/s2t/exps/u2/model.py         | 4 ++--
 paddlespeech/s2t/modules/initializer.py   | 9 ++++++---
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/examples/aishell/asr1/conf/conformer.yaml b/examples/aishell/asr1/conf/conformer.yaml
index d5d883a0..a150a04d 100644
--- a/examples/aishell/asr1/conf/conformer.yaml
+++ b/examples/aishell/asr1/conf/conformer.yaml
@@ -85,7 +85,7 @@ scheduler: warmuplr
 scheduler_conf:
   warmup_steps: 25000
   lr_decay: 1.0
-log_interval: 1
+log_interval: 100
 checkpoint:
   kbest_n: 50
   latest_n: 5
diff --git a/paddlespeech/s2t/exps/u2/model.py b/paddlespeech/s2t/exps/u2/model.py
index bcbc15d6..efcc9629 100644
--- a/paddlespeech/s2t/exps/u2/model.py
+++ b/paddlespeech/s2t/exps/u2/model.py
@@ -239,7 +239,7 @@ class U2Trainer(Trainer):
                 n_iter_processes=config.num_workers,
                 subsampling_factor=1,
                 num_encs=1,
-                dist_sampler=True,
+                dist_sampler=config.get('dist_sampler', False),
                 shortest_first=False)
 
             self.valid_loader = BatchDataLoader(
@@ -260,7 +260,7 @@ class U2Trainer(Trainer):
                 n_iter_processes=config.num_workers,
                 subsampling_factor=1,
                 num_encs=1,
-                dist_sampler=True,
+                dist_sampler=config.get('dist_sampler', False),
                 shortest_first=False)
             logger.info("Setup train/valid Dataloader!")
         else:
diff --git a/paddlespeech/s2t/modules/initializer.py b/paddlespeech/s2t/modules/initializer.py
index 98466ebd..30a04e44 100644
--- a/paddlespeech/s2t/modules/initializer.py
+++ b/paddlespeech/s2t/modules/initializer.py
@@ -160,9 +160,12 @@ class DefaultInitializerContext(object):
         self.init_type = init_type
 
     def __enter__(self):
-        from paddlespeech.s2t.modules import align
-        align.global_init_type = self.init_type
-        return self
+        if self.init_type is None:
+            return
+        else:
+            from paddlespeech.s2t.modules import align
+            align.global_init_type = self.init_type
+            return
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         from paddlespeech.s2t.modules import align

From 096e41520d3618cb03e9df2b22bec101f5d859af Mon Sep 17 00:00:00 2001
From: qingen <qingenz123@126.com>
Date: Tue, 22 Mar 2022 17:41:30 +0800
Subject: [PATCH 041/126] [wip][vec] update insert picture, test=doc #1543

---
 demos/audio_searching/README.md      |   2 +-
 demos/audio_searching/README_cn.md   |   8 ++++----
 demos/audio_searching/img/insert.png | Bin 81912 -> 50924 bytes
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/demos/audio_searching/README.md b/demos/audio_searching/README.md
index 2b417c0e..2bce9313 100644
--- a/demos/audio_searching/README.md
+++ b/demos/audio_searching/README.md
@@ -42,7 +42,7 @@ b2bcf279e599  milvusdb/milvus:v2.0.1  "/tini -- milvus run…"  22 hours ago  Up
 d8ef4c84e25c  mysql:5.7 "docker-entrypoint.s…"  22 hours ago  Up 22 hours 0.0.0.0:3306->3306/tcp, 33060/tcp audio-mysql
 8fb501edb4f3  quay.io/coreos/etcd:v3.5.0  "etcd -advertise-cli…"  22 hours ago  Up 22 hours 2379-2380/tcp milvus-etcd
 ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"  22 hours ago  Up 22 hours (healthy) 9000/tcp  milvus-minio
-15c84a506754  iregistry.baidu-int.com/paddlespeech/audio-search-client:1.0  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
+15c84a506754  qingen1/paddlespeech-audio-search-client:2.3  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
 ```
 
 ### 2. Start API Server
diff --git a/demos/audio_searching/README_cn.md b/demos/audio_searching/README_cn.md
index d822c00d..a4cb7312 100644
--- a/demos/audio_searching/README_cn.md
+++ b/demos/audio_searching/README_cn.md
@@ -6,7 +6,7 @@
 
 随着互联网不断发展，电子邮件、社交媒体照片、直播视频、客服语音等非结构化数据已经变得越来越普遍。如果想要使用计算机来处理这些数据，需要使用 embedding 技术将这些数据转化为向量 vector，然后进行存储、建索引、并查询
 
-但是，当数据量很大，比如上亿条音频要做相似度搜索，就比较困难了。穷举法固然可行，但非常耗时。针对这种场景，该demo 将介绍如何使用开源向量数据库 Milvus 搭建音频相似度检索系统
+但是，当数据量很大，比如上亿条音频要做相似度搜索，就比较困难了。穷举法固然可行，但非常耗时。针对这种场景，该 demo 将介绍如何使用开源向量数据库 Milvus 搭建音频相似度检索系统
 
 音频检索（如演讲、音乐、说话人等检索）实现了在海量音频数据中查询并找出相似声音（或相同说话人）片段。音频相似性检索系统可用于识别相似的音效、最大限度减少知识产权侵权等，还可以快速的检索声纹库、帮助企业控制欺诈和身份盗用等。在音频数据的分类和统计分析中，音频检索也发挥着重要作用
 
@@ -43,7 +43,7 @@ b2bcf279e599  milvusdb/milvus:v2.0.1  "/tini -- milvus run…"  22 hours ago  Up
 d8ef4c84e25c  mysql:5.7 "docker-entrypoint.s…"  22 hours ago  Up 22 hours 0.0.0.0:3306->3306/tcp, 33060/tcp audio-mysql
 8fb501edb4f3  quay.io/coreos/etcd:v3.5.0  "etcd -advertise-cli…"  22 hours ago  Up 22 hours 2379-2380/tcp milvus-etcd
 ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"  22 hours ago  Up 22 hours (healthy) 9000/tcp  milvus-minio
-15c84a506754  iregistry.baidu-int.com/paddlespeech/audio-search-client:1.0  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
+15c84a506754  qingen1/paddlespeech-audio-search-client:2.3  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
 
 ```
 
@@ -129,7 +129,7 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
   
     在浏览器中输入 127.0.0.1:8068 访问前端页面
     
-    注：如果浏览器和服务不在同一台机器上，那么 IP 需要修改成服务所在的机器 IP，并且docker-compose.yaml 中相应的 API_URL 也要修改，并重新起服务即可
+    注：如果浏览器和服务不在同一台机器上，那么 IP 需要修改成服务所在的机器 IP，并且 docker-compose.yaml 中相应的 API_URL 也要修改，并重新起服务即可
 
     - 上传音频
     
@@ -158,7 +158,7 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
 
   ![](./img/result.png)
 
-基于 milvus 的检索框架在召回率 90% 的前提下，检索耗时约 2.9 毫秒，加上特征提取(Embedding)耗时约 500毫秒(测试音频时长约 5秒)，即单条音频测试总共耗时约 503 毫秒，可以满足大多数应用场景
+基于 Milvus 的检索框架在召回率 90% 的前提下，检索耗时约 2.9 毫秒，加上特征提取(Embedding)耗时约 500毫秒(测试音频时长约 5秒)，即单条音频测试总共耗时约 503 毫秒，可以满足大多数应用场景
 
 ### 5. 预训练模型
 
diff --git a/demos/audio_searching/img/insert.png b/demos/audio_searching/img/insert.png
index b9e766bda96f7f3046a6eafae5cfb58f7fc7a378..a01015e4ee786ac27bd54e1572f0ce41bae8f635 100644
GIT binary patch
literal 50924
zcmeFaXH=8f_crWrY{Q^1R!{`SQ4|XxgM!qkD2Q|mC@mr?ASEijW)uVkAr_jnh#)A4
zQ7NG$L_t77h!7x1Nhm@HEkF_iA>}=H^#A`cX69Y*hv&oltTnD#Yc#pbIs5E=?Q38A
z+;{F+Ss1VWW%DmfmMmFqa{TDYB};x*TC!xh!p|$fCx)GyP~d<6@i}RHWXa3sEp+gY
zWv++K4=-8r8YRhdUJm{(arO8)pCwB+g0nU%{m10w_NgUH>TZ}EJ$yRQX}ni5A>&dQ
z<C@(>{_Q7cH%K|fXKxwnH9CB_;7G8QN$qv3^**@6hi=?9jl2Ep;BuGz{lCXqd27FY
zB6a@C)aDm$?|+wl@%HhPbsK)(ee3pbho9d~Z1fq<z#;xf7KXbLFA;TvW@(iju7#8O
zx*@7Wsd_&Oe0F@_*`L6$pg$vjPFZ~W+mb(@|H4@I{qHIH|9%Xa;{TXUF1Fg_3ijtW
zd61r1qtbjl5u*f`-&Xf%jKmAtLVSt0QAJ^P#oj(aBWKb`U7R|UJlrxB+0a#djhRIL
zIGje#9a?FGTKa8uQc9|I<*;)P1Z_yZTiEk}R+&K86LQ&+{18&zjG?{Uh-o87b~Yt<
zqWm`%{Kc?Q!X=;0Wf<P(=m5BSZf6R&gPmKent(V^m>sD!MmjXV*S@=WhstdQ$#0k6
zr+X_v(*s313HRNK8>twYMn5{xC9A|Er5LZ$Ohj74GJ;#HCAp+w8<_C1z$cqVw;*Ys
zID3_zJEI*%M%%1FK|T-uq&x-VY%h9(!7Y*Vg~N5cH1XMQWxejMrM%IJ!}s9S*^{tw
z^|%oU)qE7%x>}ROwUF4!i3`h%4C%<LZE^X}x6544n`&OsWRBNLkU4MPhJCVg&Zp=Y
znC@Yw4a+M;>*j){bhstcC=@55pSMirT|o?-eAUvuE#mIl(nlTEN7UEOH`I9%3<fjf
z+?%cy&6h}jyX-!rKv$nxhPhm$$1D4;1kJc``x~qGoU&xOy55>Kk|?r_RsZm|u9#a^
z?t-L*G@Hf}ozl2ka!RYT@Y&zP<j)DGKv$RHmz%Xsl*1isx(bYh3;Zur1yXijoQx$J
zPr=!0Gg!@gTnhyTe3)qBGkbTEVPPck?fy%)MFwbIJZz`IbzJk(J~P;cm8ex(GF@H%
z4o$t>e1l2iVOIrQO|3L$b!P78+_OgMZx3|+vwt=vZR1}1M;#6=Xf8(4m)%tyT>E5Z
zzI&6>Clh8AvdZ6p_BfNjG3Ti(#=--m3S&I3@+%-UT#~d4MXXyB+Dq~~9L6G!ajS~`
z+Z&2mue7HJ>y^WuVoB~YtAvzB+QByhT1OG}-8N)%4SZ$DMi`OO7V_>fa<d-68tGp!
z;JDR>mD@hXZ6NADJIIu@FpZs+)9!jEue3_AWn*FItp0k}06UiI`owqjih2VcioSi%
z&|mt<ngfFo!(Q@tR#lavbiKOC`U8GlQCS#lKzF(0>&qi0nyw>*59dO@d$@1*RP*I6
zr90v%+gXJpCc5L-x;l}y1b6dPMSWxUp{SW__8xvW!mRCdhjp$O1Za+!Fon<6ycC%6
zKV7QD=p3xT##vXtr<gl8zs5gDkVXPxW!uA+x(`+ixeIdYC0cZqR)w3|s;Q|Rv8+ok
zp1LXMfMK5Ka$DUIG1Ry$+6bw&Yjw8tPB$OqsJ>2kyv?>7K|3*)X;DbWcSHisypfc8
z4JPbk%IgYfX+;;!yE;dI)+)4Yl-@F59*ql_=tC>CM<s4k^qK73_3<-?bnt4B9gM4X
zs=aT_x9OT6<Ik2;Nbj_j#PPRxc=+#LUU4?ib=yX``e^<{SQx!HD|TF;UOd*YP+GdH
zP}_6d=X5SI;q$|djvb@B8LuWc9d~)DZy5}I^0MpT?EGj&cJ1=XYoaJ2s^{N;07E5e
zsmg>pjP32l+@mO~*Vy;_e5WMuz-m@r;yaFf{4$%n@`SsBdDJGAV$ZK*yLq89%LZ?@
zc-cMNxN3oCxI_0u*~KaL0Sc;WF6b8Fr>9Nid3nD3U<#sL`TLM%x8aKZ6HDTTG{mNz
zA!n4bC;GDC{jcb)%d<0G(&!3Gtn^l`*Vo=o?a)4v65uWTYCw~ZARf3AU?$55>A^Xh
z>l7Bk%RfI%g3IWU!qP~&tHJf<-_^%3kZoi=?oYzE`b*uJP5kG=;U@RrHm~aWtUGMG
z@94H0<}C*}&M;x-V3+I~PcNMHkp*_fn~9((v+D`__I;)l?BFhF;4w@o$tsRY3AfVV
zWT1^7)sQ`Jgg%_6h%umk);;Hmhs`mFuZ#@*KW`_{y2wK*7%lD@)k~-J2b17Pq@6bR
zwLM(;u0HBCta7$FB({UvJ?s~IvtGK0?SWvkSCM8GJO`O=RXe(bJ%h;1!w2B5V@2Kp
zAGDqgF|h_kcknhl=QyqjBR;oaSk3TU`%I7DF~>xKKeD?&LxYjSc2SLz;*Um+4z^&f
zJ}W;2urYF@j6n=s?rJqI3y#=Fdt2*1)UG19HJ?KBa7lPoKhq}TAg@X(=__`@hBjyN
zr7U9YaZQ3;OtFGzamT|XCu^Tud7kG_!AJE=*$1<B9&Eg^8ow>P<bAr5i&_~1cXjeM
zZBxYuA7WjxVEmZ-<jDPFa~7-QNTb$5E@Ad3vVPi0A)EgkS(95Fbc)SHT<hR?S@4th
znoCOe%!Env$3*~$*gRcvg+0H*`|QIcsnt5eIYB#LNFftz-kveV8W8W9#IKFo?UYvE
zWUe3Ks>As}YUS!SEJSM!+djT4)M0!<v@CO?$#)DKab^b)hL*oCgppqh&7TQ8L)D9T
zA>9KDJ+svGk?L#gB6<F68xD3~GpA?tcTaZ37RK)r8w&}<LJY&E=I(SP9Ld&5=?f~p
zmKFG7Bh{YGc{Y3iHxb%Br(MeHD&@<2@V6Ym+|jIRmf$<=;A*@OOdb`=$2hH1$s};k
z8KDC%VA>@F{O4`FEwLD{PvrJGqI)CRE8!b9t3(Dgzv?4vc=Fp<cPnV{o_tPP(aKe-
zQ7mfethaDZ_|@T}CFg~uX+2Kvuw|%q@TXg-X=4c=Td%#aeJ|vKNjPWJ^P}7T92m3T
zqURsVY@OND6}TXHrwI@1x}EiI<2%d^x9OtIKEqlVg=veGKGgllkt1^-U|MEV@hkYQ
zs8F*&g$OSv?7`5<5qWJp{*~J*!6%8Y7=?HyVmfFA5gY7!Z|g&Or5>z0J6;ehEg62P
zc0@90U0{Y9e)rik%c!o#B{-|I0a<Ut&ssgJRIV5z>}f%omWDPrrNzSJ!f@@hjJ{*-
z(^a&MRNG$8{*FLc_RQeJO*@LY+(vS6!GgW8XGidwGB(FXsF7bq6U<g|rd2N_E23cJ
zK+GL(kpjm@TI+s&JSS+I3Yx&?9DGe44TKfEjjW=bltW(rt07;EIR2{nVoT5se=Ex%
z=ktL}ZTBinr!@G+ug#C$UejH6<tt&7dPUgo{we+Zr(@|eKar=5^e%hbh4};tJM5(Q
z4b)4A6Y7mnUc_{lkW<1=M2Kgt?y6M;H++33qp<x}x_V9Qz)yiwW%q$IA|9bPs@gva
zRyzwDKC&<(d9LNl2F9<i&Ud;**$0Kqr#Q3G-2*R{dJ$}9N92Wme|)sAP6=?3{R39G
zspy1k&{hN&PLsFs$$HhkyxN9?YU2)gur)R_wHu2@;pBmV;6{w!O_?(uRWuc0Q!F}<
zTao9Qu<sd9rz@n&*)ELV?xA1S3Fl)&-_v%ZT?cAzr-Bjgd-94s=5#KTd=QhB>XTn=
zfYEeX=M`312;<nS3mWfmu3Hv9e1HfS1{C*_JB}Jlc;QO7dGXS_77*VknQ!4#^QHrw
z>eq~4J|ibz!F#PFR0FXU_w+NeBd5ykoTe0o^L2_!%ZS(Da65wmZKHhEud5K%GOOxG
zOngv)EKs-b3g(>S%B<GmLJn`1J-(E)tWMACs<S(BxjP+jz&hO(ii*6{m=>=p&;Hvm
zMZSZblN&00(hEM`bzz()7h1SQM&BPte)|~2x2DstyxvXwXZ`eMLed1}py+l7<(?V*
zU4brDu>50^jI6zb9(87JfObycU=$XgM8vXbox@|EJq+#e%uzpWF938@S{bm`5Wk;n
z%l{M|_Hr(?+nR|l;eD}jEgoW?fi;e-D$Uo<E3}i6><Te2)?@e^1YG(Z)yt*6(1aJ<
zN~IzEqSl2JT1+yCkwR*nX1GP6F#a2Ph^W?ewVLu5u6w~qculMvx>-ME=eYNo>XlU<
z@0S<!4VK6pstc*A4wT8MUzj@!+^63>Ea8<oF)b;tM5SIL0)`sxUE2_=_0Z!h(T@^S
zRa_powR*}3zCR{hwPoy5>>Xi`vE0JE9e%QF&f(z}rD7e<Yvj%-Ax?K%hw&;&=9flH
zVJ)$!Q1aEFKIa*_=`*pE7wV$6u^xGuhgxnI?l?5|yq0hQ-Fb^LQ(mV{d`)$gdx9a<
zes(DvEzxDMvD0%Mc1VPZ+HTZ*s24BPE|YY%u=xd^e6$gREhIXoesEdowZO1XlcCxh
zAY%#H5j$q>GCHYf6->4!VYa_(PP;_!8^?Y8D@EJq3HAosjhvx5Y^#4-=5=ZN=!L_6
z)-GqasU|E_#Ne0wL1Auah_$Yfn@*WFFz6_aJ=UGtY+b!k-@RXH+T9EP8SaF8<j@`d
zQh#TmWy`V)eerS6g%Eo@gBW#u?f=Qcm31upH7vOz)@Yru9MvJ9c?|k&QfiY_TW<rm
zo$^5pRu9-5SQ_<OMiZMp)KQ;=cJ0<qaFusD3jFI*3(@BZ)n^x%QIqrpb}Ae{m`=-4
zomzn*tHk+Omven0U^OQ2vkGgGh<4&m9mc3jjnFA>;XpCl&V%jqvbjj$x;D(wu#4OC
zG>NuG58?hEX<WCUs~E1r#*m>LJj2PuqdlGt)AqmlSfwiBEx(0;{CoM`Q<v}-{`*Rd
zT80hZl;3(=a;ITQ#o1SS>^K2=Zky`mql8%GDLZCTKT+KwD9b(2MK%5|C#CO5<3U1|
z^Ss~CV5#Q44)l7M|CNE*L-uOtm%H3FPgXs_r2auz&1_B2BCf%O)X9&LW|kIUvi_K*
z9poPA>6CQvlZi{pWpfgwyX}Of0XN(X7tokE^2fXJYjq&^v61=3K1{xUdF#e2MiMTW
zc{n_qX22!J3+8@ahqLM$2vJ~!TnYM*?&-{lgEZ~u*AB&MJabAbB%rK?y%H-GCVqCp
z8kTSk?9wg$qzbFzL7W{g3~!#m@5uE@_}t-QEIaSxIZrD<lSZyM*+T*Jl|fu4e7Tw|
zN`LzA2;Us%vpxu-4>$XcxSd$aS=A;<X(b4I?h}lW^fNBPjIg;<KH}WNx=@L1)sz>w
zDjf2xoLai>OwbAaJ=_N4RUV&~r%bDk%q}CeUihsL{s&Jx%3g0MmR@@>JD|N<opDgz
z#&<fnb9v54hT6@5H_x<YX+_gF@6&X@mc5plFoy5Hg4}MEl4e77Fz`GpXWJyHAm<@u
z#&I^w?$6GSw7@m%dOndkJr=bhJCP_a+-vM)M!7y*gG?H#C9Y9K_Z?%{)P)k;(HMfP
z9bUD&^0V&DLagE`eCCe88`z;oCDT5gTsw5bE$VC)wlJTleL2*M&-a>Um+}mv_n=)q
z`CvJzZu1pa#(a`@%yfR35*i&_8fT&>eD74SZec;=IoUHSj*%^3Xp6XlFMg`|(d+Gj
zv4U?!YxE;wVqgB;YP|d8tNOh-&#|WuyyD6aQr5-G7{OnqR_>llTwd#91ZPpRD-QjU
zuP*a8U(&8-GhbV66`;rOXj}F(ZuY&-4p$iLo;k3(2WL>I=j|Q4<}i>04kM>LHfZLR
z)Z3u$MD2)BgsIAISf$}g)UNSH@#j4q4sZwNaK+aRrxNd0xSe+F>$2e=%bqGZ@JhYV
zprV#cpz-!2g<jZhVdx-X#pU3HU#pr|f|@5JJ~T56ko%!}K-2sv&g-iB&Ivws&*{`x
zzAc*=xI7;a-sa;y&n$#hHN98OucI>OsJ85i^eXEk^C=@od^1ut<QT37_n97E7l`V!
zj8C1_Xm?#q(~P={FjN<FCzafORN!Rx^|eYOj5IR7W10s!-Z+E!J~JdmvkEm0Cx|Th
zxg9v&`++@mPNx;huDH*2u6YumbzJTg<5g^pZu(lD=_DXfgdG|F)x7<}NK$L+7}nvj
zENMhie}NOvCku})l_PN_lX7n%&ZQ(dy)y7xHM`m{uKen0$M>5$M(W33>#>g!vE{Hr
zM?X6|UedgK|C>aWs_2gnvYXfkR(o2=VHk2e2mEu+O-k`_jG$P(;#bFFcXPXa<K@+d
zYs1n`36&yX1xvAU(!*O0!c?4I?_hgv<{23~^sEXusNMH?iz#aytIRGvSlJ>Immh1@
zy_x1p8l`2fXw5Ak=Oi|6l;Y>#soi6fS3*W?#sz$3mI&v&0#|>UQqIDY=evs41uZB(
zSJ!K;j2@P^s%~LHopSGdXKj<83MDLy*Tp5O5X*;Z7wED|Z9VYR&f3ly4{zE6x-PcT
zKC@`jvrbr1r*)quOMBf;C60L3hk12mgvep;RL~RwThq8--{BVI|1eN+2Vc5lj*~a}
zX|#Ac?5>b25^Mf^xsjs1^a+&;gFsE+zj)5}=es;It58)HHCok2KD<mOYvVBn?z6W`
zCI;7e0N3XO!aMVEMvUanbqjtHxB5J;6g0i7{Ovs<DZ6xy%$?yCgsR`&&QFau2#c<O
z3^Do>xwMYpwx7{6()*j;ftmR(VW<}l^1^-3_~f7vBtqxPNtx621m)h2+S(aUO@ueQ
zQw8nXSU;xv%7UcqJjQ$v=e$>C#WOWh$8Yskm%)*}L}y-+9CcZyOApz$iLBR*E`*_`
z!#56wO0Sy_Q3o^boNeBkZJ>FFhVJz6yNN`uFdv^c^$Jz=;s<;q3l+A3uJyMF4+WWQ
zclW8z4>wIC3MkovTEU!5^2(FK^2Wy4%{9NM&F(#p*SZ_<ydtow-Ck98HR2O9Q6@2p
zJn$0<_mG@to~<S*_597N!uMM0Iz8gtO^iEkvOAq?^GPz(<8x*#sq4<Fbw-YTTafqE
z=ZXwM#)-iqleCp~Rl4m*OtH@i=L?Js;G$e`XlZt&4Pm3pMKA}_;fMIg6>ORgcLAe%
zwbbX`^=_9z^kASO#%$`zm0peme&fy8Z{MrFFRdDJ#7FJmX<&-yxCOLAA-AqvmYU>`
z&eloh3Y|{F$m%du674h}j|ri4VD1zO!=B4Untf%|9#zm~m8^sZ^!k{M%wto&+3AG6
z&|Vf)MyR;?;O;qt($HQuM&?`fdZ9l<zT=@qrT2&sg*@s7D%b{RP{DR4bZpV&UDJ0L
zz}QclT8{B=#JNv=tX<CTmyvF?b9~k15^h)JJ=lrd>U3I<x{G;=*b!?{vzZw>p?5hg
z3vPRrojb=&THVu{9E|FnEm*U|+rlnzwyO=<IHG9u84?EyLP*v6*|bgGPIjRtNFGzM
zV|XKc0hqszu-0YlOB`Q2zRInb{X3<ji{RGA%pG!XisGW{->QHd>`c(i;S<Sq9*^%^
zA8~e)!!U=QBUtrXg(l1z+Sol4hp8>z<3yKN6|wm2HpkAro?$9&PD`W(3mPx(IBhAc
z%AFP}tSZP58k}hlSM&PW31feS_T>7!pE=#bjj5ea2zeOh(8%za${MV48Ux&(mwv^7
zlLCX?ua~-0RhlZyjb{eOnT?m|yd%K2y~i-82x#Di_Ep~Bl|;9#I1LS-R-8+6ZlVk;
zmm+DC`?S6}?Q5k(0?_-U4El@T(RODtii_t|PK8#Lt`1kMWw2qDbGUZS^=}g4b7cb}
zulVYhh8d>kwJr&u9xmOf6`JDH$14u3Eu{gfcCZOCM$>F^)!I}Wgy3(^$GC*2YE3G;
z=-P60s7SxAgT9Zv&yumo!;0T(;E~P!8B0`-5ezO3=9l+>7h@ruknqi3O3Rxc#4(MC
z*WO_gMGy`DAGOvBG!ZJSrz=Se5CG&tvsmdzX~9upFI)-B>BXj6D<UEjR=l*+8F>Be
zE=!(HSv;28kR5r)zrL&3Ej_^Y`{xxlfjd9P?a(>VS!|#J+ZU+t?ejlXs&#L8vn<1g
zy3VIheo5e}o=q7(O8l-!y)p+U70#w7f2#o<fkeun4v&_&vzO;Rt&RR<uHS<GE-}jP
z!L9p#tMmEQy5BFbBzO0}A46vO|6-G2bz7e8g$2$6Rrq~%mRtraX?8UFdnaLu`5z}f
z9Q7^xZs*V2|NU~wzeWJw@S_pBCXFSKeryIwLW>_zF=fknx`$B(>kXquJRg-=DlOHE
z9d2GAsl)g+pIx$P9}fT5rE(_x6RcLP37>;YCom&s_eA|tduVHL^TfHuhj@cSeu;5@
zce>Fl{a1O_@9wzYy>gQh%YnU+1+U%LyMfS^6*xKK<8Aq*=6nC@&mCSe*^#!a-n7Xo
z#3&>FbN)biEOAZP<ixQ>S3d_kF~a&S>q>~?jzx~2+MXTxX!G{dUc{Qnk*nX)<*`YT
zf5K;<b_Pjp#bxh%R%b$+2Yig-B+reD2GtS2wxcHqsl6^`UX|A#)=%D_cI>|am716x
zxfFTOs(EQ@!QAF27_ZV8sc)Hi$&%Fl9!3&EEb_Oo85v>5OWcCMmlJksp=7m8Z#dj;
zt_YTZ-C;ZJcN9+Z-0<HUK5wx;LOerI-26N5!uc!PBKtQ)M2^nq>6mq9QdG9;IDEI3
z+ew$~NG*=E*_dpW+uUzie<V5|VTBf*S!4+#BMBM7Xw}RcA?u9RZz<#ZvSiJV61C(O
z4}aRa+Dv-w?uYU_a8DSNsrk!ki>#xebK)Sg1D2G`&BNT`#oYS1YF$i#Y1z!=0%<35
znpzb6c=77h8A%`}6_wSPmhg3(wx8ay5HE42_IQ5^TCg>Aa>wGEf}{^-Xc<7|oDVvO
z3{L$7hTrad#Cm#CqL#BkAGvPPk~e)&?mtCHp}@QMB_2L<NheC8_Va~O-iheN<NV`9
zN^^ioN<28AB1)pDUw-iw{6Z71_DQd7Xylwi=&Y1Tq|5H$O%W7+Mu#jO)aaaY|0nxA
znyr@ag?m<y;+>C7Z)x!%CM>#*yrPmr5Qjf>Y2l+SFDBAbFhH=6F+gnR79|e!O69hy
ztx#A6qnX|LuLHCJ^PX0l#hzjXZOfQ!;xFx|Wk|I2e{c35dZ=S?BIV8#q8!~fgSTPb
z;G#1xChUvbK*;}{i_=)V=Jm!B<bW0oqHLjRWerjHhiO{jLJsA{8Xzw2h%sQ8-Tv!1
z9$^KqaSqgS(d`Iqi~nhRzB&UgiW#ZI;x2ywvbOTbQFOrK=mEybvcg3i+8Jx$nnQB*
zT?qSOB9|{FOg)GD&e|S4iq>eQQ+i;(hEpCb_732Tx8Q;-A4byDM>sFQUgz92{}`3C
z*joF6M-MnIyYAgj6#N=a)lmAcv$x)jTl-6f;DD65A0MgzYdAq;*WxeZ*Zu;<RM0LM
zDm;W5LC3rlgw5fba~3V0hQSHtuQTxywZHXm2;)Ri2hCy^+2@0D*3_`z2CEUKch-qQ
z73>bbRO1;K9eH9=I064o9B>2?u%**!chq`G<ZSd*i2KRKM;L=6$iJlrjVY261E?^T
z>awDz7xr0)nuF0%KUeA=g)ew6HcH3cwJE8K25BVmXwj-&Ot^RF-!K0)!jjbgS5GK5
zJh^cZx&n%~*zbW$t^aRw`KY}0;UaL|q;zqS|3Y+dQT75iHCcpJch~;0DCQnmA6bl^
z|K1cz%lrTTXWpC%?}V#kh1lFQ?}}ZHK1~PcZyHtQPWGtwgfQ?!*%g>Y0MlKooF)5v
z=+GXxNn?_VX}x@aLm`t0L%O~C?6vgPq63S96Rhs8Y1U(Yx*EDKQyMu}W5{^&S!IzE
zrj`IG&sg*1^}C}R7%#2v%3W7mgx4?Rl-3jco!<ZocVhabge^SGZ+_Fv-o;03R$3ns
zNems<=zn#-Or!bWHpkHET0=r+mQ{}`WzE*$@vGX>i?d%-zV(q_-R<kMBNX`~#)dVw
zY&H;bvt;)iMOIxOjMGX&H5V+Nz~zLgAvL7-mNo0E_!Ed^)D=wSWW}D0>kYg5eKrsp
zvgQK47tdw`n2o&DlmEPXpkx|lp!vlP?@>KZL&FVzM=y$|rWorZ1>bne8TCWAGj*}>
z!9{5Qd>VWWSjq^-)2$uPp`<0L*lP~Y`-ET@GtxNJ+A7|V8fw$3_1kXJ$43h=3QX-^
zYR_*?UUa7e1}AP)Y9Gt4F$>UO2c_=*ujNhxMc#%`k+qI-{#X1qLkmLnV)_8SltbXI
zeX{DitI@0s)4?}=rCoWl$S`s7Yh&Q(eZk6Cr)y=B6*DN46=N^e(FJpfQss+a1aQ>q
zOUaI!mzAx0$j^<CFOR4$GS?4a0Iunco32yeb_1+qHgANb`4C-jXVGk$Dy*yDTxx7U
zSs;Kbuv3VazPJ6CEXg2^DECuB81fqHm3xB63Wv7kVj!3NM5BMj#H7#{UFCqz3FVl&
zpJd>n*XCR3^p};BpiTTshAeFVVoqFVEYX5FqrBemCOuzP`);^E$uucJu_=484N$-a
z{Z^}%8KG7fFdq>UP$^)yPz$O~FJNmP==CnP16WdP^<QsCEkz4?v_@mImjh&t(775#
z1tzZc=O?m;Ur#Th2j|pJDBG2Keo2`=)X!rQ(QSV|1{A~Xo|SU=rK5lGw*wFnTKxn*
zXr;VDlI-APKL3FWtwdJ9+1r_UR(H2t|4XoJQd++OtoLe(-L(h%_rT{*RM2zNqUu(j
zshzk`Y_&@22~4X8OW;q;SPm_=#km6~ls6+%_BqDDC#-O~hKtuQ)h|2p3GkKILzwF+
zcL(Ix#{?Xg4MSS|wR*aqV0D@o9OWM_Z<ukR(+bBta)y=S5Ve@%UY=0S$`2$*-icoq
zqYgPu(j@TxnRBIWqrEcMWf!x|RCab`{)O{axUs|PRtE>~YP15cD2@Lmru)`~+g4ht
z$_7be`xd!#tG~vHl$Y1#7hhwv0<1{<Uq<PueL}gxD`jycdKgO_(_Xz8Km6FrS<1=I
zM;0yDOGTyiL;Duf=!*$frTD+V&L*W*1>PyIJiok|u`ViF<U~di*)gD0@nZb1E3b1&
z_;r71$}9a<YhxC<?Ixw-XR^PqC%kbEk=?%jpQGsCJPs1<pF93#ED+Iv2;u*Y-0bv<
z70@32C68U62w}UHD9e81pH}6Js)YujjNyarCg8tRieP~t)a3~u{`BX~?>uuu_A=<J
z2iyI21C4(4e@;|CaZhy0$-F;KEO{LN@4^3C<6qzSH&XtEf`5VW-!!r0-=z63D_COn
ze=QWmgiO6#NkW7((^yQjP|vIL)z&cK0@af26yh~HHf9Q&dnes$89wvSw^0R~Q^}~R
z<gEi7adU?CPfPs1%azY$m5x8i@t!~-c`g;8gl?^sfu|b?rcrq{(lEUVf>1y|M+~;%
z#uDaYSv!$4JKd8V{Y{{Yk~A`>@3MF_7!2RHT<y5e@Fx>u2!+`S7YxC3cN$FJai?Vj
zSV#wbFl+@{gAOYth&lW;J6>mZM(E5(dMunb@cHrK2iu?9Yy8Me+f+6%u3zQS^YXCx
z2`|I!n-_ei(rJ1-@d7r+a%cM0BRaHnpH{7^={Q!lB44@pTD|-b`U+-&L4bL66}!8A
zoodH~`;9R7+7&Cy2lm}?N9oTVmk#NyX@&0U%W5&sV>as|MjST+HdwJ$3<%3@9OmG!
zFC5buc^nZ%nSLz$+m@iO_vzzBM1{?eGV*9NGi1`l)+y+c4!i9Xi8s$YWob$LT)7uF
z--J-uyz83~qh#`6ra}2Xf9ia%Oi$lkyWQlZMHr`&Dj!UaxbWvx&QASZ^4H$4p+AXL
z-5e|v@ug$6R?oiu_W}pys3{6SrprsleO01AMDub2tO1Wd2Xvo9^4~@1-9{1*w!4Qg
zTwx36xSjO|Ntx^yu1{ToAVYt@&eF`*(eKQB%S7IQt?^!mvV9X(3zLRRZy|a0$n+`9
zeQ>c<uT4rztm?n%-G9bd-`sA}<z!!S<xHv;!YO#%MKO)SpA5)jkCBrVzX&Q>t)?V<
zDz}4`hveoN5(fIysJzSbYfK*GOzaUeVE3t7VfcJBaxR)xoTslcfu)Ai(ZYFjBkJpy
z-T<mTD_(+#6fjA#POK-sljXD|#rxpy24Y}TtnUkaC|5dc#1L{Oh2WJp(kUCVdaJnY
z7%9BQ`%`FcTUHhrdF$*wF`xe8Fio&c+I#MSf6ujg-%slGIcNbZZ^*a;y9a~&L*z>$
zHwFg$GUEg$^5v2EHM*YXWu_`Dbk^r?i$C~#@70>sL->4db3oNYr-d$ZE_5!mdb=5k
zz@m;~7ESd6D(m^?bZtYK9!wQ?bjGP<)C0a?7d#ycPnSB_w*;nU@?ig%*u?kSS4@<4
zw~7pY*Hv#DzmdQ*4Z?I4&Yh7$1-E6lPS-2KDcejQw1tAlGuza^wKN`INchd<!DYie
zwlB?4;jC=Zot>FyDcK)c1#tduJkW71iNPI3txuDk>?nJ{xej9IrjC;d0zVzrXG0PY
z6*D?8a5$ylxebo={$VC7r`WCK$&3Q9)Mc>&x1EM_Ta#NSD;b#<2m2#u6wF-%s9~JX
zn3}+rSf_-S_}MOzjUP3MMZ`Llb8jw=z-N#X9`HR3>pUIiCk>L6?)bfCgN1R??H)D$
zcv7pG4R4GeA*CaVZ~AOELXB}IuW~rmqZQ6?hd&J<Ec6cP1FX|{D7L1L^5-`EBr=4^
zh}n<Efk1B<OiH69IPphq^K==wBAhJ%rXum>yaP$kc^Up)A2GkCELU;D1-fmSiBRM{
z#u<YXz+(hIMAWD+v=-osK3p!D;6%yJ)rDj`7X@Yts|gWaBsxt){IrXZtCCJ-&LsFY
z9Uv!g+)7!u)IzsTysD9+KQL9@#Q^b=D!99L$upO4LdT(6Wq_rA`Ieyr>*(6VAd8Cr
zD7l7^{oNJgZyKY*3+s-_IG%<aW<Os1Zbwa%Rm~y%#`at{4MJV^pbBQOq-+KDGvEGO
zGD)1{54M}b#e5~V(lTuH6g`L-;#9%zux|u6s~m{g)4EMz`gRta4m@#xhS<GQn=N2`
z7K@b!)9(#sHVx@_IoVbEdj)?mRCoaJ9FZWpnZ6jJklLZVq#&HD&l|%Ks&K~L_9BPd
zra1rMn~9)7s)(Q0GL#I@^zU?zW-){U26R9B^6QtnTu}P0Uf5n6kxAyo3~62h5`_%i
z#4`9~#Z<LqV@NSRpw(pOQ~qo>1_ywB|KFVw#_82ku`hjrd+MMw{;H<+zOlkN3h=>H
z@i@zERj=RjkTP3}O~fYHFN_yiLYbt|9vr=nDCX=Jb)kHhuI<yx=TPC4?4D5u!%}ym
z_*4aNrm-;uB(WVyVs8GCRih}_r_aigHm5F7I1M2Q9zq^bm=2|h)s9wB25=Kw(;cKn
zxL~IELT$`U(se|C^)0$QBhWa=6gW3u+(^X`$k6>TA&x<&y`Ps#>)p%bkGtl@s#TSV
z92+lwN)98OQz;?_1Q1D$A#Ey-{$DP+b9w{7ZkMotAQZszIdQK!UW@TMIQF8b!G5%q
z0%nsNP{qs`QUOpjEzTai^D=|4==A-$6dYX_6Z?z4zCK-!I%fo3{A8HeoVy_SutbG^
z-F6BNhX<i5N+=Wx>xm5h8MM-;(PEbM*Wr$2lfWiI!Hf-Owl6|jiasltMF=&+W{c1&
z2%y20G>ffaps2U_8WrZr?PY}9Qb-wL1MASobtlUP?+pkihXsQ$!8U!syDT}B5QJ!`
z>0)=D4(=UuO197*N&t6ezg;Sw1K^tN9xd-L+kekcYL{Y52E-@aKR!?r8Rwu_Jotr;
z*;7x1jK4MxDp=Spz_z|ysi=4Po)vxAxf+m<djjAhdVGLYsu09%P7@nKg_PdZTi?1(
zt_j$Rf&t-D{^JO#U+RCZ@ni1m?{q<#EAskQiu~9`d|~pT#H&ByZn+2HXS;OiQhEs{
z%<JHh+rDG_`a%EogovUIH>HVO+UW9+Was&rXoX<6Z-LT#YMOB0iJ3}{9E{NE)iFCb
zaC=*8Jxnk?lLS2IgP7wT;{(VsuN;44${kb`o~4lFdwq!d?i5^5|Hxzc-h*Z%fG=AH
zR<CP-VsM+-*w?cK90E)p$(tT3oE!nYBClX}0bD+Q^^PZU(Gti7EHbSbCgh3kUMhC?
zxJh7UGgQzgmsVo<Y|%_6uNxQS6AT*3o8njP$nK7kNOrhx5<K4scY*+;OFWueHV6m!
zV8QnyhNzAKy@|6_r0}Z4z74&aW+QaE3B>rJD0Md#b8jPChsSV0>w@+$CUMMWmKKt!
z)&K0gjxTASGw>677$XG!52r>2kj%B70^(0lMhx|0ko>V?YWTt}mz(r9m6}L_Kv29e
zS(%Il_)~iE$Hj6F1l%mI)!JD845a}J2Z@!w0TX!a;O`@m-kWydSWnf=lbwRyp*w#B
z!4kZfB)3k@0`|c|2xk#Pg?g0`6KZsteREasvq|qhvpKMm8S#2EA0T5t01nFjODU5-
zV5>C_T7lrq1tLJaEM${n+=|ObCTpP9uKEHQLcX(NX395M_(IGDH4hG#2#2tV1lY&A
z_~n;Rf)n*QJsOVT^FvTjwM~e@Q+{xhhtorwLOU?k0;thRuT=}bc5toRk>;96h`d6K
zd@lfo9B(mjKiLu?ZIGfc`zrb==&%>ycA~wj=7)>Hg5DbZbH|X5*&GKc<oqujRfwm?
zV`CkKa@_V6<$lm=$y80<>Q_Fm`N1#c70E*Psff)09IMb4!-8XqfY*Lwk`38%-ep^x
z9%<~{vh>pqZK-u4CZrZdHx*5}No>lygWC(75+OE}j4c(0b4lo20=Izhi}8{oYwwy!
zP)!cfmnzO3g4Wm1N6gwW0Q8f67bdXaY=)+1=e<l0j&@5%=vtHxV!_e>To=d&5R5!e
z6AMKasE4+HLlG}3f36;8K@4j4sg4zNM}W?1K(C)|@cV5F)=(hzibE(!P6zRmewaN7
zFDg{s_SXs~OjHT8=Ru7q8%+KMEO4*bY`LGK4H67Mi#rKhO08lSB5ujdV~w?aKObKC
z6gi)a+$CCDacVoQDk187+jk?dz5e!rH;^IBYMH86y!cXvh2HdgU%)96LHwM=ih&)y
zoEYZK1)u|`bXDmBLS7@u?|uj|uAzXUumHh4El-Ou-y~X|La_k{gBE)x$h6c%U|(+n
zf0l(5GKA(30#!>2ioOggaLS;PD1rKMmb$^#;l9(%gXDz2ABkT;?J?fKg|d^}55@*-
z<CpIk1Z9cM{EOah6>QIg%&Ewo|9Bz<OQ{;J7C{A-pc5?+e6<v8*PnX3R5a>1u~GN-
zrXKKU{m;rRR+Q1nyK8d^P_4+PAqB1?H*)#WjZOxL>;Q=b0F|4Sow*?bCK?UYh6)H9
zM(chA;+)CJgi{tK48U0%?0Yk;t688jU3ls@)*)I2EwNSb3xIaZM61>{Drc?Z_fq+e
z+yNQFS&?t0{FoctdgTLpuF=0fv{Xg-1sF+?s6ZJY(F{1{|2P^bWA3qeq5S>${%FNa
z0adtw{6_<|7%OH7=xD)nTsv@6GbAIFrb~L{nkgchy?-|(zcy<+w3+4GEzsoT?IvQ?
zTScWyy!MWaa4!A39d&0uY$JVfMLYh}Oz|{*h6#H9lk0aoa%Qmv{fq!g&_9k`VrTk2
zu%XB1`_gLR`d8BjV%1bdq4ViSzBD=s1SHFxb(VUszt5C8++^y}@#0&k{mt=ET_JL5
zY=v%iB%Pt87GzQ})38T`B64CVQc*cj$fnNKX(fF>UQIZoMv7W32N8hhe|J3alBa;;
zW516_#3eutcj})^v_Tldr&}U}zd1ieFh&sUnJ62K|A&z6&5*3`2K%%rtDFEJ-2c(J
za(ex{ReGTvK82G_%Fe+eL8icooiFk&uut}fQV=i7u<-Dppn?z(dlZ`&XYX%Rb@=AG
zsbW`$#J&u_j&t9yt~;FK5)}mmrsp7Ve`pr5z@&8W3%v~B<p|*A-Fuv-W4=|hP@Xv;
zmS_Hp=S3KZM7Xy+Ic5kS^xf>32R0j#_id;pcazDsAm4gG+9C+a5K+8?hXa0)HUL?U
zXFW64&&M68Ol{73d$sl$zr_zc7_9G|uOOI;mJaJJDnx@Cgucr%=)^zeo=i|%f2$%7
z=z>a0TJ61MG>F{SyuICafGEv%ek=o&qrtVemCGKhsW(ry_bz|RGb_r5>H<KGtDw5z
z@A4@EkZY1QE&UWFRld+3tBP9!bBxJ)I~61J+V9$&KJ|LtwZ0oGlZa)7r!2IULdSg{
zc||-3z>hh)&(D107JrjybwJtE+6_v@nU3)hAmm&L{7-nrdczwn{F!9JmWs!M_d5;L
zc^ZD1v&DHOyfeQ<Z@GSa&Na8O-4^Z`p8LgGeX?vczd+>GX<}=6y4blUij|P3_Y?<g
z!BXy5M+KQx%nzqE5<ALc0v_Ks5c;N?XGeA;2>KI%J&JsHLJYX`gGW!HQ$aDPn21fq
zkg_AIA)$4q$nPbvt2(cT<gOXFRuwl@4Q0%43D5OM!uWhNgAwXFw`X<gl^tL!Jg16-
zA978n#0U#Zhz1LhXrGFvT<hv?AWirJWf)?eC$WEX5pd%RT}UCn(Y{IRRCeSMb%inr
z1c-WU)ryfPQ%v6m&&^5@KLk6#ym$WtCp$AukQ7leP@ZMu^1fUK`?wx~JZAUm{9zpi
zj@gO@BtQb7pTR@QRFqbZkWCQS&h=#Mw>OdiY|J!dqQn;RzQ19%6aYE7KF!*BxOPu-
z@WwaWqDu)=lK})>6~~HV$gv<Mt^XH|HxS|wBvGu}08OeG0SXL%Ag;AhffxW-4g}|!
zi+6VHuQ1jhwWiZxLJ<Asch`1l$r(w&fnM1P7qa31MA&_e<^4{B?RV$EIoG!+t!He1
z1@iY(Aa?EI1!m2iolY5e-BQ1;$T?vuR|~&Olr5kNTt&<sHuomlIz7q+_DRB~?!j7D
zZ;jq^GTe0+B%jj%Tq-5YOc|&ea5+<p-o#Hi4%`KDFOgCK_TV8z)t`7lvFi#0&rt~0
z73^qqPne1iz-9AZmihs8>~4_5HR?zVOqY8t`+)w5xvx86kK#c+2mxcOcrA7;16ZHV
z5KJ-}?3)ru%Kbn>IzI0hHx5VvrK5hmFyQpGYc$L@hi+Z15Hf!MWyPonWB=w-C078e
z6Lnb{>^0z8ch>^FF3&T}Y#M3YG;4hXfk1H62=i&8fryu+C8IeHElwCK1YHdB7>4FF
z?g|U}^cFm(+APN`8>s5vfXtmM%2w2}dzc*wH3Hn(qwP=t=G+&v_?333C=?0Z&}LrF
zCr@qgJP6=H9$b+M59*QEv%S(Ws12xH3iiB5BkI`)4xE^4u&sJ9+v7tNjj&TJhqU`u
zOYF=6)F2mkporg1;M)iOC-Zfckk3ell|1qSE@+Nk=%I;9X7O_WG+ES{nu>-&ovAFG
z>oB)p8Y%1ar|S#Ip@%HJhM}iOoI)nfKwcap22T5(c4qjV4DRlx*icfx#31h|I!3yt
z8A|IH^W2*}!E-8-@VO&UnC<!-xuq9iz&HI)q>)`N)gbC=;K9%>fyKAig!U~}JPs*{
z4d~WSHt8S1rqBk+3Ma?JtWW|$`orPo413B+En6PqgK!!qFcwh@NP4pq2cg~=_|vPi
zUtE*1ZH0J77{k<9G0Ybl=yL<H<Tzo&Yn-dspS(r~blx1OPlW>qra(KV!{ZRpk;FT7
zcUX`Rp>p=sW872kv8QTb?sh=Wgf8yR6C1jD&Pv75efnUJu33Ri;ckXCmNkt+>haM+
zCQ`I&6XG4ECwxFuJh}kWApt&j4ycWL!o6xlIxDkDD;d&fOOnO-!$;OT>F|B>!6R8i
zDnk@VstlrK*h1@i1tvD<hZA6Ld63Rf!l?%Ph7zbHC=dE;BG<knKA<;HUvr|EWZ~Xh
z;Wwn;P*~Kgvq#i&l-8R%BvK6Di!s{$i4fekkw_22fliaiQBHnYHW0&dMb5g?yF4%|
zA^|;6j9}z;85_*pXXV@C^KnnNh<Y$kkBJ3}v6qmf>3&@-^ity~GfCK-0NkMbkct;W
z)a!ClTsSLQk{R*CU&cs&Q3PD2(Z0(m4TcQY0aRNgn4J>ua`>{x;6fdJPC3yW(lmom
zrC194Wx!L=_z;Pc`+@ZF1Mf3;g#p${0yXkiU&sYHZXtBgpy>Nt>DU88LevyFC8k2O
z`jW}yWdB}2vM(iwmI(R{*oz+@6obT@1N8z#XOYBPDHbf?-fAM%67)ItGFm*nrUz=>
z0Z?0jQovQTHY~9feSXRjBuM%l%g_sUpvwvx4)%-?>Vin)@g+M3B#L?ffLi{*{&dbE
zk;vd-4fG=m9eU1;LQ|3yj5p=`_CG6gZrr#b>J-t%W_B`9q#r?vFYlHlXl6Y72-U?;
z$++PIbU@#IL{ckF+=14K5M(tX&@dP@roa=m3zIWWO};Q*zcdMS+j+jAt0)?&pQf0D
zYkUE%B^YaLOBJBWNgTIah*x&G=Vi3g^UzMBzMXgjp~;t)8RE?)!T2QKj&rVwoM$8;
z9Vwmwu__?`Z@0+(Z+dS>p&jtS01i{2yvK@$@v?BXq+WBa-ftZ(s87}#zA!O_2l1#O
z9=tp3%dR>H*J)(l|JH(WtJqDsKd=TBc$5qFnJ#?m!HHXzRA^VSTfbkz^0;XEfc|eq
zDfs9Q?KJ=(y?>&F>_pe4&RYaw0IrHRxl&hqQE=J-$$rxZ9%~7otCnd9InV;>a8h{P
zYVagU1B9=sVzNrB=P1O3gDK3+WGU^h_gFyDEyLLkmU#D8u-+u&_u^*~F=miWM0aM2
z$6y?vPZfp*ch)E*QG^0co=ADK&F+5XY#7{VPk8|4@f@&400Z>_-Dkzc&C*hLQZDOd
z0N!JSye1vUIEKRRWA+enjt<v^p7$X^S~Lsv21Kpxzv~Q7usNK(8Y$hq`aoB*0BtPe
zRzQ}m!*dpPQ#SRdxWt<PtU@Nm!vu3s6a3qC#2ac|PFw)#>G3+F&sg`O^|q^kW<RtE
zL=E-;2~kl<GVcOU6|_bhu(DVHPAYh}0WTUCPnokFg;<IXXw5@1cq^0;+q%2$!?$BJ
z5tMY1Itt04A}|7yg)CRt2Bd_pOtHu*)(-$(!`!7(2hWelFN+DVo*UL8oD09W;05@L
zAtMI#Fm-e0nY274pNv$2{ciH0m@kGAyP`F{dr}PMKU(I3*0IM3c)sFBwQbc*;>1A4
za!AL{3J6hN@iQv${kB#ikW6$lf`%U%GNAr<F<vYq@4|xpD(IY{#^k(mA6hPC3i=sL
z@EM5hRzN*VsIO*_Uk)L|YW%{xf!bJ-DUig92jg{fq3Fp7oBcvB?e^ZI2pD>Tu^2R6
ze<`~plx_+?0kVBaH04<TR_UPQU9RY`*6Lf5(MC(g$QVmVnVcV?BS^XCP$r`*3a1rG
zkm8dG*iar!$b?w}IUQm5=Ay|)g7*=?7F5N6rMC@wE=rZan_)d7hVGCDjZOM7wP5Qm
zkZnmo_evEB=TTa?t&l*oiF@W(sQ(OwRx9*45=&JQ>IJrf$khD`11+ak=&2VL8-vsb
zv3zBOtg_d!Vm~NI8Jocl6&YOLa(W9Y+&8#qP$Ah#Fq51ol8Jy!KnIUqsS?%#E;{+)
zN9pE4>t~Gl9JED3wQ6oaq!xrhY)f+E-s6CdT!pGLk}2R`Spjy+bt`UN_%sEA)^o(x
zaLe@%TSW~=o$2@MNHH7sw2rnHMAxkZIwzvR6?oeyMJ(TeZktJl*ftyR;e)p4{~o+V
zwCU(yYb@?A{cDYXtpWMKzqkQCA@yzJ(EkBzwEDJTFL<X`3f=OaR?Q6dw&xZP<KGbn
z@g6In9N*?BM#Ibx+ym4?uS0(*x|^QaN0i>I4AXm92y>4vu93=CF&}G01=yyYWNZaj
zxkq`9TDxLE1L6KVW+zBHQevF!K${Al-q#{iD7mu)xe2E5)0XsL#4g2s5#SL-j8Ar?
zxr~C|Y*2lPeeiJj%Dh`UIk_crshbMH=2xJn|FBE*igI@3-88T4NbA{y2UYMnFfUcK
zPseE-GrBf%yU7<ZwsIiB`bb^VjP((BEPrEONkD>S&vEc3_^6WSoUga&efW?Sgxt$c
zl^~@9{qBb?VyiDD+`HCYuk8H6n|X@SH1v>oKCSA_K7Ai-3mgUQ%G4I?N`1Q>PuZE$
z(ny+lXlxhCdgngr<1eWQtENQId@biCE<c-F*ax2d+ExE!63mGLj~|ykQdHWD&ksHH
zd3d!)>x^Lx9JOzjcxHH2)#QqXdg4(dApQGU{z!998iJ<_d&V~_dAk3`MpypEUWW9K
zeH7LUZ6R*c7oXXIn(W9XxsKeDF{3flv?xAv&fJ!rJ5*~gU=S=vzc^!ApZLSM5h$n3
zU`*HkNx~C!m$EZ>E1|caUC_*v;`73C_sMXgS><ZMd?tRSkinP9IkppHR)CQcddT*^
zZ3=UrP})P98Qi$afY|4=n!h_Dv)FPPsaUF<;or|)Lq%3@VCcQN{;}G3s*gEdR4?U@
zrL8icZj~gtC(q_J845{PG~?Hn7Ef2t8^XFrw1+b)Ux%LVX$#Mca1CXbk}0D~KJ}pj
zd4KQ|7MRTfu0tBv+kHB6DR{Hy8IO~Um|FM#tr2MC$N~GzS^@AWQ=8laa+=A~Ku0nc
zU$nMWsI^xVE6dd~G@H8@7^T#K;@Nl=;)Vm|*42kb2^ZaU0#PHx_>yj);k==-y2=*>
z?LzqLeN7%r?bz5+$>s?JV(Mx=_Zo6_brPCXcTV?&n|6wFB%JEY*Yhwy#6_rWncuM5
z^K3>^m3yO`QZUbL5bv#Ia*`${tZlvmWKm7<I(SaJ!NyLn@KJnuXtlL@V8T^uU0cXm
z{&GR5xBKK9_VXda4FuaqzTO}L#_4jdZYm(%^jG_&_)1xY3wy^ex{$E1^NgvzLmNDy
zBVMr8tefZq_Hj&)d4f0D+k=o7t8sIpQo)-7wB6D8SZp84prd*#epm2A;hng(nz<CL
z*%ULr_4?6$A7e*qi5KW+R4z+!%+8ltyMocKu)K6_YD+D~@@$ad{Jnhlg&@s?CzQ=<
zh)!VJpBPVmo^`oa**UFu*8bi`bfr$Aqb)B}aLircV;8r=-!zsShYy6^x2~=$qAf?M
zj3;Hz=N_XMOOjWG%X^SUePr@(tt<}Ky0vlJP4Kq=-qxON$$1n%zB_5y870@5&Q)_v
z9iGtS&aaTi?+9EOb)yaIiGGcee={`Z81%8Lc(#?|+jVVaUgp%br{^?Ixat2Y%hV&y
zGJ6s_Z~*`yk-I{}Rtxkva<eIiW2;&(fL}^@=ibscR>A<k8u32ahsiCT=^Vig=M0e(
zr9qa`5bM^pSruJQ_&iG9>S^I&>2^)spe&F&tK1*o?*1;Ne#5LutXeE`S52Mda6Q4h
z@6SXRYDs@myL(GW$SJD)(8R{o4yDS0&3$AUGu9ZsT&oGC`dQ|3lX4b&%z0edITe|C
zX_~W!;2(_@e2pN@_@#{^+w1xYE)4xA8yyn8pEEw9ZvYDVyuWuVJDyHd*5tXgjkjbv
zWl)>sih-N@S%*1IEFWxgZ)#~FypRs>Jyf}RRXD;+4_<%?K%WAyV7}|s2cpSh%C)r7
z`bcHTJo82BxN=snW~P3?I8zbppROJO*gt@!ze++U4WU;`@Csz3u;C^w+6q{G)6mj}
zF7Wn@!J8x33WdkQ&P=OK*9*$!TX|=M1pGo!;R22+U<iLZcZDqbMtyzi_=U2pxUI3W
zC+rP39*ERTQN6k8kBARXQa-OeDs#B_X-bH-neO_8WAEH|mBrqCcx2P9N9!xiRz|mO
z+LZe=)jjIUTc*6q5yn`EfiX>oz!Es<(2gr2bM~Ni>Xqfa-rGpiG43cNn$q@$KSClA
zY7_c6t>8N`WAy7&g3D?R*f1Qqbz(qcWshkZaT$TJ9v@mWkt|I>U7`|au(X_Yib|n4
z^z5uZDT2eutz`6^&~gf&omzn1=!QSot^(~$R08i@g5I}q5khQs+jChZ2g`r~TQgbg
z$g!zfBMDBc5$ls=tixWX*;#I&n*Q8;q3>Y(9(t<}O<GvBu?UGty7n&l7ow{Ai%Xxb
zf_HeUATj4X|Cl`LO*E;OR~i>O>w1MD^c_R`PgZ=My}pSFJ;X4o1Pap=ch~NQ-gg5U
z8{m`Y%|;Toq=~+Uu?yKZYBGQy>3VGmPP=hS&>SnE@L7EmY+v^l$#$R1LMIjEu$(ax
z*FSO1&Pc+GgYUbfZ81BWUOc1RpFL4L-#U<edy{B7(B`hcPVn^HW?rMiL9dWs1c7c-
zA7z!sr48a^qrvYFb>*o&HC1ykqeZ~&-4b5Ctq(~xme7TjZK*=}U??U<mg(18fk_jM
zB&4ye?c3_t&`7m1D-?-&WGP`PlxFb|qqt*zL<xGuz$56Bu=bggD?E&)mrGp<`b8k)
zeM?Q{)u^&FZ6Wz46`Q1KAd7$Z%q4E^#=zkl#Q{4VQQ?fXL+6G3IM#R;_MB;QmA^)B
z72)CR?5r{3%X;Kq$+)%gyg>UE+wwpMZN%FSQT_@!kx_0^eTSO)FiBBqJhw1;m>S+k
z-{U?wQAvL{>69wCTFpTKP2;2*lw2<)Scw?hW#|+5#8X{Nv*UjGU`;QIBHrZrLfX6K
zgEz>GT)Js(dEox5`n`|vQu?;t^?`Frzi*UN+N+h5@_dL9GGsg-Fqv10Q53LUshv^?
z6qrI)vETdp?itQF_+f#ErA|_rhabQ8-`1N^o>ft%zf%P?)%b<JyMh;@v*GAQu+cn6
zqy_(VX>56Wm#vhsY?Q&Lq2Hs_RN=yNxIb<iFZH0;RZc4nJ1h2Dkfr#UH(WJI_?EP<
zU_|d_Du@}C>E8Ts)((|gHSK-+eYZ1%p6r|YS?xLzeiSiJ=lOv6Kp7#{wrk4LeSp;t
zSFjuAr#PCNhyqxq@PlY{x+gmM)9p0~<jByD>na?L>}4hof}zgq+U&>^(CeE15PdSv
z!L|;~OXh8p5f1xz-4Q$z1Zwqyw*`;!u<*So<E2a0O7%_O3Hmtik<CSjCd;)O!B0d{
z#wxX1yRxvl$sKl?{my5*O{G5F-S!0C2IKv)Zsh?a)|Ez?eakz>3ZceRU-CCC^|y;!
zwKVMRT7`+63B?zAjh2c6{V!dOEkEa`b!*57^}CFv`@0FDx>fqZSBWn_5xi<3b2>pH
z+I!`!2jIOIe|~~KG4OCw$Qq)v<UC|rhinl!O<2unZ(Ojp<89Sul-|PS?prg}%FgE}
zm~!+<R(lt}*OwM&+8_9M;@oOz-0-fx@)57Jt#vjPoR)Fwd4|BIf-*B}OUV>G9!!?L
zty;#wg{fpnQ%bLm{2tt0q>yfJnoKEs=${Ac>8*5UJ}q_KCOdL9;i{nAl-?>?q`EQc
zXBXn?=8=|>3}Rc$g!UDNE4c;>51F?KzK0g{4dRP+D`4-DjMxYcEC5B73+*9d^>Y6K
zEXt!*VD0!9uHMBC4hGNmpOCe$yr;V5+Sba6O+Rf^RLUt93XCcON4)YfEM?Gy&ws=m
zygD}!s4?Xh)NKj3AX7Wd(hv@o$uLLR8>_Zd-41_$HD{VtTf1V3eEeFQ*R%6Yz8JOv
zseM8_e`rWyI4*i;;o!Y@VMB@Rem7TsVr=2yZSc(3-#G~vL31Ue%8nx=OA6e_`bRQK
z)}vN*wA%GQl96)fKj#h^Nto**;41M_F?I&=eE17Bo`G6b`TdtGh5ESLf~$KR9Y_5u
zGG8Kh%`x&WCtNjf4D!$&+PU4Wps3rY)1rbs(2qrH1!>-`rG&dp1$G8*E6ezM8|lbT
zgk9BeJohzkg7!imxvlqH-w6-B&1DZt!%I4sb4qAht^WPHXWwK;=4J$Rke*&ros~{U
z4%q~M=92ge5vaina|^lhdW}bTGt0bWS~v_uZewidi~py+GYxC%>K?rvlv-5Y+G<3E
z*gBxKfXXOCQniAh!b`OdOwm#qg@7PK$WTS3EkmrR$QTs_5fPOkj7gC~M20|uAOr{)
zNC+4*LgwV|Xy3k;*8lx<pL_4~&`&)+IfrxhUVE*z*Iw(lsJ<StCNqU)nHLb#7~T6}
zXW6d~ocnBgX70?OcJyvw$Bl%a)00|1*R$aMYXr1@14iX<S~%}EY~&CFi7YI2aomIz
zFVlm-aG;ta))u`r>29Cr6X+7MtS4p=%|#2QKM2jFRq=-p)w`hE9zNT0>cxnAow%=R
zgP-Q4K&low52A3tf;c~SMEamef2?Ji8~D+2%@?RBhhf#=o$G-Hb_8=W>6*zhrhEo)
z<nR0L&-7c9%bd2Q_thKFMsCMm&$4fD(&OQ3uJsX+WHLf695R~|#CYL$kjfu8k@Krz
zj<N%5!Tux~^|v*4c(9<cGD;g>r<}Q;U?8PwKGvckE=vpM;Hb;h9PRy{a;Nk;X$-Bg
z1WtaW9-9e)(BUJ5M)eH;ew-&P2xs9GhnvEX$y1B9mPay)oSW+50i|D5&P+9IL#qmN
z&cW$LnR?MiS9^eM{j2t-nHKERjjb!qRTr!Ni`5~d>LlVyuP(@Zb<L(DI{w^cL+}NX
zwx_xpA6LxZ4AuYTTfp0HEGY&1%q=_I?&{Eb>n;C79XU4t%fb6!DSR-0Ds)e9_0?^w
z|1uQ1AJ^$(yMNK-x*wmrHs3YSJz+|RKKYAZqI*K`ck@S5_k@KP*10$fBmBqWS`dl<
zc)bfE@kS)R)$;hfJB{bn6@snt+EM>u1_P?rbJEPY1a7aPrfO2$PvD*L87>7{Gwz4m
z&?|hnk2q2HHs_U;w-s7m{i}MSwN;aeIVKG_+hI)XxLPl`f8W+s40j_=*PW;jjQDsB
zeQUUsx2+jJJ9NN1I99@x+)y?jt=ri^^>Ovatx?<N-K%Cg#vLRiL?uWN(W)34uiSTk
zC~$^ypf%Ad*B}n1pO#c5=8A~?&h*6}6FxYGdL_@j7a2Gmx|93WJjEKllSfLh^u-BP
zO}0-AP8@)c8nR{_qd(c_>AciyC#2gQ)W{iIk(l+qa6=y0lsI!R;r;@Pi6hQSr@kuq
z@Nb$kodImREyRHI9xNK>iyrgx!i8Ffk+5dIsBqL>;AA8u1WJ^Jxt<rU-%r#JQbL{f
z$6_A>)4WEV?1ie#sAig=VH$bch&R3tlnmAd3HZF}W3)Sv2gPMfBROW}pnL#TCwg)x
zTJ7SHzjZf$>p#5&J-klsW@3E*PCK(MC#GO}V?Be*?q&F@A04R8?%J~ZK+H=>_h$I7
zK8$@g(C0<Z7^d+^C&k=Ox6Agx<IX#Mb^Uy!LYv~LU!lB=u&lFYzD^>;f6dW&$NE>0
zJV`N5!}1^r0c?ZKay*|r0oHj{z}1^w;92v{Rq*qfJDq_mNWrypQ+buPit-kd;X~7*
zdSuQ3aVAaUU!txg5@Szh1>V!dO-aoJM$*hUMwge!AGXabhG8yTE>Np9*BkOo4yj2%
z)T+Pkq2PcuY*P*&P@2lA-M#7{q153V)B9z0{Fy^}av8sJba7=1lC#NW>9!Gqm-n0y
zuAXygke968lv}Oj^zJej4*c*!KzcqW{3;TbSM>@rwb6cz(((xIkq*z_q|p?Qy;{wU
zOj~&rl^Q*J!?=-IZ7^Pg=RBb})LJ}<L!g$$7Wv~+Sxm4C{o8>3(n#Kcx_&W`&Q|x}
zyl*BToe~6{qN+42fL0vPZ&Y%d7}H!@&~FhZ&3qO04kZ%wG9`4x-3US2uBa(;#OxYY
z@FvrW1~IXgzG}(jfKky%jGCvXkK3D|lEw><j``;-v+Kj{Q07aaBd$4SQ1L83jN*@P
z3WK9_p6nA|Gk06ZQg<QlGUEfCe6c7=^#%^Y$HN=yOnlYDT*Oq49`yoZ-=Pe>k5PmS
z#uf7#=4gqrHY*6iuf>msuy&}5n~jKl(zI<{wq8PT;B!w9Pmfl1mU5qTsOutuyR42G
zJRu?!3{<>0z(D%CayTRWYy9C*vIVXIqzIjpv38-|mE4(WJ#zKx5yED&2|i7<_*K<}
zc?7!B*wJQ%#y!D6zu@BOaE>skX0;`b`JC@jwp^2Li)eI=4gbIcv2Li}8EH8@%CDYP
zs2JsU^bsQm8J1L-W&FXU>y3Q&^_s@jjkxL+gR4n6^KRAR{$!#74i>B)f-B|0+GcO=
zpe%~<itMjXpt=z0RXyd`Og@#8oD|ZDYm5R0canMyqsn<KEzRMg1ai45%2rRSFdHby
zi#D~Bj<KZAbE2f;p&=d(JrftKLF*C7e8`NgKD_#49`n$20TYr>mGy%@Fwdm)l5)Hk
zQd<2<(@olms+jwfr*~ICD-y^xJLK6BGuS#QqpTS^J(v_dTQ;eIY2!>(BOuG6*cK7$
zuE~3$<c$hp(p_3riV-(_4EAWqKf+=dJ2BeN!XaFSnFpyWnAEE1gc+(IH*~;7s8)~i
zu@a7<Yw0{)R_Vn`*+e~K9@}%uF`(wEE|$Pta>zV{dC(i_w@$_ET>O(!u()hCnpfLq
zn`439MtWK&u9)V&oD9p>M;*IFmO9sIXf0#5zJcUNb#R%?0uM_Ub6;+oP9zQ=v@J4U
z;fIc@SLf8y)5xCacsAeiFgB+OY`S|L^0YU~8%gnk9XPD0^Ho(QaRuJkUB2a%l2@9D
z$C>%cRI}0k(e>#T-`@9ivo-Kd-HQ1Dt+=24$&-m_bs9S+yB_fk=b9D<cKCOC43VTr
z%+BjU@rJ+VjZxrIP9spC>olYkuA;OwZKWuTILAFuP?6tAHB+T-C}<57wjLcs#q&nL
zI4H*EkXf?#TfFKNtkD8=v_%*eE3u(V8-+=EDTx&~hHiHrs9L5?ki;>`IpO2tG%SYi
zUI#-HrY)t~?{(2IA%^|r7IiTM(TE*iT$`km&GOZBsdyETP4(EEjQ=7vv@b_$`%vR1
z*%jK|+{BalV3lpGljt2GypX}Q1{*fxOS-Q}ha(RR`K1iQsXzxCuze1U!0~4k?{msm
z%y_dH5i`QG+yt?7w1nybCpa>w=Sw`Fe9NhLn)I~o22Vj1IaQ+{J;dV<$KZ0|2pfpN
z1)n1HKNnsZdE_v0OhxX@!0jzUQijH3YJ~-8k{-T9&P+&{5w|?N)AgX&uTDJGq>P_q
zB$Iaz;-syzb+-{vTs}rUBV$%nGt}KboxIn_JUn$A|LbmFSO6|^o3Dw6x#6&s-iE=7
ztSRj2j8nglDU~YBCML2N-q*x8%59n%jqGXQqY=*tImt+|kE)9QCBCUK_JMJOdgyp;
zop4bMd-~!k@^y?=&GHpaU7Mk-!BJ*^V$Ts(yfbZT-qxp#Hf~x_OpYi0`!J-ch&-^1
zbhokkdS8D14(Z__WpIMsksvH{jIc7)T){!QAv7_93}S)-EwKbS$k%?a9?hu<Jw`99
zC5$k%0`?@UzGl?&>sN)|7B)znx$iIL50*b;9?X|-6)`!;#}sIkm|i2lFmdbi)*a^S
zS2=Vgm2k!0^|?0i8%iVIA<U<$?h3a`M352TJIf_;908A^lKcQu?u9U!%!9{9fC1&e
zJK=-ww$2cxx%T7?#(g;R0T!nzUM8gtmvfY3zg>aE<Pbh%qNM`<3+?l?f@HJZtco>5
zYz_i!*E<|}lCGwtQFgz<^2ON{ep7jwu>u3szt%W!Lb5MY_;6IP_#qoh&+Vz8T1mwN
zk30`YU22vR1(PrR20I`h8DP-#Kogu}>2Q~R1G@Q3Y4IKgl5P2s`AR;&XgG@+l|i+H
zkg>5dF+&Gp{ot~7pDy<nsSAmj$e<Yl0Xmc!0DD-X<lNGa+%^Qqhp71OV)<4@ZoRE6
zPVAqc(d>|`qU9~^+?UKDcL({W%`#AvDNnWdH#O~fYm$HU9QB%7m9tY8cM0^!l8boF
zQnecX%0yu|m@{-KG<n7%(Zp##QG{XFrbQ#!7Tarm`}(?|njt6av-YYp=B4oJ+pWzP
zMynm;I^Yg6pIx$92wQ8Mi}tlPyj3|olW1?W>Aq+USF|G`SWx2tm&H}n{jd!a+E_G!
zd?Q@-{Y*5hE~77*kl7y)PUR}yqRGQ@Pm_UM4-Ua($aD#DOu`?5E?<?+OkfSy>doB4
zHwhnF#1WcBUkA_iA5(f&8+MITdQ|;`6;S_zS>|5XZS&QpTZaC(2Qc*dV02{ETuvS<
zWss9^tdfrAJZ2R;<>QuKY@>)?8KpSZH;upldsQr(d>Y*y_l<zpLc+i_TZwxihS{Qh
zIb7sR{v8LLrD6$oY!7Ap0?>GYiZplema!q#;vY*|nGXrLsi|uHFaug=PYX9Csp5rP
zGm!{S_&JaV{_jpA(=tlQFRn*G^A1eE2so=f0ou)sjABR;0+=_l{-*H6H+#<)HSsXP
zvgxS3=Y_jI{D5#u{{B<Ru0tk)VkoPrqR7Y12x;PznnRXh_}+R%?rj{gtz~dFIyU_B
zWg|j7gXwfX+QDMG+m_Fm1h+$)hHY?r)d_!+FpL%hg_76(HpKa5w;W4jY#Dn5Bqb<h
zS-)mitms8PFsSD~5oe`U8mq;X&*UIFxf6dhz~VL$^WYVid6lcrDs4cw9Z7(bVW=TU
ze&l%1c-EHz(B{K|y`tDPaA%Gt57y<s?F>4ngRGv!DxWwHu7Tw|QL$c*Ook>6F#G3_
zGaX^L+XWX_nQSnI2e2&jR0+o9Gu^%Y0hTyIiSn83v}ci#3Psuig^d)G*tv4Y$Xv>3
zZ@&q`fHb$p5oW-2F%5jw+y9tV5dz)q4r_g`ak|=@KX2#p<ss76`d-_{ob<Ju?uY#j
zSyH6w$Vv%iA$^Fr#B)W$q=$3GSCFL79aP6xH%>Zg9E>%Wl(GvHiE^#MSLvuUb(<|E
z@Y0%MK(t5wqG{VVHlY8=3^Ldnc<%Qkju0~{@Zct68VDw6oJ9B_!km@joiYMXzb9Wh
zYGI9=ISWar1PVpG>-Y^MUkVgAE%;vaX+0fE?J{uQ`(qZ<)}GG(-W8UdrN2|&fwyap
z%m7W|6oV-vwnf`Ux?B)HAj^1~BWKQja!+2u=T&bz-n^OU%BXmBFr3p2#gPaS8@ITF
zQtJ1IA$EZ-VHIiGR~Coh2`%HS)Z~y-f(I!mPPXR<%@9*_f4L<w0ii8QO<t9ZyN#}0
zjUF<1=ASm8q>RTbMLa0lsx=Pu`}SoVzO1H+fj3`G9cnG0p6JAE(I)J&y#JIyXLsG?
zF^AePigA~v?J(y$Y{#X<Y9WRs;A>>epShNVp<^#^4UI>6Gcb>Mk0S0<PPVtBttXVK
z4ohLGHoP5U<^U_Dk-uY++iy$bT~l=V0dEDnD&0EiWD|7rV|7lqSsJ;gL3h5SChi6c
zvy8bV&1ESISy!EI7+?giIxDUdSor*s=10W#NO#G$Y=%Fwjp_*zrOC1gKN#UG<#!1U
zXAHm3Wqa=78Vx5_Gk?P3(6_?S)nV`^dvlGt8ANZTBemB+Z&DjahA=cGkPhX`d>#}r
zlE}Z`#`QlOPT<ZNWcL@D3{mq54Xx4D9Z9SLs$HG$X8drK2X&ZV?1M&0Y?#?~6JnmF
zI$Bwv1;*;0{N-@(zALIYxa5j)*dg;p)qXx_zVx|lR4w5(jBYuVrKEc<53k?GV&Xtp
zJZf8*_H)IVY}_-MkG*zitm2u7vJ+Q7p={eJEi3T;{;Hs6hHxjnjy#S%#~Sqyul>mo
zao?#`Ug`7GGRm1|FIfMq4Ee0f;K4x?ga^Xa2#G8nzC+p)l<hdf(r?LlNhK%YJ~%tJ
zWem({wJ3?nO&Qb!RhT{<9P8QH!<vLn@`nefS$87{FvTNCrO3Z<nsTOnay>j|3Bkc)
z32vlY7NwPtxh})Uyi7J6P9pLnp@}Tz%OT6+)MR)C=kZ|w<HIb%%Uxdy;;vMFJ1@Xk
zdQNba$5~x-(y?D=G*|(_9y8wbRl@l=)tr77DK7FGxzRYC=z?96BqyNtEz-ZqQ!Ot1
z9uM&68jZ|oG=&hRQcCp#&&H#BY!TaQulF$r58C>f4C9KK(hHo&uP`$=)lL3vQE!W|
zDtb)bL=itj4Wg%=>+I{c_r-j&aca^sJv_4w6Y{wwUnw{o0UcVML0IM_YuGMl%AdF@
zJwB%rM_VrU=?9gLbY(E^@Tk5DP38XjSUUSafxZjaG7au02KqtKXLG*#Uc<l6H*Fsx
zrq4_=riDznz8R{?TD2~s7bCLcWZ>>$6??{i7>|(!4|oFvo%YaxZtrI38x8ADJriE6
zXJ$b@TT>9uausrzi?P~I0_%TDg6>TwnXDO}VeRgpjN;009DQZ7#c#xs8BE&B6~pOH
zP5~4s<%=RF#Z!9iI_2q+F<!zDW%Q=%%r|su?oN{}osGS}%OW3MZVvp8WMQF+yIQFO
zEWI5o-!`Y2v<}aVPaEAW=icMs&MGGL8-=!qV{z!&QE6wRd$c!;sD<YQCxA2gRfiC3
z9Pg<z@-rA4$oaxs{SKP-%$Pfw_CX=?C1*8k=`WDWUJk8F-&cFoAjTkS;u9M88{<Ae
z3oDjmwJx!=*|EjrRnc<qSOfKVxJ;OC@#R(O=||-0Go3B+v8-}mcO(8xx`o}k%E_b)
z1L-i<WK3ReQwtKQCzbDtf~*p+;f|`DA+#dGu2Yiu&*A=0n=Jjm67Tw6v?DWrPXq75
z^C_Zuy#Gnc_R=B3^&qVQBY3)V8&@UZ_txC7fj!0T$Ck;XxC(qMD=OPY`sj}}<nVHC
zZ7)S#QMK5jrj-!&lIoF>9t%tMTfGT>4kfUfDTB6`$ZSu;c_i{Rn<5t*YFJ@H3*OYW
zc6kb3xm)PZyKA}DN~3~ddSr2d{eC79Y0gUu&cBoZM2O=ioMxydFWd_1y<K#saX4Y^
zrghj<iR6YclT%*Iz<s32-IY<`Fhx2mAyZz@-bp$id8gfyzX4v+G*n@_%!?N~&_?hz
z`Plp)*N;v7Agijce;FZEb%@;ze-Ct@ZM=>gPbYtZHXGIiNtkYCsx;O_J0x?3^G9;l
za`UI`z<=9`zUR$NUT7N%{VW-BtSD#b=mFoWcCAWd0z#N&DPjg{o*Zts{f**5MgKmq
z1Y(iC*6N~^C{%83aGt#>49&JczgHKQ`fF${65102!RX<KkZCZ@X$0y{u$8@Czav}R
zLgz{|v<yKE6q9~@YT(Fb;y&SRSB+hO`Eg@TOuou1`jdb=O8S?)<z268!5Q38X{<Bc
zFrk-O>O&%Y6~(l#6Y-w+wyP`3g#PuaU9qYc`SJ>H7(C5kL|%vwFAi;%;snTVu`$2y
z^{+l<25sM}v^~u~W(0PtEXS#dH7l|kKKxoG&A&<wnw!<&?F)8~F@-k*k8Q!_b{>F@
zXMT~biM7xY#|RDzqPe1;2uuGM!oy=<EjDKDpz-{xPhWkQ;NWYlLCKlZ&Bp4uaBb|%
zn}Ru~8UKhEes$_+O-hrIV<Kg$X1T906qGRe*_R5#5OI#KULfN3_Q6Wwb0h5$QEgT9
ztkU2aGZiaYKa7f(q)l^H0H0#Q^NDOW2J8V-e2%Oks|PPY4qV@Y@ChNE&&9noLz@AQ
zBESD*=9+MGuVqN4oN&XK93z~N<p^0RJ|DFE{4r%?MxqR5yDHS2=)3X8GK5=Ty;$Aj
zvDRa_kA*jFz?c%Xuhd~8PL<!9glO9`{0y=^b)^VntJ*qR!kCRrUD@iP*Jxvxyzi1D
zJq-$<=FT}JTU1&Tc}*=X+k&SddeY*7lw_A!$$@4oxRbrYXQ!>(adnKjcI5etN9ihz
z`8ur?Mo=rhD^idDU{Plh>1{AIp&97w-B;a5nUf7{XIm6PfrA*wvBYRkXj{Uk=`v{V
z+@suT++i<Gqxn_BiOHcsR2^fEE@4s0V)!ns#IS0)MS5%uFO5$rXSx%U;XGKZ&<+t{
z;@HK<YH6eW1u4Br6wkndixbcd!>|BQOrD728Tcs*<1bx0?LKo?`K(7eESqy;`$zQ3
zBDcpu>kH0*E2~XtDL*ZqEU}tfhdX55t@sCnL<xq6k>evJo$Xc8Tz)mJaW8ZHOg>Ex
zqG8zRPa?q{9%ZMvQ*zW~)3QOiCuu!)$FMA3P#?=L;+5!|>B2`h^q}o!%q7eYHOQL4
zgbHE})zp^u^2M4nb(*OrO0>+n$zV{sUb~Uobd%NX?KI?e%!L_2Qp~Ar3yBn}o#2&G
zY#@$LaR}aqhp&Az%i!pHNq+sWGhIxBRvcmS{XNQk7ucq~$;{u7yBKXA;~;)B(iRH<
zKLey{nC*{2GZu2pijHAsE97oAoDYd2N1~2@sv4Qw3{R_{Lut5D&nV(FYt5!*6i5M~
z^>uE-c4_}Qo4AfMnk}dg$BQ$tHon+Lh6Wu52RB2U9yFefh`htQeJ+$${Lp0R7zq{&
zU#F5Akl^RAAYb@7lGDadGk;o!!i69OyV#UidbnWI(?(>Pvmq<cZH?8r9fFtr{EK8H
zpXkjE5UerOekL9Yy#l?e9Z4pbH2BEpLZ&#H0b1|ke!(LrjMgqXvQ7=Q<^H)IblL5h
ze&jmGG@M;PXo&VWN^W~PVg-@8T;mR99xy66-~R-1P^w2Rj-ELTr3|)vM-i#-2lmdL
z`4`2pzE4!q>KL`lH#|IrOA$YG9IdmF`3$JYLXQUN4mUf+j-A@a>p+Zy;k;B&+jo@&
z3kqlHA(;nb1F&2mLiY+TvS3SG(kM$3sf!Uv4fFb~?wi+0*Nw^Ao)ewRf1JZc`m4S&
zO&ef<I6U7WhudmIq|Ldec@%o+s9A5DSxT72Fos3z-S;Fam`Be95&N}}V!sWTQEn)=
zFBt@pSg9+&9dwP=Y{qZMhHKB(X<7qbJ1QEveltFdZB(k1(mJV}l&?a$Pk`kxLRurq
zS$&8$XcU8fJkto_v-V3k0Q)d9QnE7eAkv1G16NR^Y_SCmBh3SF{O6S;Xq7G%1VFQT
z8m?7eDxZKq?AZ77?otv{D93QYp3ARBmqLK@pRqA+z*Cva{QMiDe?5tkoi)$H%?vhC
zdUJUx@~ZYkq`<>B$X+VxRo{|3fk;cMQJ9oCpdZ0OtQ)rVBWIgi@%goBm9sc}UAam4
zS>X>;m~>=+S~Tl`9(fKF5+{1kYC5BLF+n)5HLyzFhpL+!nv4mHCua~U*V#qi3hVA@
zDVxamvbV*+b90%fIt7<~#=Izvh`a>3pC0Alv1Dr~m)4)n@OJ`9Bhg0p(YTOMiTn(^
znt*y$4g!dr{Ky}qoPJYCN<QhvgGQ_%;S!EK##^avEP|Wn;es);NQAP!=fKRkez6(6
z%KKhi!t$wUjiJrC^WxSVJ!x5+ztXGDt_HVGbs|%bs8lU~^xI<o?$Uhf!xH_ZwCWuU
zPiVWp*7kICy}R~A28BCHhQnoOIEtQ>y|%nFH$kIfsH9DtQQlr^;ltkySNYw3c|5Rx
zaD3C&wLNzJ0UiA!`YaxIHA3S5gSgwjkU=@a*m6tmXs@nfj!!3cg0#c-0%eDa51%4E
zR&2S&wZk_hifYbD0x$YM_S}XwtB1#ZmCENQR@B)ykU8Zg+2zI6D_T82uq5`#HdQV8
zn7ZWi5*_{QtponTj9@;INN?>0;ae++kOa{liI|B4IWcV0mwm~Qe+Ot@oflg?D0UDt
zdrdSS4qO}P`39@xJh;<2Sqz7Sh*mBNMZgs~)d}RftOpZ;zuutvd<IShBv;lbK}HH2
zt^xjas2*HiP22bLsnSMfmN+!ub2L;eO#(-4(?q&^9Mo>=i``;+yb;$euDYMziD+Ry
zW@-gT>ul<7UM+i_fHGvkWG;bf`<R_8!Y&b;GLc0IiSVgOWk~NRUyvA@p&wcx9dN9F
zn&{$!`0!_YMJm3=H;3SZ366dg)GCh_Uh9k|^7a~%V`3PT`i$_#?I=V~(5YKXi2b8N
zbTE6)KF()Iby8sBZK3l*MqG)AtVX9kl67riok+d)3|OM#vnn~~PFi&R>4xX3-{)i}
z0*kC6?;)Tn<7T4-co;5pMxzo1UA=6vvn=|22|8gmMw^;lOa4asYa-4xT#x!PO2R32
zEzP6umX54-+Up#-{;Y%|Iu4xu`W%~9*27>9{hBS(0LP?ZD)YjzVHz?2>nLI6EXF*-
zImvSmJ+kXTS=*1L6HgSf)0Tq-&)c(+y<=NW$-$99mq|r|c6E%Pim4*6N(=bxp=M5H
zxq96>#IBJow}NLWqj!ZZ+Qs=z0l>ydu~qT1(0fh&Q`3B_52)7}cjZJ}v;#sllGua0
zT6u7ub}}8JD1(tmTc<dX;v<bf3UC&<lIa@VscxQ<0rD$Hsqm@WPX(-DSLsM&X3P>Y
z|0xmv*wHP{Cag<c7*)ddA4!O_2*cIX4~6xiRUqMLPM!Cu>_xGcNPP)%8hWVfl=dR*
zxebv%lgv0UOq|P6vcTDH?11v6rZ+3MH_V~Vm!CJTbPP|73en`p=2Vyrkxb7WHKk7q
z<(ey;*i=Tm?D)E0v<=tz9-hrqcqgZQF&vTh44;@tg(O-Usiog{?!s%r<vcMfhMW1~
z#?)QL^ozXT)s}U?4Ghhhs%0isG<j}Bb5y*Kw#X{5SCv1;l3au(TM=*+y!ulVK|gwJ
z*GzBalz(9+?l2pc88a&n`uQk!*#nw@d1Zop-B^<_r)lD!nB1@qW)APcf6P5TFKo@-
zXaxTbH641Pe;|Zvvc7~oM<4Fb(Vs1C{Jk|RWQ<S;iXEml|8-ez%=5`8R)^sZ3G6{8
zP?r}-LKYtx+D87p)$o}vg=toY<$F7O)csx_)YmEQ`bCZnt<mE}1p{Pk1>Ofqjr`$y
z?g$k29^t+&De7D5{Fgi4NdC189=p|7AsKAiCy#>ATa|={eulN%<>-*tSx8Rv608dS
zXEm)ssS!Z^M>>UJWNk(pt-$%MeU&c7sxV)d)c#f?Chd_j4$e~^0R>DOT|Zt(Thb+m
zEaX{zt;@jH4aGtxoAc5Ik@zzy|Nl3U$UCT1D!DxcE{hgLOabt-iYmB$bK|UzJ@}S(
zrU`Y(<QrJ^G@{r5CR?utIEVKZ{dkz8Q7d0V43f1{z(e8m9D9645;sprdNt}9m@Kn~
zV>W{YlQ#z#0_0`MOC1~>3bbFi4$srgm0p1OtG$i|_zJuRl>1+awow3LMYcvx(C*T)
z6xXUDu2VYP%G*BR1R!y%nPYUO4YM0`VYt%e)oF>bMo`^oEFGKRt=!RIxd!YeR}i-9
z(rE2Vt*O@^5{b47L||#M0knt!uz+09)q>rvLw4S>zzS1X+BqE$y+fle9Gw4QBptqD
zHkzn=6y4flUv+)Gg91@ZR!aHB<p4Uw1)U?@IDi`U0+6RNG~mNx?0&f1t1GH}dk@v#
zpSrS(W*Y&x5{2@vXZ@9c_KPD+{&nk+_{_%uq%$(<bO7Zu&+dWJA6@ch7phntIvC&>
z@vZ;G_)fbfOVj;$i}|DT+6vN6@GG&0>cQ8dYm<TwcC$fei2XWHVl!f!Tt}Z%9Nu(J
z*Z$(|c?a;K{Z}?}@5ins7lkcZN_7+;v|IlaHe2FxM+bJ59sIbH;bKj_AkYDMF*=y_
ziq*HJ85q#p;tU3U#eJSvP&o1Dt6&Z#Xf<jHAf;YvKJP3(_)#b0{<oI@mI0vNrzEDc
z<;Cp>FW2dCNPn5U*Ll&=eGZ2Z^y^EOcCNV<fAnI;B%s}l39?q-hEulhp3+qU{zW$a
z{FCTI-l(=q><Nf_9h(|@X+;kKP?a#@lA~ALL4C+OZ&Cts+KzP}Sy4l}Pj`-fu81Oo
zCIzV@duXonPgFr84xK1W=UplfCX4g<7n^^4ptrOZjFg#5>@hX*#YW?KiQAR`xn)*u
z9VXuSu?I{(erxw{kEutxm#rZI=0&=hLl8iBkRm4igZ$k0(AwVhH+2t?h?aJKeR4nt
zYUyggvvC%OEz@-aYAJ@T#ZM2cE%ewkU-SZXW~{KnkAR@t?EO*x$!`F+`l|^HZHG!}
zahTb1t0cbm5?9(2H&s&XO#YHv^DZgGU=J}GzG+r{baEgw>}ekGpMo`ND;;tr75g16
za{kHz5Hy`K0QpVX)$yw@a8tCM6&tOn`48Ppt=(#HT1|&C*vewZ=iJwg0N$#4`cv#N
zK!9ivtw+e!ZXE|DfQ_-<XsqLmxuILW`;R()``1dqmgICNX;}>ipSY&4yp899m1U&9
zSNPpLhtKQJms4cV0lwUmj(uA89iw!*?ykT{0B<DiP<-cFI{*0Y&#9cX=IKatQ9W9_
z4Ht?s)@`ynL@sZQ^z+(7+x@QbM<D3t;07ScrxZF?j->anM`r_>jdMZTmc^69@+41p
z>#8tL|F_tW_UGaRHFS3}U2Z@>%DqbK{%y-{dtKG-+nM;+sjtfh0s&OAhMi?p#GGq0
zC9G5b^6qa>UHOu&?a;BN3lP6nf}+5`YOFuR1|TPgRD=^KZ`YJ}$|V>LAXAaIA!-lQ
zHv;fkd6UWv-7?7fZ=v0PmjU>->ZegaGd5H|3(1oJo-@W8-~mAr+kP({R`#D(c6oVu
za88O!4ADS=;3}R^IJ9{_(e?Gol0KIK-k{z8Znj(zGB~(J#})#&*ma%!pC@XD$1|=I
zXhAoYQ<Bp5*eeAWO$q3;Yz!DI!dft>tamPQxpkQ3o*N_?GY+VAXL6knkoKVbvQrAR
zRd#0f<!Bwg@1N$a5k@XbTe3(;T6}%i4sciFn658TJbbyu@cO*@8{hW-tZOD%*A{8p
z;qM=x7+C2J&Q#NoXn!`w3Vdq$>y`OV;n!r_psHVgne6`J_UIwcCu*fM5Q}4<SXr3y
zPQ|<&LGLp7cr7p#9kk|)--TY%>-^XE8vi&z0&_-w&fWMfMd04tU)fpl833Yxn(X|Z
zu2+Byvd6bmaLe%P4(&i^yPM+!&x;b*c{P2(K+u5^1O~wjL3g%bbx(>6Kh56p`(Y2K
zRhrt)a##C-ca11rV$IV{c*OedWqP31Kx2=0-a$BkO+Cpx6J+(lQdiS?wYC3UZiA$v
zFr&?2#-ie6g0V@nW2J_+=$R;jJ`>lx{(mq)R@g({Yr%byPdso6dXG2MS1dB<WZ04|
zP`Iq!7kb0qvC2k*)!B18ikjWJ8+{F`J(gBFoK}_&ELQ|vlT80kEr^Ih$g&oJ)IH(l
z&!(3wIO|Io@BDI=H4jWBy%?DBi2V`Y>db|Sa?|G<&2&8{{?YocCKai5Y7cFb7<A;}
zy?|Gv1D_O9be%!|@#jZLMIcq)h*a1dN!T@JNsu4ZVL$Cq<yTAs-}Q|rbe7;mGJD|D
z>`cF(F6q&;1CQ$r-}ZG@aal`BnD6?UwO}&x_l*Hv?(hwbJFimcPx%mAmlaax6%5ft
z@+A*myHKQ}Ym%D3Nhb3*;7Za?Q2Q&dm>iq`+8m>r*QQ>ceU!%f+h2`|iJ?J3qoN)y
z_|cUvKhTPbPs3?AxOCvtp3~9y{xWWAIThz_VB)w&Z2_zz_mvL$h>~eduNFR-zdXx8
z>ouGkaCWs$3Qz#<N*gGNAGoAzKwt=N{+na`JS5+240M;+k2i7BdCuVJ5~&V^$c1Xg
zplIW5+(+{jHwrMxv^!LnX1coE6~Oh~scJP}v?$lPbIq8prB=jooeQb><>-|<9S-m<
z?~EH~&GeD$T2%l?T=T~cf!ma*2fTL0VTG>yKpN@FwL*#xBKg+$>xO6{ps)~7SP%&i
z-dw1b{7|P27Royp3TWRlEej&CP*Xk6$yyMJ1(8?~2@sqG3*m3&xUW6x1z+NIka5A6
zSb%Z7y-i&(9REiR$K!yj0??6m)gthp``bw#x_?--NYeHOarNfvN0{Loy3L!*Zqq+-
zU2iTGKfIxxy}A0D@CR+~&E;gqAK~LSmzbSzxHE6AZh|AW*G#-OmkWRN^*AgH@9SY$
z5TrM&VqwX=9)|_xu%H}t=f!Up#e#BJP!0>8!=eSp?9DP*a6w;>!~YA;wpy;GX$?B6
W)p&gI_-DG;a^2_t&4WEBFZ?eD-`^kr

literal 81912
zcmeEt2UnBb)~$$C=}2!X3QCb69i#?9rAzMt1VRbDSLxER&^x@+J0#Q)0s;!sqyz{Z
zAxQ5n1a98%8~2>^o^gM|-D6~=jEtQ;d#^p`nscouR$Eh*lAM+N)~#EVuU{#@y>*L}
z=hiKv=cL5=Po~4{k@zP9&$p^dw`zvj|KQ);u~*bkymhNS`R<h!5&k`y+bd(wTelb=
z|NS6%{r1uRty?!=Un?u>`B`kwlX&U(hsj?L@yuv6wEy7Aw)_OOiC`u5j$Me?s8@i8
zZNMY`z|W|ZyvR}%?km%3c##n((QU%dp4}l$fNcAX++g!tKPjwL(AEhD?MdYyEwsDt
z&IMwU@<tAVGB23w!#@0TYzXSizgGU|IOZwaef033BY<AdmyY(I15Q>U6Z_BUJ}lgs
z%>CzV%F3?co0|We?O~CGqBrk9XQLg<E@=DwU(;=9X#fAe_y1Q<&-WMGYV@Ot*#Vqz
zse#n^AB$IffniJ|Q4Np$G^WZ9WN|Ns$ez>&ZRndyUD=nXLvo07nz7uK9Yy(H?6EeL
zQ6^IV2*4uWxXoeT;ZL5gU&Yl5l3Ucue#hE4C;<v<nHu;I0j=;4-3B|3wjqu?ebzgz
zphs)ztG(2zD5GGkA~9bB4Z~8@V|IbiKZ(TjX+*0pZ3Tszt>qK+`PKdr`NzB3^A|2-
zJ}<w_f?DQ^nk%@DhQUezE7x2)YpngXEiU`7u=2duPikv%`mn!x98ZEW7Ty7@wo_LI
zq$^xsyXijtulcBS<}WnI++r)(BlTxOK7?<zi#957Xpr(y$j7$<hmH3)O{;%uDFJOk
z@PkBTq~XFw0$FzP=5WUT3ud*6fqx!fxQw1}C-Ab1AMQZ|%M%=U30!OOeMuiq7T&5h
zQE5yT@`UN^oy96PKfKQ;HEh2^<JtUv$J=Tp3Z9Nbl?bxuPdtRTCf~A-G%iW(neSr%
zaplV7lev3Iofh9K*kh`v<8;gd1H#@(fXg_rnqU*a^1QCr{ogMchljUxo8jn8n;(jy
zk5WZ4wCA%ad1(Ft3OIrC-5~qpgK`$N6g7$Sn2BrOtG(_9l9P*8VtUxaicQOP#GJ)_
zM^Bp@rNw(CI}@j&8O-YU|FPF^7cY{S?;>U!xs_d`qSf@)Vhl~EV&(Yy5!Q~&6qx17
znWAKvi81A|dzt=}qHXm1%_b~pJ3!)mUQOw32hDDB<1<gbE!Q^y<h}LD5!4q?a#nc1
z^$xOepSt!%Ee(SkqsvzRl&9-Zp~t+dqc+?puvZni*iY%Cnq84z3xuKQS^^9ZX9LoG
z5_2xBaEKGk^CV<c6IRO9@gfSoaxp1=`dCUPGM;IeSXn>MG-YINoWU~yMd2dKtWL87
z=wF+Z{cew`5NCVIptAt$gWhF?&Z?W4Chid(r7O~oHDg*k+jd!TO=MrTP6AHmb}ya=
z59wW8i(R|?14}o6g>6UkfLUs&Cz;MO)^jEZs_iBVF-<+}=&k3WRwHocQR$izOm(%D
z@E9f06GohKrc&53{PT8dw%IAD!4MxAL)rRBuQsNlIHwU-9VhHEQNwLm_XU)Y+);$*
zgDLr<i4+Y^T3vWjr2P8yld17NvzW4xfdb(fhU*_tuA~8jygRi&HoYg9?g19r6IM1+
zUw>V*0%6^?V@L6qB@!WW<F+B+EB0s^No*=#4tet1UBnkVL4LT14YJp;@#>^a)lJ7O
z&4#M$XK$2-9I<AR0gIP02G8;`UwZ!cwGtM$9p%>jwy6Gy<|<mMplEXH5^M8V9IR#z
zDc+jWSsNmu7X@Hcg1O<7FpQA4{nZ6Df{Zm|^Rs?)XEdxKfVh=ylEmUqs`tdK#XGEx
zlk9R>fP+jAe`~DawbBc<)Ud<0h$~8Gp-)RC9*Y%<1nF%A8M8Z<ye-;$291{Gc_X@y
zLE3@*r#;h-7{d-1z;e!pWuh3|S!i-iJ8ODm@KcID#(?F&0yPQ`NnLbz<ZIcNs%DGK
zEj1}?VL8Bq)OzTq-<k&gAH!6|gzjqx2M&S4FP8s8$=GL>ZHd`wg10#bKjW5W|5{@b
ztrIi(TEaD;lamu!^*nX&HxAXdJ{xf*l!s98_o4gLFnFLOD4MdXZ!bj*El{YRZ3%>)
zqB2mC_mP9u^&qA3=CYmIO`Q%qz;bVV$q6eIkuQNBS$v)vSUbImHLZPpVl3uGt&ge1
zQDmJ5PhNas5wzGfyug*iCI$FA)@3E8g|^IUXB(cSqUEs(Tkz{gEtAi%4L6lRrDcxo
zC9-?uCyS>3UF@lclN}wWmo=qX0eg*P7>7D7QQTZ&zAAW0dMk8FgYt;%WXkeJ#L2=E
zvc9M#=!kg>Y36BicxgWLC;mT>W>0o9R~|lz^d?qLwPmm9G&CJ93{z~7U{f!%-v>T>
zQ1{);c(ZH5nryCn>mq&Kd(%k?pcu$sq^I&>ZJVfOoPW4{49%#vndyNzx5gDFUOIqw
z;KVBae9cU~I{R4`p$dw?HRH}#rxo=-BmXqCMeG{$UQU#$vTXS0MJ+{mj+Xk2DtW6!
z3*=Ob-BpN)FdK>gcrQQluiw^Rm<*wzvLlx<9*=TO<I5Qt&Z?Vmf5RLTYQyj(ckg8z
z*A!nNm2QTRbY)vrPkahgXzTA~SDbmbn1do=UCBIU*^66h7{E<7OAi)`E{uo;j5w&s
zhYoJC2vl97Ykg@BY}==B8fDT3!3vFsd$2rg&`EZ&xS!c142r094V7hW#at3wS6<rM
z5JDL?5hLli_D(ky8@*aY1!dO7i!6YVnW5~N!HMsL0$0dMQ1)oji3Pe6QGdAwF3=KP
zmd>7hHU#OZKyW}8A3)`Fv~c>6a-W6ulS#X4oSs7g#~O69wT7ojx7S-FL*d_>qL;Wi
zoS!l9mQpC)-A}*dvP--7^fq#^BHjP1%S`CQjDqK*Nf@M&slCB%%9F4^(h*#VB=nc(
z@AN}-AZlmf{?O<86XCZGFVo7tKysvD3r^j7*Vg>hzSXqiWd#k7lw~3Mm`wZpYT;GI
z{0GAWAits23Z2ZE`+q<~&nrfN&T9l}dRqqBnZ=#95Bb0f#j|j+oePvF=Rr`V9jVUW
zXE8>=Z?bb?jBn6zg7c@7io7LT(Wffy){*9l1!Q+LF+Up>(<@5K1cR%ZGF+NoPGg4|
zXXR;dFb1;T<!EzeHWDo6rR6O)rINac(3C9f+^XH)I@zg3%xB5anBBm|=2Z00aweJr
zU7hc@9|oRyicRsdhD;^qwYRN*foyj;ZdN0%1D)}aT~)0ox|PQ-www{SWe50oVlD?j
zoHJdmq~FAa;!cDP219QcLQb+RR}B$ZiK|Qb$eSzQD|D#-1?Bb5+2n^21S9swyyIq`
z>xMq`Dm3(4==A~O3l4$XTYS3rb@A%DnW2A5?ppOYTmFg>adX7Ae{+4xbhQ?G6aM#i
z_7Pk+wJXV?*W4jk>>?RL-{NLV3(;>Wf60G+R*YC43BBO5U94S$#|2!Qcf1kXti3i)
zqlcVgZ!QS0FAo;?$NIUJg+g)hlKuJn@}U<Z*F2}ehkq6i4tTkqdf)VOT?5N&E3VT|
z$|o+kZgBbMN0v7U$v4-+au>hpgN{FToXRiz7A1vZM-06?s!9*xK`Mt*R|o`-g)8>x
zdc?9<_J@t~g=OsC?=u9hXK#No^xAc;{pRwspIs5WBCmCWn=g00v_zausljwe(L}M>
z!>!=*bY<Zat8GQc50KKN!s`>k!P0faZfN<Q@kH#s36}5W?7u5IC!dUy*f;W@A?7xz
zt;*7w&ScE1oyRjY&y?QIqifoKWD-|Un>nH2&6mWUn~_4l@3*{Z%~H3Ln`6;MwIc^b
zT82S^J4gYjJ+D1EvuU-ux647QZ3X0Y%$ENYU)G#q6P$IAx0Xe{OynrMn<d2i6PXyv
z(LxaZ1hX;=RJo?qb>hS(<!_lzs(23@UsQd%y~zTqJu}Ics+;YSHN09cN9lUcyYgh*
z*RxO^->fXb$Jyo<AK|q=VohvV-&pIYhn8~AG<ccx&zg6;zFe)!r8eP6`5W9wC0TIM
zYm~`NtEHpnO!{(yU6ViIiNuw}>_q3}97u`&5|)KOwc{bC|4t7UxDYh&?k_Li?3-R;
z`a`=yx2{-1#m_CTpgX+hi^GBkXPjB*e>IEC;i?$qa2;YM$H6Z{uCeyBqwEDSr&Q)v
z@-t+z(Z^wSi{b$2&WAjpJWvZWGIn{_#y#le#b*DFk;&=!OD9ht1+!D^)A-WWLEFYU
z@X5ofxfVKpSXEcb4<P8*2n8`N0R(o5lk2&(xt{emYPBNk8(}7*KRP`x({c)Ww5xU`
zVy-wBUv~ESnxR&(*PilX*3fXS^qJh|40*7ssr7Q-V<UvOterp(QGYB@cyrL*sKJ5E
zj0)1fKEDO<S%2llI!%@Z_dcjETU{l9?7x<GA1cZ<d2@K#i_wx}@;)6^QAOu5OZeV~
zwK@0Y$v!iJyF89oXO25%G~aiicJC-#<#zwse>nRM8uW12wDTrVNbs0>&U6m@s0s~d
zwOLP~J)2N`L2Z78u*f=p^#Z6H?^BokE=KrnU%|-!NTIz;0%Ne(7bjYrGW_W3%VkcP
zjEussWye8DJsp<g8dm^TgP2}#tHO5_ad62>%~LjU0K5mADl0CjK;~?kHj+?ojVk2(
zyO0hWT%ZmnVXP5ie2vUYy>~nOYKk<Wd=YB@D_7p7#vbyi8B8n;xT;sXcbK$K$V}Hv
z?P_rDS7@IF71jRK<wqNF(O8INq^y-6Hgiwx^o#B744oY{SA`+F;{wZZ5{t)*KLSWt
zwpbKup9rBF8_XSZix(~}U^etv<KnX_3oU&_kPR|2V7F1{#5{B@WOA$0@d!`Tp`99I
zUGPaXOn;<oB?!+LrxX^BOHyz>*Omf@;GzZ0{1;C{2Yg&2j5W3^-x8jn&M2}`hL(GK
zr874WXC;}*8wt&Sk;(D(Im!}8cXPdn=nV}yCb?=p%|Ecm#oh1jH@78KRAu>&VlSf>
z<Ogm{vlgbpg-xU~ilD_y?+g;3HgO~r$V^k1_lG74^YdD5tmxEb?tC!LUbkLFY8Z6}
zJ`PCO<KF~?B}K%(jd4koX5|?aFYRFU4<-iA7Vo{gn_wx&pypfE<)UTCj@XFOHH6m(
z^}nTH7ojna@dBRd1OuG^FtbZzh-({T>aCBIyg05|M7Mty{F$H%pn?#IG|`<nQkVNp
zGC;;l4m+EAv08$Z%XRNWPjkI{91!Ev9v~T>>Wr`%w)NGea$F0$8x=`XJLvG!!G`!>
zyk55RD}EXn*eD~aVhbECUc_mxI%W~^k}ZDEBu+X|i}^#8O!`W<hUHw#9}?eWU<NFR
zI3SVr7Kf92hop*b8WU=4puXrCrA|yx^zhy2?zL$jzdYams>H@It5?)ubmq#Q!6krs
zmY~=$p*Ol`t8RINczVQJslCvAbavRt-=b)w%=fSSq=AW>p}8U$s=C>FwD{6s@x9_?
zmkPAHuOM~cjFKu`)F6Liy1>h35v4Ddp&N|FM3B{jDQ3uT0tm^AWGn16%GkU7;njBS
zS?$V&9b{IKo2ENq+)y8#Eg^=i#a-y6tYt8+h71!h1X2nff$uIVpPYuU;jRQLee3ED
zXEz;2cqqHY4ScI%I<v6DAa9a{$z+Es8HN)JL)*{_jAf&QYh80gjY^*VJl6R(bgPzk
zTgO~~QNIR3wzg$>1D}+sGs6UFfb#?Lb-hny8%s`N18SRJV`?UH8oDbNn|Y|ZP^O`>
zp~Xr^mySo*A$*rQ^tJFdJUYThBL0Pr8<;AOa^hC?2s7~|`RlW@b1ocWdDSIrl(~$A
z!_?-n<S4tePv2;RNMv*(fhf^uD|5iR<tIO9B&oB{Abl5Voh=+2$;EQ2qrRV{nw%*Z
zs(*u;-vpmH2olnU&-xBM*wa~T6Ktc{B`Tfi5e~)DwfOyT3+IJxFTU5YwyZ6Lj|O_l
zCm4&7#R#$Y9*D%$zw3G7<*}QXHpMeqCjn$L)m5C}-JGC9eYFv)8PPN=Z4vV4)G*}^
zdRKFIiwIc1HDQs{;z#FXCIR|(b&vt`;5iLw>oT2ygLVWmlEq`fT6iEj`6Urc?<yvq
zZaHz+G(&SgmMt=8tw5u`5=bX(B$7FmAklk|hArhH%maNuKRetvo>x&fRsBARZAj`H
zW%PzNh(wi~&%I?w_YG5g?v>?`^`v>Wiv|QOJQ-u$8M97#;N`477Meu_<64a7W}2^G
zUoxS{vZl1bQjOL#p5cN0_(7`XiSFBMy3SV9CZ}9mt_tD?3&;s2vmT|h9@>#2tt@D9
z>HaI5lh>()Zzr2sZCSCMHHL&LwJ-@h!xn^<V<z&a29ehT1UV?Q1+=nn6I9lF-{Ony
zgQs%j*K!v79tXAKzK+!FlaTai;-tTWPYf5NA;}a$MybmoP}TM<)yK7aX+@1Pl6jNL
z#o)zYx-I|6(WaE1oYQ2O7u2VAZeq5v^~~(`&RL<gu1v{c0Goeat=RX<UzYqZ72>qY
zBznmCY{Ve>NnoWT7ZhWDVt#_CRH-d|9bDb%QxlL!*&I9T{WQyK_sxkMb!q_r<T;1K
zwyVX73F~g{&{pNp1=hi&y~42|*U_aGE%a|!bwT6ivNE*S_4?fOs{F+AK!=I4sVs+9
z(RBEOwp78w8!9_Psuh$fF?od&(es4;*ssszVBp71(H}i|6D-6Du*us!aMQ3?Nw51f
zZnBL!ogPb5>`_tj!woT)u}v^>6*eb&W+|_4_Nye;bBC{|H(@O8bIn#`eg1Vh;HVIK
zL4eDSUQ51wI6WrGdJ^_{<N3BoQ=RZjwLq3PWt{2&-EgutAD(auGiKo66iujvD`BNS
zunu?rj9ms3_SiJtW`Fymk-(<(wkM5hFcs+MoX*Df1V{C^6L5UnZ}q#+weMS+KE((t
zvdM7kl+rfcp=9IUh~I)`iZSonQ?G8n<l3HVq~yjVDDh$}Mw=M%A}^fm`GmH+pvek!
zXX%+yhbQf@xlHWjiocP1LquBCQ@PzmVtC(4&qGv{WFVRZtF?p7c)2=iJb~&nnagf5
z^v^G%F88g<ADHI%X6$smJ_X)0tGw<F`N|r>JAg+xq=E9}wc&*rvFOCqx={4u{-(E{
zq13cqaHWXCFC&&zWSUBv$26<C@$MR4)>KXoUVBcEgj-Nqm){|etNj^v<c8q^#FTqb
zr5cM4gx60NZL(HOK>BAR+`UO!Sy?4b;+HU~y%sL7YwaSy;U)RCWVqj|lsp}YA=hb>
z5=KUg-epvuri#NrO1S}i@(Aa%+5V~P&S*QTv?<S##CctK#B>E;K7@PZ2zvYW$%}}J
zPUD;4iOYE%N`KL)Hwb;$<!*5vM)!n$vposV&Lx+#tq#E}_`D)Z4;G!ZW3+dmzoQrE
zRNyLBTPu#)E6ogbniMO{<*RAW45j`T@ha~<X?@q@dUKUO;6WJ*IV1Z!^J}W6<WGA?
z^ZSs(k?@?d^4#!PB|@g#8EnWb3F|3MgN)81)mb*qq;?X&jY>w|iL%aazbjxjU6^QN
z`uKZ+#bUB*zvl34K{Df10NB<k+J#%Lr-jI-{)e}q5BqF_aC|>9V2UQeT!w)}lS@0z
z!>HMls4a0jKCLK7AD}n_E9%@tMv50{c)dh}(X1=FN8)vPtC7{5rtZfR4}O}9FeQ!0
zfY8f=lJ3i-W>l_GlaHJ>&@kQGBqphaG*Y_ZnzjJNJmARGtBI!wiSXz$OLX6zd@a_<
z;03JPZV|QF+1|H(WA0I8YYf<uyQzwTXXAsRzwGQ((a_6Wc;9i)WyF@-bqV!4Br9f@
z8yoK=N_X~)^YfGj@?daxzim70r72V4;ww*PV_yY+eqAscsvk~QBvkHOm)~cUiqu}}
zJz}EN&0LWG<2K}df{LENot;+bo>QLzbJfbkz$=WFt%%t)?;@?C06vEnW&N}=*d$A&
zgR{0F(AGbmiPXXXp0t^EHBPwsykL1v1Umea-n*O<WXPucHk$xws}dhQ3CR~2$r>>f
zuHEel-U%x%AtCxpPBPovxN0_S&8Ft9f%*6d3ZM6dOn|!48yHt(PZE7La<&Nr<K_U<
zDR%7|zx7FUgU79=zOy0r^BS$(z#8e!WrDGakeMdrLe^QSMM;Icm}g4!g8fvakKYXH
zCKI_>GLKf%$5hcPlwD%0F#%ifkxjm>GF{*COJz8&;(xi;vEoLhJWAA}_!{pXtmAMU
z2KlL3Zd{Z%x%B^XdvPzm+p>k6pe<K@Lvcb^liT!7+qGB~JN}eGaxj|ZDKavIp2X_7
z=zv+;^^tV)Hc@ctDRZOq!z6b3@GZj{D>Tx)E_bJjB4IGJwU)i~EhhPOKa$2I=@PsQ
zey>$^QfOlp$(bZ8>^dZ{E#Ju&?<0UDb*C;i)I>h3o!^%XCfzbOD8!V7j5Y~JMN>S`
z8_u}DHC{ujp7P+f&DgLCSh$X|%1*|5d^yqWXg1!T-`?tXI=9>TPes36`GhqzV<GCp
zNP#URfw8C~txPSN?w#y&fnRL~07W#CG^=Pxtcr3B@)I*1<aJlmQZkItC6F(iETSEV
z?rTv2C~8f7g_PF{tb)YGsnD{`NK70Pk6=E9NI&OFyH8z)7<0SkcI4uYh*7(**mj9D
z@@I*Nm*A{Ye;>u{>7F&fDs#XKHxf~=T<t72%%rc<Hx3^yHOVZOu%JrsnBAmEaVjHb
zcBD2rInXigR7fkA-LEwld-c|-QiRk%tV|zL89YqShY`?~Dh2J9zl9kDiq@=TS-d0Z
zfmq}Rgx_IxJ5_&cY+cBHJzhCb7Nzm+>_{)1Xfgw)D{cR^ZRhivZs7Ylc9$P~hHssg
z${bU+Ca3nhJ~}Q2e2i-Ryw!T`GwimD-Sor<pC8wGeFRwr?&{?|GxTJY>K}}4>Z0M|
z<}qG*%tYj+LKY&ooSjn+CU4oVEe2If^C#8bU=6b*>MBA;5*H2en!^(rIq=*uVvN#Z
zDN}4}eaJuL#L0rg`^Ik5r2;bhrPJ8BWcNVh4?X2^<@%MmXsI)t53RQ|16RTQO|n?5
zgT?%$=&~H^9^yZ+4rY0E$aFrG9};sjcEcvOGfr+2K~Z3XwM)}ZAt01G3aiQCu==CS
zF5`X=6Fg!y17lzoou?{JPwL4zIRXSvsvk3>4Ou{jN@S<<!A}aq&DOhUST$8fX;6>S
z<-sDOcYt=(Jv_}r)4Q!4BB&wchl|zvKi_>?^1XZda9g9kdh#14KW1otJJU9Bvb;(m
zJo6|g93RjQGxI?*o2;DZ-j$`uf+UN%Y}GA#N=&&JhxIx!bz?s<nI-Ramqiti4wv6;
za#jwM{$3YT)<3L1VNUA>ek{j72>ftq62E8q)HY$P`wcY@upKm8VmuwXa*N8Jz3W3J
z_t=j@eiPH!w%?zqlT=1MzBEdFfN}jr&E0;=-0Cb~rq#MqS1XfaM-dil%X`dxK_u?q
z+J)(JV3)4v;$})x;l<<?ak_`XAic7%h@>7FW_2SEJoFsm&CCj0C++ODTgLkp;tE$n
zX#MM#&C>&`xTUu5EW`0;%S@KSuXOAyWv9ncmAeABJOV#PfkvPi(m{Mv0mR&Dk5>zv
zy@``6h&?JG&_S^4%P%IuYp?gu+<QMQ4`;M3RObcy<sv_tppJ{!jSqBI%q;c~kHZ(?
zkKUD2sti_7Q`=Nder2sVFJym>Q8qR1AitksB54yF&s7xHn>fY1r02lJLRChg4nEE!
ztPRLB$75whSec<IH$>O6eU&J#2e$+Zsc6{#Mwi~WpDe@}=rn~oJ`yXWzWR=gEIIKE
zeOE3KJiir+wy+s(ETUw^V!;CH+K~4m1$oBIe6@mq+}1p6d~e-o+@WWkwk-E2Y)D1T
zeZhuC5Oy$liyu3)B5L}?N|RIu3~<2}vS<~o9dHDWcWnwU*<^jihMuC|2jtH?O)PeF
zPX0vXN9P#>@o1TNX8EtOR*^5NHU#T>K9YR%9+7d)A&C9M7jaO%_Ul%v8qrnb?<$fk
zafU{&1S3&mi_#ag?l%O_%sbzo1WJIe*Dsi%VA0Ejtz^bD$o@fr!2rc=M{k_oZ|B<{
zOBT=a3w|mAMrmYKPJ1SxBK7@f27Yq`t6wYSM(v^8r&#tbyXHh@$<w>cCTu{}x%RZ2
z*se}NuWfWWF}-N-7H?X>hRi%`{8?4k^=hEoQXT(kd;3b$;?t^yMpj%ubz~UC^l?T(
zbtKi#FQD+E=Na0x@BJ9mQ$#G9nttgjJuqm@t-D@gY{~cPKX0`e8>*M_5~TQc!pw)f
zqrvN5^`um4^cIf~hp^Sqy=^Ht6Sm{5J0OAKG8xdHK<T+0jAc(c53oM1klGNORnRi~
zob;Sc+0?zmPiWJxKb*{Q>jI&$?+9&xmJ9J}A6ZVKB)ZI;u!^=0q^V(LQjQ`3QSVA!
z%eD80CmL%xtbA$=qPCwSc!x}-72md&@Ico18XuGijg2iC*c6l{54>gML*G5v=36v?
zKuMA+h+8cniPY?wTn5<;MV-}l!5Z?G^p{=99O^V#3nSU5zDEY|sRGAU!f_2&P*srl
z+S*BKwWL<>86{;yXCbJ2Fkqs#-9G}J*fh39wfiQ~A{4kYdfw<{Cb9``3Ker3N@Pdx
zAAzd`SzBZN7ug6chVH-JYDbkJ{DYyHq4G*co3#^>Sn#hs;(xK~n9Mp8;+!bt0&gE9
zOs>edd?zE*QmtOzSF{CP=YO?zio_>l=gyy`(sqrKoFr<vw-uRPh!?(vjKG<jz1U|d
zcZ#mILdv0UIOGMToWkFc3kK?fO)?cm+GKz78ruwh6r5h~WKa_qLYkmXragnGVH^v$
zvwx%k>YeUF*nr><((eF;P;(jEXhN3zDabo4Z&C<uOF1DI1&>E<!lmq7%}JYeN!(A1
zB!(^a`(ls!0iN^e8ygeV*2Kg*`AKD3!=2hMaasIeqb>e^FU~QNBbXqgce~Bh0Lz9%
zhtM(ckFF1e4PPQkK2sDo=_@hyQJDiAdemDkw|f_Co-s=!2}IJhVz3uI`)%&zgFZFg
zOxyw>@FbOVn#zjB@3(Y^((g8uEV5&_Z8kaf`b*kWa7Umj{il_7+YG56_e^d)5C;V7
zOs}LnpdO&&*%Ij`5k`wYiIj$em%|1Wton<a6NXZCcx_;x@*~I-5a&dF3)K!+I7odA
z&@V4_=R-|Qp8}<d%zo-`(#;Bng?e_AL5Zc<sBJ&M<cylDov7LMDV5-}S>kT*o?Y7a
zU}|R_|1Z+qK}KGteJ^Vj!|bgK2Fxo?rtg?wuPceAvY&%5Dyjlx#7~1$?Wn56y+-td
z6(0jeJCM_tB%PI)+KuRrH?k;8eb*io>ihek&!W(z3e2zly_fp!Qx3^^D`avp7<e+Z
z$ZK)kDSD6GVR-SS=Kvq7#$%pxa;)l-UlcD4j`m7_4K6CB`g~cOY+c})Q|<MZn6vzs
zCGufG3~{+rd%!!7SlNR&6+&!{4r+H6M+L~BKk(_7mCY?2at9>b!)Qvq(>2*nKpLAI
zmVYDm6Ww%cC&nu*8cJXi=i%sm)r41b9;bJ7n;OI-V6C*$l0H2@DJ`whAFHQV>+hn^
zT)GLg3|rXeiTg8?0N$H_{bVeByqOYl79egn;BoiO6Gf}P?$9?W<MNF*iw840Gdiur
zldu6Rv(@pEp)(}y)cA~EiW_7-PgPx&_+^cyDb?wd$Rj<0wStf#hTjd$CR5RXI!N-E
z8n1PF>D{S4^9ZC!4oLFVm->$w2^}S->X+t%BrM&2A6jb7?=Efb#6_r@*ZJTcveTV;
zpRPMp%^;|fL~_O!8boG~CBijB2{$O7hu7fgu+s*4hq6V5|0W;NH}w}@v?dtfE0g%z
zb7lJY`=f;?Un(iL5#Pga?4-&pf^XI13-6<E&@85e4V%|$?yWyAVKTtgwJu#~wPvu!
z)1$ZsFGaulOMm`mrVD&k^4c*oa$Tf%BI5ehK<zv;ADVjY3DsHmgpcEio_*3|%%=%7
zV%ET71e)!7YG}#1y4F~UPQpW!1=Vl(T$Y^MU&8PBuRgK4Br{zgr%fPaAO9Ai!Arbk
zJg?R@HZZ2N`dF;BykzCnSz1S5fxWjk8t6<(c|P@PbgAUHK}b}RwB``23vt@L<uEBG
zcI~STaT?guot#>Ni-x@kt)9+$UT5ZSelsB^TAx&bfTIKr_k!b!5>HXlm&)|B<zb#E
zsZJ?d21nFXm#ZT;PG19WzKPs~IBM@<DiHY^Ey@3^hK0!C%UlQ~mnlv@XttE03G#dC
z9G&@e`LR@ni_;wqQy{a9Z}|Atve6cd!2%WJK0Cs6Bq69&S~#By3ePcCBxcduy7<vS
z&hs7isgdwI(vPqb#w<BQ9uE`xR`?ma<TJt0ugMv?HOAqVCvp!G<Hc~MN2?CDu_q5^
z(>*W=1r)A+m&@p|BkmB&?ee}+lXSl%HE=d3uZ*1n)B(c&R+Uqz2~c=anW^!$RZ3xt
zfvLbel)W8p27sK1dU<RUo3pW4q=-m=2ZcX1F=6h0XGHyTrkw&$Ud!JOBK8knkS>b!
zFa23fnsKu4392G$lRS<5S~g;4W{N3<7x4D5ia!^%x^fxH*k64fb#V-JJ30<eFJ+t3
zln5O%CgO8A3)^{UsFPkb!k`v?^Z7(9vxT~K_(suxp(3#UB^LzRa6{qGX|Ha+XW*Yl
zUK5|Q?0gWgOrm?&UclX7KRtbrPEw+Leqp~ckmJxF(Ggv-<reK<j7-RHjlT3>Z2bN<
zdb0v@pusi4`>*ul;lm=Xe6!AFL&SyH_4K9?c*c@R8h;vt6|j($<vT2q0OmPP!V)xS
z^K!-r;Y!a_n-Mq69Z>K!RZgycLO(4eB?65<{Tp8Z;lf^gKSru|Gz$5F#Azjf28S4=
zXHNY^SAbvMi}&N07j+}!L^h0x`o!DNs&BIKhmbq6Y%JRNoB^yan6Y1;N-#^k7e&Tj
zy7Fc5wOLyLBX-Wm`ts9{^B`r*4HLbl+%)+8+LuzD57i3-<|n`VxJqeral{;VZS>OD
zVJ0}M+|0uiKUZ0S(VAw(RU=goGe8cc<Fs&q?;<(7+)nhMz1Gt1*SjrVhmvOI&DP6p
zM9k7olF>goqcviht#QkO%<Tc>(s4Fk&WVLq_GOWh%AvY@k;=*%UtwR~q-k+YHvC&<
zC@UYkq!dQVLoX_y>xR?4DHD5-vl*=y3r|Z7dtSF-^-k#Tbd9Pprl6<J1?(_GoNu(j
zNBHN^@=^`IR%Kt-*!UvIQX2`r&yfGVAL8V6u+V)gH#ZRU`|9KLS|?>`kgVHupuAlz
zmM%XTce`3(O?L3GZ<OKe#1VF<{NaW+e#vB-Ncsfx6UGg-mgj6;oo8E#drD-A8-WrV
zACB-ETWPS~`joi!IND6EFwsn@@Kg--5N$F(K+|Sm#|%KyWhc8J4U!=lb=J>kWd_<_
za=>=J%rN1R+-{{+U>ZTkE5xq4l&VJB=|^PP5wzsKfB&>~2dRQeoL+}<pn$F`7H>(D
z7UdF`EBku9{FrR}?rH=AXBH=99?*^<j3j$nKe9-Qu9pi4hY9%UoBj&_H{E>rupsz9
zgoD+(iPQ?%Qs^(SRxpFmU?CN(3aBRGu0=tw!mUw8pGIPvjVs!^+&`q1nN)5$66~->
zk{&Ox{l+v4nY_ECqF9DDi$ER2qA^<mG?liEYAcb&$=*6Yo&UuMN8dp1AV_3}@iGna
zGH~!K-8+ux+N9KX`$Z-Iq}MBn1dHbc_BGh0?XRDW8f{s&d8UhUA*x{Gjd+jfQ7_TA
z-DC}dwoeqbMyoqY0%AQK(5%I4_X(SPH{)iqF`$@!pDHi&sF8k($UN`BrUpiogA+NW
z8qkY<lBAN__1jmrPSU%i3KLKn(nENxlU%ynNw=+8ryGG`?d7tfI&=66aM>UCNTnw0
zy$^SrFuvhJ<38%`4PN08xEQUBlCVy8j&H~^))+SEDUo&I6`0sR+Wp4bSD6r3hW}IC
zRW%nav~~C}MU|*^a|8R13>Gv6f~T3gyo?MJOrA5|g$Z=e%5E)hwo=D*%+*hMbF?Mn
z*k_A!1hBdzIrn==C!usGZhIMRyB9QjVpYef`$!BuvDMh{W67XP8Xxw#aA8lCLbwJ~
zQ-gR|iCMT+-Ge3${@t%C03?!Gu!_>^_cRL-llaz7Jp~y#nVsHcS0(n-S^5LU?iOX>
zn2RQUG4UiSsiHM8_7^C;KyY}xTZtM{r4EvxC)hfz3LA81S_|5VIgW&VecH6<ZNglD
zEgPxt86>*)t>M0O0eo^*eTbAN7jAw?>&r_Xt2p+GG)d_@j)q;{m+lBApmKxHPQ4sS
zC}o1d(g6HAQhr26uDjS?XN-RrxciUeI)2_O6dD$Kv%0s>G>+geydzbOnC&`2+Y>`%
zY?_+JRPXZ3zU==fE9`r8+wVNDD-nC<#w*FpJ8}V9=pzElcf>qz>~62i>8VsuVqkev
zmh%#cd_~65l%l3*l#O(mqH$h85ag9oSzVO$G6TBV@fSI!G}qh*(wTbAUK~lEFlNHe
zZMpV~A`8Yd+PrfMU7jSlzD1CK)L$Pk_|_;H8eXmusMGvF^qV_t#99>@@hAgkqEXC1
zwCzj8{T}Kp@o2EE?S~{Eb3pK#SBu|HU&qc0EALD}YSdGFJ*H6V71zA#F|ua$**<kg
zQkl$yR~;OQ6Ya0RlWGE8jCkT6Hl#4+#qAdWw&yV9*#}90CPxwt{$Q|F+?FRGC0Gj%
zq@xv+$<Ss`sAl9l`Om)1F&R8HV8yOg_gukahD_BV-I;VvJL;gN2?Czs+(Hbf<fr&3
z-VHNqvyn>5k&31d!PMP+v%;r`q=coj3XA2kO{rmlCzqSWm<fZNVtVyxZx=5+(hK;(
zXk0@yP<qO=a~Tva+{B?V`gR*Tt;B^7hpvn$n<WoFHW<fr<PSBT!fa2WrUp7&=Y-K;
zW6W6*Yr-l;loX@03knmDcXgYRYf@P*lcC?PKV#jut4Iu7jLDC52(7^LR9P!Vlq}EO
zL?dAl54?IWW~a$L)}vWb2I(HSm1%?1l^5bGLF<=wdAlD{u~XKAA0f<*elOG^(uJu|
z7L&NZ4e2&&A5LFIg)WV6#L?V4E)Nfj8x8<aW?P3|zb{7b!@qkeFHrVO?Vv|PzbkAU
zODM!ELJD^D++B<I%6(rwEDD`r)6bLyzYDZTcrsEB(69V=Arg;=3uK6Md`nO}0>|{X
zBX9C_kgAN@3;~f|nLD)H^5wuuZFhQgcl}`8kgkvjNalLq%Ekm5o3dqY4|+(f4vYM=
zq&cvC>yR1R##)4#ppxZ1?Wy}}1F)M^&Y;?>FsX^7cqA#r5?f!%+Z~YfnNCxiAH~d}
zstYC)>=kn|Wi1{|;mm08AQxvhih*LMy8KT8UOt;dxnT^pCKTe~X`cy9WHx{(3oSC<
zjX0ciM8c?zvb?iY8{w|puF<pEwZ>)K*z7L~R#*|`PGW&z4^dH7!@bDAJY*feurRXo
zb<+q6&KC51SYU3qCO^%ai}-5FSiJaw0Bkc(l~3^tSX!TNg>`3|J3UkVwZr^-POG<g
z!a7Lzlp!xYb=69R;Caty8d{q0`F0<qU6bswz--aiubt1aKz?5OnVSij$Fv`da-~XY
zd?Qe)phvyDa5{W%Z(hy+z^7yXzce%c#hA=mbLh3!-!3$m^4HC4d$rL9F}*aGy31b#
zPG(aeT#Yy0MG=t82hj8|B^cZ$gKYPn%w#F_PrZ7O>;xW#|6TGh8g)>2ec3-II~Mrs
zR)QtItPw=jwHi($fRQZdDa#WY)Z;x174BipPIeB%O8HjFws7J~O{bm+ZKY#$WJ0|?
zRBl7YI!(fppkx$P=pyXs0&^>vBCX|_Aw{-4D@SgCQvbjm)+yo@NpANMk99)A4M}Dt
z^a3+0Bk{(IDhUHlqxgsgk~+KoH$mc6@?HXbcBZ3VDI#1SJPq#hl$j@IO|1kSnpNMq
zcriN->tXurT-Rem0&9o#O%vIG98pC16yS*A;!99_|Fb&_6bjY~6u)lMYp0n+hQ7cu
z3gyYFhIlX*T>)tWj-JiUQ(14%kpVonOUw|_`n~Yj>(Yg%|1M(w&Ey-h`|+Vj#Pz1-
z!Q$)<wIFs&so&t4?EW}0y&UTVd4%0G=qOBZU)u9_rTJ8M$P~zEs4gbclj&;bu0~Ux
zGa*EV`*Di)%WAput(FlhiBQrK$mLsM{~n5;=vJDT;kF0T4{rZ(I4}x!6XdDRHqf!N
z5o@@*{A0Azb{n;4u0JqncwxL-TcJB8G*awyy~&pr0EkM8U@@7A^??b$bW)@D;#?q!
zU`psxLfINMIh8lOQ%bhVM~=1$9t$KEy4p~fyCbrwJYq@cgB-t7>tbp$puwD{{;k&N
zP7P*&N5NY|H7B5s<};25!xz%XJFRAs5ruT&qYV<#B~cHgsdi(DOqs0HF;rF4aQ0x8
z<CxQ1gkB7&4UY+iGuQc-hdV~!%<igsfL|PncxdMserU>g6uC&=7lA%o@|-fwJXkp*
zV+43@mfcaG{jf!c)BbAn&vyQ?z-2tc6NMb%TV*WK!M8cEB|@><48$q@-`~GC(^RBt
zt4vLOC-pU<;1oVFyKrw7rgHA5&40{{V!8_xLS|NC>KC_Ycm(njIzJP4gWLj}nx=F&
zKdwz~mP4js{*o(!_v&Lm#$`m3e&c<~%`L+6wfDl$=<WNzz8ER@tKR3)c~$nLxFc@l
zc|uAGd&E_T`q!Y4khH^pFL{U<1M1nXA^sqRl|<31vrF4-6pb0Rx{8T&%zmi9+h4f2
ze+_n>B<M@C8-;#U6J;*$rO^#?uB4X57kZ!`0%k9oZ4mt~)0%a<b|A2W?JdF9gop^|
z;y92Je_<g;H=@{26YpZv{k{+tIJSB>&zaO14T7VIw@()mA6QBk!!>x1ISz=ZIAtRD
zyJ-Hp)A#Van`3St9><2NS-r-AA{QtXM^6;yIszE_N7(x}acltR8A*y9+jzOp*q(3o
z@a>q*-2b{gDdaljVh_=sZ*oJFA2MYUoYJO<Zv+umP~#77s+))?ffhRMD@4<YzHDVQ
zfD~9&m;Q{qpUz#Tqrz!#UqLa=CJ6)PyAuK+^8z_7>vx;SIWszfSk<Gw`byK<zz)Tw
z6UMmn=max4PX5!{XbjOvdIwJPAz*?_O6t$|k&);<4NKdb`uLT0e%__vNmKvS{I%<G
zlot21r{q$t`1<_vu~<Iq$&br5|A}GC&mCGnl$*k08-9mJM`X6}+JH?n3woAz@v4_X
z4JOz<3wHuIov0fnH>#*^4UaZIk4OTVHnBTLBhV}b(Bh|6m}#qcG%@>u-1}vbn4XE6
z&kgF~FtCk(&H5es3cK7<ADWTDAEc}iwvviLj3fG}0p`EuN#FX6+kty;4y7kZ{s$+}
zo}%YRBQLGD5XHeYZA<VqV-lBav^j~H`)Ch`YI49zS015JXTVb)!Hn?wK^~jyYg?l=
zC_&@du<F@ISzCF%y`29{(eRq;=*$&yLn*geL;&HJN-_7XGtrn7AiLDu8TE9w3=*Cp
z&rjweCG0;Pc;7<7*&*O$PoZ%z^Q~1RAaO4lAL)fXA2<h=I$#C1PJew#M@FShT>{wW
z6pdk4r<bkM0bm(T3+V`W9tptv!t0nB*LvOwmvlv^dmHAKK4?p_1FudqUPhqd6AoTN
zw$TyUXRb>j-^?>cn)FYzDYkAOu}szFN*Mjm##nsr><Swvao^cx*p`P{UYNn<j1e!A
zXD^&8*wn#)z~q!9y5O0Nf|L6;UhbPz{SHYP&9-s|+ZJI`TbQnpIKnonKa_;nq)~V0
z7_FD>KfbbFs&B@K1sZm3ZCk0Ei!;nepM;h8rQN6l6bF&ateOl>{OJ*3TgYFL^v>%P
zeGRzEI;)$2pB5*N?Ve>PiFB6IF#2&(*Ua}ijC`K%3?wnwHH%5aHd_s?pJWI-@eosS
zM()Dk0C1M9LH=|CA%SZBLZC=MyglD#VF2czw+9yGH;3XT5i9c7r<UL`OT!D)8I+`A
z^1F6;S}q0MM)CY;O-7rwwD{h9xL7;NX35KxnZkvH(g_#W@H`0QbjO@T#Aw8)uPC>&
z-tkOYa)gZx*zfa(Sut(0K^(7VJjh5`Kt^@`w3@j`(V<-$=XvskC}4XXc=e&Wr5Hy9
zh6-5gjvG~(cnYM1*4CEKN;XAJI+Ik%)iL79^<|8XY$^_->eVxVt6cV(GXHx!LV%bg
z=H>u8Jc-cx97QXPl$$4BW~q_+eusS#;gQTzP>W{$Cicf9Q^~jBfqK>Ka(pafM!f^o
zRJ&_;cy;{J3n$3{ik-VQr8Pz7Ltj8Z{BP8~!zkrQ@8P>%m~6Tpse#eoyJ89@V{{tB
z$vz2i+rJ)ldye0qRMd<KF}-ubm&Y8^RC`f6YW+6p{hH1Tjhy`Af_-Fr#Vx)^JhyQ*
zR+v-441~{vbr;hFF%pTmvv^^@1d-Wn-|&T7^vBFtqIC|7C=HA`;HHbti~c0Gvur`}
zzX%tjF`TwvDjpiOejistLcjgHfr3FX{8@2;&B?A<i{CsoYLCdeGKPN}Llsd|T~5b(
z{_U+HyA8$BADEFln74z_WpyI8(x@po^*I(<%GfsAApZW(((tG!xH2g1#xZo_le^?f
zFf;OO@8QyvevbTJgsy#u+m8{Zz@!%I<COyd>zqz2v6N@Tthe|#yx>31eNGy|MA@Gp
z=n3$}-ZmQYQC=*MSn7BuI=uKIrtI8pO+O?JJ7hl#`hDom4D)Jwd)oMPy(q7y;zUB3
z>S|CSEGHF@roRJOv`5`j28}WbQ>$6Ls@4w4ct)X4@eA~wU6J~q>;H_ZI8V1zji+e=
z7lk80zeicBw||1R=U6<&hx&eas@A(NN>2y<A?0AxLHMai7eT&ghrOtMk)DH7=x|Md
zZQRWYPlK2HrS*ZiH}U?=w^D=4mCM<c_#d$8kaxVN><h7pG6vl{b$4CNTHn8~G_14h
zc`m8JmJp^}Eo?O}DlpSZcXvt%+4Wi+z=*2an;;9eM|WFagN}1WY3zJBY%TTn3jQ0!
zPPsy{vT~ay^$9?y82nzn+fazUjkmUvdeQhk&%{AL8hT2E%(YQtsL0RK=_faiH=o0*
z#~j`X{hX<1SFOP=ztMYf70wAdxrS`WxYHbO4~>R?BEt6&L@ytyARVK*tL&-!n!Fp`
zm(B_1I{X=~C2G4Y*|$Q09SCnqG?jBi;*(PTq*T=C#2S0W2DJ+{_5xTJwD-o@ZSk+4
zY4;&siwV#eanM<;ENLu-l4C&lA)ovpj_u_P-UoX!)eg~1YNkIC^<BRM1u6A@y6f?j
zNcRBF(!}wz;K4$uHRS!2PsxS|Ox~_4EAFJ-&?Xn)aG{r>1!dS-ybH-~w;G81;U*6w
zk(q;mzK1U}AZgsIlKeCWoXHw24uZ1j)ko>PzPUs2#R4A{u$BD>#x#|U7uzBN3owfI
z`DUABw0H~GOrlc=#_ed6NWff%8rM&OkJO}d=R*Z`6B)wzZ%579^7rhD5KUjylq9|1
zcLG&6Oq-*^pqbn8n;c*MPEY?mLi$1ugw7+@Zwl6!0!Hg03rn38giUq7bkl&$&qME+
zCGGZAw_lTz-$hDENy3APenxwRbwW%fWA~^nUPQ%HE4e)oy=T8S*VjGoH0Jw>s58Sk
z>MJ^J^|yo2w(JHulds|SiAT>hz01DjTC9k@)=GFSb&chVWGANJW&duD&uLm=f6FvF
z8jq)>QW>>AiH_<q`2Nzb#qJwbY(;e4A5+*(pAyGDt&f{%Qoi>aQ_$3+fQPiyy|*u0
z2c92hl(hi1U+jj5S<ZHWUXe^5mFt9)QF*l9u{m%QkTZ;?t4K4&7vWSzz~uO3meLg1
zR4=Y#*zdwD<s@maZ4?K^qO8}~TkG=I5;}_!U6B{7bv7d^+j4D$s|M=t>uppwzO&HZ
zF_)2|2cqM;fb{4$7QkK#s^=M?qMup!Ya73|EHD~G17LfRR*kA4Tpo0P-@WJcUQdgE
zFB3{A7r7$VIOe<kJ`VFRX-CF^CR&Iy;##EI>P5$06C*Z-9+e4R`$(E&=l{NoGe8J;
zh(dh3*}zrL6yACLEPvlu%{<7u@NQ$Fxp44&`hKh1ZN)P}+MpY%B&;9kdsU&^xd*_Y
zt)^GMUg!xAt6{|M()TD`W<-GWCBEsaSH`ew2uqrjFPF4~ZC?IutfTrp_)grRp{qsM
zo|J3O3!i(XNt{SyA*~$7wW+=NTi8~hm1GN{=vvjSR1u!<NZc&S*fi#&EWov2$*et)
z=;<G_7Cwt$*F1({xYt?|7u{Ryl+T}&ln?mi4R0*c5VT$#y;1%xJe_O^yEl#LUU;9p
zwF%$<`&J*Ipi0aXX#i6J+d4%0B$<PxCK)|pKYQ8;x9a<BAD0Ygzn4<80}bw;NN&Kj
zf+U#WDxp+BENsBtqc3X8XD9L${Mf?u$J#@^VdKv)`t-ik7I4P~&-$#)n6fD@&fbHo
z93!_iJeT|zbYGO}l6H8|qJrS{UOQ&T`I4m4nmhaD)ek7x(Zxq~y8w<^T1d`24wdj0
zp_x6t9_AwX3g5T?;rruXUS3=dafQYpGH`I~`zE$2W>n`+v#d`V>A*9reEDmLtn|e{
z(iuf~1-v2E6j77QB2bF9>dAw)1nj(c2n?s$2&FVVb?#aZ&UV`@qJC)okd}ifF<#=?
zQ>#0JnWuG`YmvZG*(LAmdoa%F=$S2yb>fx@yX;Qvkzug=&ZVtcdk|-*9IyL&21yY7
z#j@`o_F*&W_-a*iD+yO-^_+GEH8ab}x1`X$Lf9#HAZQqNA?bc`C&^qoNvOmW`OYYv
zmH0z&|1-`W_`Zy!50W0IKQR51BLUAobHQ$Gjm<+qEbYAh$k~Xa;GUWs8_9t{6=k7=
z)h&Fla$S~P48C6<w7LK_YPXe|ay5>fC-LdRHZX~|yGD-ZAi?r2cTEN}$-dlrpLrih
z_Es0)0Z-1O7y0D32mPUbhzrE#jyXS&`9oVSQ_{B@e1pUkiiE)MS@+&gcL7VpEkYh+
zrSU>Rxq1=0k$xJ8@`XVWHMBvZVjT4j{{P;H;6eH2;$cB23OYkxAgT1BWiy(UT;+|V
z;xBJeVEPzb7+gZgzD}u%_+h{-I>|V0sL}spJdYIa%cfJP*y6tP&}Hrz2!2KLiQbx)
zfi})YPIVbDPL`i2(SNQ#?zY|ibXz8bM%Pd?=0rV^BF8TAV|<@A8d)tFx<C>pQQLDf
zYdhBx#EE*AgPa^>qVYR>R&!^ej~qog&?;7a{AEk8J!TdU+WkWQ%XNFSUM%QR&CF*u
z@;%F*s!Lo!^w^-&;_-|H?|~pW+xri#x$ZyNrdoHg+@A_3tNwz0Za{pr_KQ?nd<7mQ
zd@P7AT4eTAH<M#=I;#qcM{D}Ad-mwx#P9EW_ouVj6`(#Wj5acQ8?o^%-;r7kO9JZJ
zsg>PNPXkx~RC$ohfK!Mj|AJd-gc697{3#ka7XtWhJ!F^hq(If2ujC}-4M2-X0(!x*
zUw!ukmHk=34mRBOE@h<9vM7*psNUXR|IODY^2DJ%i-C?PmPa*4tjK$PK<Sl6GQv^U
zA}Q(Q4KjS|!_|BdgL<?N^4Pnv9{cTq-+69tm`I-6!CZu_s@am~CG#%voPx98lZQ%s
z+ydvE50!SgpFN~#b~#6`b%`hB$}@gWQ|}(wsjRczGr%Xg^tirI0$t27v*ZvlzfW~{
zDqA`p>Ym+pUF_1Ax5SH9+U6Sb@a7TW9LeC@d8L}5`CW~=oZVZ3PK37p?`O{nOLjiE
z3{a5hBO`ZLu~SdSHMTz^Bh?6)>JRqqyx8X}H8VrT1!!5YOKS;aknGkkK;0k6s|vUG
zIkZZdy<IAGqG6r-GG1KQQYu0m$%%my-I$;5bH@mZd^NlqefHc)t3g=?O9i&k7j_EI
zIZ+ra^jMGJ!FPzKh=@N?dm}lpr)=&MJz6&v@ljWGdWba+&U?DtvFPBJ>($dR6ff-7
zKxr}qjVdVvnp(71sdgBa<)|I2o@l=L=gP}lhC*(h;%`CoP9(6F0t?!|9QAdM>7*yI
zUS5wfLfFf$Lo9A>Wl{lH=4d%okH7JyLnT9x_`<Di+=Tepug?hGacNvGX6=!I1QN79
zdsMq5=l_SVw~mYQ-TFoqsi8Zhd+1WSX6TSqx*N#>q|2c@B&9<@rBMV4=|Mn1Q0WE%
z=>`#bFP?MW&)NIgzkT?Nf9JlhdtGaNYq2cNkfWQo#to!gHT#O*ap2GEzMC8@w${S1
zha#{V>fp_8^TY&ASJ4#^7yxIuW}am1(io8`VKaeNA&Z#LINhYJ!19kLIw8XL)%oG7
zL*>@rx(_#{-R=NZkgJF6Ay(^K=Uq!2Fyq*rh5JiVtduRP`k5Xm6n_3Wm`3*fbMTSm
zKKFuQvok~@c~`GuxRc+Nh@7%~7zj|eAAHObdJ%CuC0ZDqg4RmLC{$4l=A6Q0Y?4!Q
zRoRhU=SSkRJ0B<9fm#k;Af25Tv6@4E-4y-EFeBj|QG>@m1z0`vcTcLlzEyB#kIle@
zs+XW=5D5aM5<YoH6Pn`fn9+K_Nl(Au0ZS2e*0VxWn!d<b8-qhrjXr-nXt!Y-<*CrW
zx_iOvI9a}EZe=$naMZI2GbCVkp4#lORpG$Pmqp_kp+&U^`g(l1xo?{M(R!_0a8uHX
z(Z0GcTpm;wU;aCy^83u%*aR){bxW<-Eu>+y2DM9Rqnvp%KcC~uEMT{W62AT%J20pr
zZ*5JBWI*rQ|6xaaEdBh^UaMf>HjakbHCq$?9D&8B_=Aohg6%W+{*#BobzRS-N6e(c
zaSiw^y9YTpr5ZTA|A%X!4@lZg-`2M$t@L4df13aF?ski2_D3sBf(u&hOs8g_bDWMj
z3Cf<Vq^^PWRkEJmEB+6|eHoG=&ETpI)ZFV?JDm^=GI}qxa*Z@oPf5!RYwlH{6?=9^
z@5g9o5&!O3Fnpex>M!;8aL8Z+R>oRbC^rnHl-?+$BwArkC2fN)vTGh6ZXO7TG+gx$
zJl&g@rDN_pce2G|d>T^KJ*j?RBOap@y!lkiDzhEu#npTM@r?Y+IDS;9lF>nX%trtj
z_3AF3i)5$OnSjgj)ZIeoiaMu#%!HSs1kzFG(H*XV;J$_F8R4va_je|})@gt7zwp3O
zgZPeWK=RzIFV)c&{iv_SX5W;2AGxj(xFfNGv-p;B$@dDgX7Sjd8(VmC^hu^ERlPUu
z58AneV3|EJpJmn{{z9Eh6YRDSj1PPkQHwdk&7ONae`+t7E=F8?6{46p1qmRk2r!sn
z-i*naQamfbZ%bSSP+CzI$B*VJ)VJ@#g5F;ZPx8hX%upv38@Z9~%IDS<xwyX{)oWu;
z%D(^h#NMaUc%|q!<A;~3+Z8s`3*;WM8|4uM^a_ip>w~~s#Ic6s40vMABey@di-jCg
zm8`O)W|KtE7XKY=An^Qz$GLO5z%kgmGNq`+%8p`|5x;MMcD>}&Pj~FghdfTxd9x8}
zM&3yAxUJSW4mO@Ss+I<bU)W+dmD$e6Zp0fF3U{NJc9dD&q_e1gYfbNFUkR@=@c&&k
zIojEj@oY2pK%U*hA#YSusb1e9^t8vMPDnu+-iHUK&_BTX1-H?rff3#k7JZZ@ggM)#
z`SJ$Ca{@H<8`8aAQel07*>rnDI=myv-N@xQI8FY)8`|G>j(FdqGc#!CYN=>{)C(ia
z6Ym-uZoy{zvFH1PqH+M}vtG%qd70nyrS7Wva7pH@KqBGq3*Ip&TSCJxo}FtCCvM4E
z0tVTLw^_l6^B)iATA$tXbu&fH=S~undEgQv_m~BiFl)>|MbP-xtm=E*`6Is{^Bd_Q
z%#0J#WhJ(<ny5NdQF=XAGw2B^G~R-v7O+JcX$76uA0!w*AYdOc8yhz!yuTvxzVB4z
zVRoI%s2ZE8$KOnTzWI*I2+}`j{PCk3%YoZ->RIj<-PbX#0Fvx3jqV%x@<q;LP5M=v
zlzyhT?=dakMWKPcX*1WE1wmU`(868|sc_S<j`Yp;Q)o8h`R5##6PLuVgSrgWSX(_B
zBjy5M?y7&_>+i7PoAR9_=k~4b9rN$`8t83d$4NjfH%8*h5l+{DZ%6eigT%8En;Td`
zV$U9uUCW(dsDB+PN!~Qh>Fg}$#oL|f8tah8LzixCa&%>>QOvxAGla9xSgwye@sRbW
zk|cCNl|{Z7n{N^Ic1id;u36ZH{%?J531K20+rS<8_oceL8jHnxx&%z$i0Kr*P4S;z
z35HVlw@oLPr;PaX2%4j|i3vS^{UA>;eL#^P<s)yhF;bUpo#?;40c5K;CI%VnwB6zR
zOMlz%pwG3%h0z!g(@`L~4Muvtf5DI>+z*@O^t~_@zqjw)A?80%=+;pI3bOG@u1l;I
z=KFq*TOgR%!Al@&YmCoPv?{luEh3Lej!%>cYw|6ji)g=fCS4u{0V@0Z3xR-jg#=@9
zn4tZ4)&%mawe8A`)%xCN;+kgCk;B7eQo(<S*c4l05^%$>T6zW>2+2`HtG0Bj<VWpQ
zJyu=XCx!OK0e?d?lMcADEvq6R-ind{zk6Pt>gS^bSd6gDNcx!>pRg}tG(6LFcWRV<
zt@aTHJo7E+rI`tepE$!Ug-YUuSde&((Bg<UUIW^2J4+Z41-q<uoC(m6lg}%i_fYD&
z0EYwd4+{zFF^#}%tq(mLdc0%LTo|&&d=w}_)s*ofJ7bu~Ixoel^PYzrYSbZ%H}!a3
zPK&E~?&_Q`uzqFuF3I_?L>)2lb<mS*Tz_#f-e8fhw4??a{oU2^bphx1T_l<zlF7SK
z8+|XDi9E<VL~A-X2K{%=eNa`WUIKn2+ITslXz{?{3o9&bUJJ!|evU>&K@uV9>(+hU
z3)_+Yg;VIok#5;-RzJFapeYu+TIC>N{cRkQUH6o$zdA392^Uo|S_TuiKdu8|b)31c
zZ*I}o`JPrF>)zIkr%#x1_>6np=dqFOW*@F33yGwn1If6slw9njwzue{{k1s>=A>8h
zp&lmRS6{_Z6RbVdr#Udl{xLQ<@;Kc&Ztyo(VMg9FxwIj-fgI*FU)Qx^VuGkK&j(>m
zi89{2iI(;gcdwO$pL24TTfLaI-;vovKHAGYT+j5cdnrt1hB^(Ou5%Oc{Kc9iN~-bI
z$I!r=>MN-k>MZgyi@?M5!F?Nj&A#F4<GP^2O+w`+>&8l(o~}egU(4`HcU%B)5-swg
zcD{HCJez(zY$8tSrhMMDy=K)+I~Da=ony7J-a>A)BmZdU9aj(0pb%LSxveGaA6vCr
z%n@!Rq=z#3`Rt**Sm<ZU5Yay#;!8MubFE40On1dd{V-}!XI6(l*r-}vnDc%T^rc*~
z2zpJxX-`i3Ba7rrLx~c?u6#XMmhLCzd7_%vNd=w0Ki(1lNK?FWRe4}ZdoGPR%b#z#
zHbwlKU^m^4W(EQbX+;^XeV?YX?p;>zJo;)(urfqV;*ywNgftD2BN#C-JI}QhX^RJd
z=9aTej8<IKtdXdOxh($b4~_mylMSK_qUYdxnc8fEP&Dpt=hqhlp~19z_0CHrzVD0<
zN{(hm7K&yi=Hka;&6whyi)tf9OP1)a%JPR?4qZg9c2;5fne3Z_4Gc<r^c6fCbwgep
zO?30IMkW;2y=f2+qS3LC5hBBpci~{5N}xEGwfgb^vDjGWF^N+(`w1eMw9^YK3>QOu
z#@(pN?>G9SCl)wO(-9)m?0w2J5_@jGk<_`{E7MFrlQe(Tv^;efF6OVcLwarZrRGU?
zv$qhCWcTy}FG%&HA}7w8QR9(e#)7wg<@Z;n<Nj7IsQzzwczeCG#jNjBf8ocU>KA5*
zJ^7u1&D2z3#~*5!y~lWI37ZqD7p?@YKO1^ePm@ZBM8$RJ@d&z4-uFYFi*TR1mxuom
z*WzYd%wrsX;>SqD;n>Kg^dT<KVhgQkjTzkZF*+LZPrb{A8ZgHY-Z<BUXJ-B$OS`w#
z6s2|tMzA@QbLUOXuIsjVF@sVSRq}f`nk7m<2+YdYV|*W7iH{*#vox@{r;>hf9G}m;
z=C)hgQ=W?)eaKKX)rzAT9|a~y^EapZuW?A)a5&mZ=FtHh;&b<qE30lSO@W4!_EFK6
zEwlN$!`BcW%#}cSySh)2jze12c*TNa*z8UDmXdwon&IsWslA?h){8-N=GJ?E^5N7@
z2Qf`wySvX~guDacO3c1abD?-#X@#_hh8W@psRO_N>Uc(DbHKHf+x*9bR$n*9wGP=m
ztc6Z|S&KYrFFcNR!k*b#I+=>3rEjl$4oHHR4(hL`e7!}1Dg}v=OVMc^M>iw!EV|}Z
zj%A$wkClR|#3Cy@#;ei*4z_>4eF|%{!Z!e>=AIxp#NItIOIk6K-~Ck>RkwX1QzceM
zOgb4F`&@X}O5~mJI~V-1aWz6$S`9(#Y4jb*-N=b^c@GIzJh#aj%l1}WrPu)AmrK;D
zo{<Yx!NN0naGygnkeTO12Hqp@aWu}EU@k*k4G4*k532FaqgdV${`#cq8Fg>9IDgDv
zgW)7^is8E4q#MyC!1K)PIZKDK2g{~Jw>5GZ(IpRI&W;PL4hdR(O<`E?`w$K3&MYE`
zbKZ@H*xX?no)%#M)XLH^=e@5XkF}@>whW2cQ&69c$jMfM*Bda%)hV-~U1<2luUNy;
zeWLh-pE?vdT5-z^g*{JXlBxfJTI4-Fs8^&3Vhxf~NvXv#`xC6~93)hiarFC;(X3vJ
zke4U1$i@znv~}ddrT318iJ*K=`$|MIR$JTrGUA9Rh>xM?wR(YA$eBP=A%b#rfs~T(
zYH8-nQ1i2m`)J|r%P?D_ZkBx(AucM;&w^m;P|&m|8Gm6-L{A&DJsFJ9k<yd>G|~La
z7qep{X>|=EN_#q$+*5Gn@ok%tz;CZB&Wu~YYT9<{k#BcVtp29(8wLsdy`(0Be7k=_
z44ypPG$+>(D6|jH*><q`5#^uV;KLrpPaXe!!gb8b&v$cbAauGWI`c=s;4r4r2%Uf%
zD0RnYz0Z#~>iVw4em1*>UH>6=lr6z}P<48%Y%c$F)f{5D{gJ}J-4w21OXVip91SE%
z#j~5E>C+?+s_;QUe1mm&;emKy_^Bxe&vZL&Yn#vNT(y@U5naOCJHI)951PQXbE`iu
zIWK~K$+4wt2foG(uJGA>@KMxd1HUT{uw~n!1dz{yDjKm}yF{UEa{{i%S(~C?j@EZD
zJykJo#bAcNVw;@9GrKGFKVE#(?P!AG9-84I_IayVgWtUq42u#|>gWh?=b0HpH*T_D
z5r0oUIj(7H68H|WSu{opTB@k6lo6myjd#2%*nIF)*gJQOA(-8y+w;s4&rFJfH1ko{
z5goTjrww6{CVd|#zi-}&M8q*=o5$rF!ap?y{TbX9#2&}Y@NhRK2_dR_gjLL=I5JnV
zXjVkwXSk*&4$iT#c6L@$+psNbpYl>yxEwmhyxhUH2{ysU*WVDQX@q<Q$H-cb!#}E~
z%^oBinhgCRgp`1cBj9f0?W=ksEa<i@mXIfFeD+UJ#CNO(wjL(BF&_orZ2Wypz$lJA
z6NoHN>>yDBE>Z<<Ngd6Rr;N8u=UaX{L9d=54HH=gpfG_Y*I#GYJm?jC*;LY2+4w4>
zV``E@o+7FD`RWxhElEURDq-D_vuxETZ)8%npv-&6;~(J=APutKJHr5DkeKH0{NAl?
znS%~bl2)x|>xB`;I*X+QAC}g)^d|WQdL}11E=h;Z95zE=t8cN-KW-M?V62^5smHhZ
zq46eL5H^>U^y=<AtY5GpQ%d7|vvOYAH1BAk3nrUD#O5eJ+a%E7PG}Q->$r$$lK)tg
z?`2SI5?MSy7GkL7O@AsLkjN16SI<?zBkCTHVyn^UeV!S2fgOANt~x~<fu~zy2}Xi<
z^+bj{c@TxizWhCD?QiQ~rmHfA_XbJuQ6uzyhoo*|K7I^I^izqY(LSr+Aifu7XRxt(
z=a_f`n35-+=LAVgZfeAE0(x0lACve)3RP7d9B~_XfM*qygB8MAuc97HamA9FMiyIk
z;nAxIRE*(r3Yb?cI}5z%iyd*2qI}&qGyWGt>x>ccS#8VfUQ})+TC`#v2LJdP-($_c
z-%S7l0CfD8UgQz~*(tekQ`8{do$o_U6o7hxZG3J2im=t9BHXJ)r+AlsM6ND8AarDC
z<j_hyFKBbklXrC5Uo989Bk_?^q!CkIneU0^Ym?Hw4_*F;DlQUj7+^d?ZYXNGNHk!)
z5Y<qgL=?(Tc4?euN4-Nlklzq&7xcUobCLdtzu|sqT|0UgP<nDsD3Owpk<x(fpqFee
z4%0nunJg}!yd$pXn5dfk$k?m$5^i<NBT>`YAN*w~Tuh)moycn3I>#&K_3JiX&l_tV
zFSE$^SfW;?SNblVvvK%XA~x3x_y8fhC$c*mvm4!H6+J7mBhhy6s_kqg<s3$PA`^C9
z5V-WK12>Rhr}w!$pNHf-8s+)3rLqc|kxSa1C!ZRzaXv=I3|z%%#mE^O(>9E+re=`D
zpR7@pu_?1W@h30TTTNjq8)Ag(;*FEqq+7zvr?T?-l1YVDQaR0qdl6f)#45>z$^kB7
zx~1}|WPBrsOZ3*^`;1)BJ50|fiT#EMCu=Qw*${O9h;_e_p5n`L9IG^i%Z;QkVof<`
zy=D%_@}$OWk%Hl>n*o0AiM{^VYo7!>_JKNNm4V)-XdfD2d{JlNk^;`xQYQtLHR<k7
zr3R&Yh1?(w0MBaRJYVUwcOb*2{@2*kD%oa&?eprMkJ8(H+w3l0vq|mBX3Kjmg!JZE
zh@EuFq$}Fz6@D4}?V1eag@iplvyXNpk*l{vML#LWC_Uws^b%r4API<zN(-=-natZY
zX}dn_(I1Eh<8^FkEq6ywh;0OtELh(Ve+c%N-9{S?4_ppDAsPu}=&627Nkr?7TVHQA
zF!)H`%}T`cNH~>u!NHE5Ntjn3!dAwn$Ry2)nZt<~#KXBX;v96=e;T9+Wn!Zjr;fD_
zB{^y!As2_&r11V4AbYmPielto!ZE5R9!Q9nGd{V;8ac0Qx@p!Mu^?YY<jF7hUp|2!
zx$053tTQH+_m1;OrM99`N!JLvv51o6>Nf&9rL*oZRf<iSkNhWjGu)^rblh300yGQc
z^ZlVoXO9f^m(5sa<tJwD{7!PUlIbDYPOd?%7vYL|U9xHxO2#%i1TXFGChB!lyB}Vz
zA@Yu82rS(B^Ca2w!)ixWO{Id>5X<jG`SK2<lC!Xq=wxD6qvw~FL+&UO?nXy@taX~x
zQfLOy_fzXkVjy-hyc|5Pk+t3X350yH0W}5E;gSY?tFTVq-iSH*r`D_uLZmu|*^w2a
z45fek#ntFD7(WTn`1=FBl&U@)bK2=}Kr>d^?Ikof1DV!_N48$ZBr2^sUC4X3;lN%B
z6(x^s;T#y%gcnGO@MCPf7F0<H3_FQXvF(lGejbq!fqf{B_S!nsQ=`(4{ED<Bg@l3a
zpPD!R_g$wy)4~r=BT1g7Fk!x`iftHvZYl%osQqmo>Jcc$cfW?GzOKg$OR1VF`3FJy
z!|e-dpc@M-@Y@x&x}(2u6dNx(y3#G~ZYu3d|AF7M=XAmVu{!)@?x-CZhMVQF{Rd-?
zXmS3iJuECfi`cKl`cqKo<USYDeNJJCTy{>3f6(+J`pr-lVs%{6RK^R_LDLU{LR^ny
z%ysyXKcB2!-aFlH4n1b4pX*56EMn}9LTBN7fPQF2m_kOLbaDY*EAK0PCxAOzn6b=o
zb3w5fbo_8|P))}$B@sF2P4=^lQC65({54`gEe@usMvEhqmhJ&}Ge&Djvq{wf)IrmQ
zX66fF)-~Ih^J&$Akw<Xv;o!MeUwnv8mr8PcB{mzYGV?=D+Ru;`;}v+Xb9Tc{cwX#3
z&yY{=ZM(;t{(u?%?fRi@#0T%tCwxFfxEry4rOzzQ0_lrv@er@4aiyL8QMsDGVTr~o
zDG$tq@Cqg~LDxPq+B1nei+bkG&-R$%e7{-#j0Y>2Ltx)dEfwZ*;1^#c{X8DtDeew;
z{y^+=ErR4<RM;dBRfyYP*+XZMIG|@nLnHRumm#5#VAAyS*dW-OYm06D;KnQ^hE2?L
zPV6I#oeBi~F)6SxKQep!A}BX$i_>cr%EAd!o74>2<e;zpT53lJ*P^2bp*I%<i!wJj
zc5I5!;m-;o`dmlCFVgE^_PPm`9BL4bweo%zere+Am51DlL0LRmPb;A>a>!os%%<ei
z$Is=luKi?V;rT;QV<8(IGd8OdpRApvt`)DOriPyOq6gfANeSN^*X-GuWhF6gjw%8z
z!WU%0JH!7sq$IfWG1g=#oOJS$yxhPeGV-;hrLAz;c+g3HFMeu|<qDfvpn55SIrjbm
z@;fVc)l3IoMX(gH$6qu!|2*8@lJ*Fxn-a`VOnttZLO~XuE}5!=?H86!(-rg4hcMM(
zkMZX(>E1;=iYs{{8V?4+yD$|Th`f?FPXp<T6bgp;2h<`enhI)gT5IMWB5I1bwF@z_
zHQUwCJtRS9f2r12-iwLS7?qeR=}I-|y+}XkS{+9VI)5hKadRf#5%3%Pm7>3O=zUM2
zwA2>`5ec?Bq1F$^F{t4wUC22+f0sCBeqke{aVBNH^Arm1j(CI2Q0TxTn2uFcbHP=!
z|E?5#Z(hOEGy+<P<k1mfd5B3pxPt?Mf@-T{E}=_WLZ#tn@%sAUUo8y8U!zOU7_YRd
zCwuQ2ZZ_Z%mV_?{jGt~$0z!)&`=zHnHts!dk6*~;#%QY>9ZFLPUmRb4Q`oOv=o%Iq
z%T0)o=N<>Mq!_VL7ewU~WqkT<s>%_evZtJwZp33IPDV08(S7EX->wRnb1?Y8JzF-`
zk_VzR7Sen@jYmSZ4&}`35_p0-AVqAnYjU(;C3^o8>L&OGZBZ&XAv)53U{ph1^Ccb$
zE0YP7lMIz5`&f#LS3xI)0>^VaH7h@AuIllc?SN<OfU4GGIJW<a*Asza0S<h{mM!SW
z5YfI~UxxzI-OK4%=<p*LB19r>r5ho8fQj5+fWtMKD>)P)lC~9WMU0Y9#ATG*l9O?s
z3<_o+6zE<pGyUt3T0*G23p*=(xph$EVJJo$4IES@^Fu}3j^)OCX1dbZMXQWC`onpa
zt}*(VQJXSVqCeg8utZ0Ln>|M;i4FD{c^OH>&sS3E$f9>F54mhHP)x#53hHy~UgreA
zM5UK`s>?puC2p82UC%<<vWe)t8B|kKp#z%Am!%|9{-(&`VcXvR2o^pOl2?8Heut+9
z7t2@s;A^v3cFM^UWWvSBaT*v^fw=_;3F(q(r{pFnzq*8Sf0jqUMB<KwwT;tDp?t$i
zh!Bdn^d;C!cogXyIF3-BHaVX*5;7e-@FhWBQ>o4Gk({}Hn$RrVrMda6?n>oR3^A$<
zWRcbj4Wz^+toijSMltV%Fv9mQg|iPj{>u}Z0otAVc`puom;CUK37*G8ZO*oSCAW0k
z0@+8nvm`0--jznaqBl0mwd`c&=9z1tLTUOf&1@ZGAGcNyN2HKYMSW{sRw0a2u=jY-
zAa9*YdOx}<g{mxeRP~&h$cuxD`0w+W=3uN?E324bqPUo)z!)4tB~xzz@iP-MFvK=b
zVY9OLLWSwUG3lI8W`lB%D2M1vH0Y9~vSfLvXu(31xEjVu6k5p0C~Zpw_~|fOoCdEd
zlH(6{R0Rvt$(Ig$f)GJTLfl9^=7AgnA-$D!15WNIe9I|edESTOW~N@oc;hchQ!~jD
z6VeS2+j(9<2U$6p;-VG({S5OsMlznKmr+GvVlcCAHZ?~Jtl6^v|2?g+Eq$s__Cmik
z1f!pOVSN|cfBCBm=#5C<5qrBPW*CtgPf@Sv(a3K~yz432Ix}prqn1kkxcX+)XimnL
z7)DvcxHI^u_i9nmUWL`OXC;M{KDw&Tuq;Z6MFvXy)pwuempRc2Tt0=C`uWs>cEsEv
zD)B8PqHkZyZQ(@T!-KJ;1fH5r6nT>FXUB;t&H{H+5)yi`np!{=&vW=P_hn4m39D@d
ziiJa%_)?s+#3ii+DobaPk+pO=q05#Hj#YI8$=Wb>%L)~>0uX-;<dCktHJyPCekQsS
zT2XAbwy-C`@fz1jD~xO?k&@!%?QLuzNtfkO3h|L3v1TEn575HaUD&W43{wyZhwO8|
zXQ2PrZ;tu5$2WT08*xFZ@w>aqKFV43!Io=tI`4;*i!}whel#;@FIgfpHQ6tEzhX0(
zY<D?&a7<}<^~QY}TK?ID<<KVOvHp(G&7v*rAooKG1xeIS43({NKeL_+Bw49DT%|W!
zM$Xz#Ot+*og@Pu0En7t)Vy^F$djql6?PU7N=PnHPHzpRM_*4zAVcc$vQ86cykwu*1
zHDskZ!qK*(mqm-OFC{f%CjniDk3Po<p`xCor1X^M$SY0fZ{-~=2FevH1i?|GB3;Ja
z2p2$x`!Q)wbM#<)s$!`^*kgi)@)#F494feqBRF1R`2AgD>fj4yU7(D;rdElIhkGkv
zm#{D}vI**&D&ydI3hS7r^jLOZsevBx|I_Y)|72+2V@-$h&4@Mig;DEYcy5~l0}{T7
z7uMU%z+7WVm2&c8^h5K#x7h4J$EJ}^O$yli38sD!6NUcBSw4H}S=cUFAkR$QooFdj
z#93Yzi9mDcw!}*KJ_|218g;6U3TfJ_O~<nSUS<|PiW~@HKvPXS(>6tzQ6XYM(NX(O
zxBMO>mk@R=WO-m%MGgVWY~TqueZ$1diOG4GS`zf&NSH(?h4UUptQwSwm+ijlp_NJu
zK6&lR)H^YRsYD#Ea7xyTstEG&zC)q8gQw*YaR;y`P#7i&7aA^kBd2Zoz&agQrkmg6
zyyRN?tQCZ6@l;{V09IEF9<I_;Pbr*ZFKyN}I|oJ%ZhXZ&!R6~$XFc-&rlDX+nhD>P
zr3}#A6?nStM|&oXEQ*oLZV#5hi+X3kd)4%qkbU5MP^Q(udnd7Y%P}GoI5NrJ?cx<V
z+o9`SC0$q}G0JPQqs*cKd(%egHrE`s)JYQqfxxY&?)64SFmj1e>8w1I;lWc~URFs=
z(C&s@ZgD~fRA2${2Wd7vk#Qa{mv6zKbVG$12gYEY*61Xw-Khn@Gg8Hx{<EBkgN*|6
zwM0{gPl5|uhuD{xMIIkvrJ@xJ)3irx5L!veyr1~37h1;F|2PW&4H{%+Zu&K_p6{uv
zlWmZ!+a55na8ginHkjhjQ~c!Wia_q*^(ree3b9hv6E6>psL01o-E3M34ywhcpDQZ1
z_!$}x{I^3i8_IWr^`R%mZnbXzTZn197j<LPNaS=(4S6GFWqK&hH=yetQC9*~JsDon
zsv~res`!01@jITAVh5Y3g3{ksb7>I64-WWAc~UXzS(;3nXU`Nc7UeJ6Z>0}0SI_Ta
zw|Zfv`tCFD{S7WtQf3wvr@wq1JCKm9WDG`RYdA4w2qEF#70c!VenjMgl6S9kFDi<Q
z3BxWn^2E=URe_QHgaEFXQ+Xg6lhhVHwaLiZ8wEZOw5``)!Q)bTs*9dR+LQW{>NODX
zIw*phQyyU+l@ePr=gpR#l3`4(p%SN54$aEv%0j3ptA-E^#vP|-rAA#;dPHEvI>#o+
zh3HchNc%_B7Dy{yzKDiX7hoS<PaFwhk}eoLm6iIpLv^xTDKqxh%SNaQ_haII{Af8I
z{3=ztXxkE#PSWOkgS*-!u<lKy@*1vv#JjdP?6OpezQ4FFmMzz+%N@w_u1S;(cr#%n
zg>(sh5A}Jri)@?{ntbxDq(4)5O^);z$j>7F&Oy)@4tm;WMLk59*wNh0QS&sDL_Fp_
z`{;~<Jm{}=tW=VM!lqJMV?Kt`=)nGBw+GXmqtyTg9A^*HT;6Zfm6bz*kzbP9$O#bx
zX^ro6-XQkqqwMmnVUiJ{+Ip=sFcKqgZ;U5IrUT$5g{QjBB%0tc!tI~t38}-4bodIS
zcsfhdq{ucL7-hK?Aw0Gf9E=<+szI8H2=gYWUPMK*T1r+%R5rL%7Fxr<U5KTVQ!cmG
ze(H~RFIe5)&^r9{)-&k%RCt*?at>wL9rzE0>pRwWS#RJfQR}q&HV84SbB16J*CA8A
zQ1|Tu9?^#B8MFcW0)9_sV$@b0-@}d%*{;b(vl}+Uzf?D+ox-L#!b!e=_Us>PdWcNQ
zU@|6rfXs1KKB1dtVCkSu!f^g+jav93=5r*#XL{0g>P$(zWI-%~-Qi2R%=Cenm^KE>
z_Zo2?H5Ga-guJe0VTKav5biPQV&k^QW0hz)j=4t6;dWkw$G=ft2o0rPUh%Tlpc2xx
zPR+`Q=$?DL=D-1j4z}ees8!_28uno@Ys0v8%I#QPv1Fw~L7(-LtM<uy<HTxeZVvR9
zD(E}e2Z!PmbKq-s6)fyWcS~4$QeNhTjjzCiPOE77JA!L2_WHbnZssSuMR6`Kb3u~2
zpZh;F>A#D9M{AD0=KZZ9g8GU}ucDUN)6mX6<~&aMa1bxsEplKAtC^A_re&70sAquw
zdy;$fL##<AE<QXIA04XhRjD(riJ5a`J&ER6y?`8zy3M-}+sF<UOv{0U;_UZNDMWV6
z<FDy!fGv^Cfxob`&$$9|+AnU}ilCpYHS2uO-O;?afqUbe<-vb|2}h!(Gj_Y)I5+?F
zr4QK{U7BIDnCn1b6#M=mrn?{UZAxXWjIESWl8zA9>#&)X;T49_ARacqgO;|v8qs>M
z1GPDHKvl`g4JYnMx9=?;TQ7^Gv?kLc<y!fQRej~Q6~KI$L~wM7i&Y+}DyTXU7f5rX
zyu`V?Cr1x<KUgd?EQW^PTrHdYIY@TTIA3fBgG%~8_gT?1MVHZU-iLF}ByJ|nbC{*3
z=HLMd8dZmjMxu~VHe)yz;Nb*(QEx5bG^nOc!D0r2{AcpUN+r6{r?ja1s2@v~XmrY?
zj?*--*%9n&Am5lV{$}HZ$+DfaDO4tksrjxy=CV{UaNebAt_3fbunM(TD(TVbOehsJ
zhxk1X$!qZ=S#U^RzIe?dw!RuWQIWKKP~7e@`-z4lnXcdLW?LmG{{Qw}sZ{ZK-&onw
z4^Agg7kk?ke!9kCpoFd24;m4zq~zQWuX^VQ&K}oubZX6%A((A>KUb}1c?1q(P>?M$
z=eFV3w8vSoVFsLt(8LQJ)AVHJWQG#cyp2mQYPE~sK<#*Q@z<Fxo6$k@;w;wzY&RPH
z7UK7B9mQr(v4rE_yDu8qhv!>}&b#**dDzK@&TAsNmI(Mi)ymj(=eZt#4N>s5`vgc(
zs>Y|xKpVp(!-?70^hI$)FxZ^lAU}^6h63|-QorMtEjB&}i=#M|y!y^WgJ4wCd#KGU
z1ttOAS2q>k6Z&b&pQ~8i0dMeWKY4eLwe$$ue>K!F5+|Q9&_Ll?EzF2}h&R)5X!$R7
z{xuWZX~N1HDTKS&Vjl_VX{YBt-}#0Pm=0!9X~;TZJU9o1t6+5Ono#b84y(#%l1Rl=
zXJasNK-e&xpo8l2ZuTTXdLX|hii9M6R2SxhGZ@#Pqn{W5C(pZAF&Z>5Dyj<RZuW`-
z%Q~DLe}8kg`5oeST<w_zor7_ycf-Aq^DYKR^coTq$UdOg`;YN3dZzO-dc^H75v`OL
zc8(FJhpaD^GzURghn~rL>u)4vzW=;$WVMy({xb&A@R}a$%-}?cw}I3e9s=;8uBb7h
zceY|(pQ&b&%5fg)DE0JXztjhT22SYGK-7*NC1Nko;7q6&owT|14WEv#wHTGMP?!5N
z3_tM4O2R&*8RX7IWJ$VRT`{o<;zWwSzMb}cjM0SjK*7oZ#X9a_UUc2>@R>M0`3AEn
z3-OlfS2Zyv*;eLsZQYdT$h-%KW@(--ngJzZB1T4M#f*~VZ2mcZzw3zl{Eb$m1Y?r0
ziEHbj!(ZUA5Ro{XV-RPD$M)x-=llXC+tG8r7##2Z>9yfU(6_yTL%^Xz{pX?52^`DQ
z7_)jzI6Ngqqxu@r9J0^VUt54av22#H{|1ZH-aWUL=bACgt}b^Yp>yocTyk@KqFe5m
zUDa<FNG+8-kZTzyVUAIF)8ZBHTCy$-c_3B$@Nke#(8<9Os%qDSF}1NK!LZCCCs%Ab
zZh6O$Vys|wVMC1%UH$jA5r2<3{rq-NTN4G%UK{sGdfyM)QQ~;&rW+A>j+3So<|in#
ziTnMAG9<?}T11fmkR<&MG7kdMl-8UzqomkXbDlBDa-Xt^-;;N;BA};OXO<Qx5KJvr
zA+2pOa)J&IDXCh~57JYtPB(J!heK}8Ll;-0L)*0N)&2f{U(I@szV+528I<&Yz+L2n
z$MSd_BpDgkDE6Wk`;^Wf0Y);b;Zx{)%kHe%pIt6~1}AwB?l==xD?e3Y9$65(2h2*A
z0&Cxo2v3D662MeV1g7fs_BjrD+?DUeEA4pfDrzaP^a&#48EW`MKM^lx)AJ59NjxJ7
zwAQ5k&R1_`#1P$dAt4pS6d_oLhs7$Y2?_z8WDVnMx_E~JLu!xh<)NG^${l`%1!R$r
zb=W~GZ}j?b;ERer5?fa@pn+MIn@-WwKdOx<ea&Ij%BqG$V?a|KZta3j5>fJpCq07<
zJJRo{65T>NkootN!Nu>~jf=BB(Tl%V?uXaD7OSn47R$5l1s$gS;7g>BKLW?4;y;f|
zv5e*4vn`32uuu=lP4V}73vV-CCrkBDwXgu0xjx#RBR@KEzgpcUabwABu4hBrzoe&W
z_Z;RC+kf$CCA_fP*@Kxnoa3JhyAqRAMitdbW;X^uCo!aI_e7GY#Hhw}ma~|Qh1e46
zC<oeIVi^@y8Jq?oYnu1CT)8Yv)Xy^BoPDlCf&>!~rjf*s;|gK`4exh)t59mpOORq|
z?)O}(b1d%$Q;E9eDZ7|rlaSCeGVR?ZS!H09et3VH8Zq!iL(a;rfkb;iBN3(sZw8M@
zB}vg<4b4+{O&5PFZZ6^RT3JQhIkg&NaTSy>Ssf=5jjb$&mNK$zO2zgS$MRthTq^L0
zNQ(h?-R|PoTMoa!ZM5rJg-g>0T8F2T7R!C$25hAq<?eyp(%-hzx;hMCe&+no`MDB5
z`vw%1%_z{M0{<b!ra(jV)12Sl42zWV6v>@2<g0?4>XNVc%%Xor(mZuuozl;_UIIQu
z5QHt_m0#y~uInP(bkCWv#=su7DSU~WU?L4csOn(H6ZA`bNE{>~u+e|?c)?w^iEcu4
zw3~9G`fKP%wvKG8SKk~Ys^WD&&G^4{cCDyMtcG|;>aVmnK`Yqyo~VDTuyMzHgbVV3
zX70tG@rKXl+!I6m5|B#XMV-=g#KaRSw&Ag}MlgwUqUCJW4}4BgcFQy!VY)lLg;3GT
z6lNQsCNIgDCRy9T777FRH``?kbYMim-2Og}mr&mZtrQnu=R^a^`a@Z6{Ll3|YT3o_
zpp@&$aX_D?HF&V5!#Ow@r(<j!5lyImaDkxi2+&WWTntIvw+O06C$n|DJ(sbbiYtDg
z_t*Q+-d~%BF*J8==WcmVtM#|R!>OP3_hzMhq5F@bp1t<9e?Ftc!w?@{LD;?BK-t8N
z(hB=LruoJ$cW|JE1@lWo59awJF^}JO&I)G;lX2<Ap7qgG{LzdH#;V`GaHIVcL_00+
zq%x$^3n*+4$?fvtN{^j14k>=WxO2BDFHeTEfPKAR)82l#JCt~efRKGWYVxA(hO74M
z!)tm?qD3dkY=}_y7{v@l|EA1}aFt}k-RRzj`;zbO1)jS{E`Fa-(Tr6v4#3mZD^2BE
za47psO=_K*OP>5^M`6vjmx+bV7{Ug63TA$8z=$nR1L!gIh{#UyQjHPDx2jF91`Rc=
zLMK(j^~}g27kWXOo<k1svgNl&(SCnVhzl}g;g_q?eiwBr5Oyhw{X*$LWNo2r--pEz
zw%=yr1)-;g5H=}_XBjcaYwNb)68$Fx@$~KiL-=w^5(qm-{~)|KFrWbgW$&zD2J4vL
zHGQ4%Thi;2i2G5`x@P2QQqm}r+uC?~*_67j=d6Nco>{)IXRA!E@;!M2X{Pa^5%(wW
z3O`9;EpuRxpR`@~dK$niw7wLqdRHwTHzhas@<{SkRSNq+U|NHhE-@{F@YshBJ_@;h
zGHQ~a)b8}~$g3~J?nlI+2i*BG-x1612YR6JreRLWX!hNy02fJuys7BwR?$woQG3(&
z5Y2b)Xr5kasTsr|*gB<v<Q31lHscxj&vIsdJ-&4!?lTm#$(#uyq?|<MV$WZ^!~Zun
z=!><pUlgaPivdKeyHUCCw%KE@=jY&sT;{xn@(y%OpxD%)%a@I)3pFU^sZ^RUw-i^G
z@Zsg<am64WWV9e_`EY_cU1nH0wz3f_OL-|iJ@8q>L0~!fA7PMHcaw^*?R8rN6jCub
z{iWl~=@3z5jm=&X#$X!WQ`F<s#`_=*2kd^);#>~n@LamGBBt*}xB2-)d{g2~`1|Th
zI2b%W0&1AWAOZF5k+7iCwRd0dvk9^%g$8>jiRtgqd)bw`cN1v9Rc>Ka1Aij!G}xbW
zTXGxn!VhJ|OXZzEYYIg*g$o)>+%JusssN4PHM_C(27;1S3UO!l+%>Xw<L(^($Jn;H
z2V`rnm_VOG4Wvpcv5GKFE_#ZM6)PQpr|I_n^<#p8DrKcpL!tswwE!?w0tHS_+XF_@
zWc8ziswyHa3~EC_8z}iaMFT^ZGTY@~V@e~dRN!x_(N6(LjupQxr%RUgMm!EFPJY<u
zRns-H3MO7|;;q*!A9cYy%#6w#*7)baUz-L!)C43n^{4c)Aa)m?kWtu{{?Xwq!12ty
z3x*00b15AAp4K<?&H`okc<iTp)+4w}R(lRk4&~P`HnbR6_(kub`009<6nmZ|2fvWX
zl_DapC-mTWa7S2lG^*Z9`MamP@hON`1lxTisFq1hfch-PC+BBaKFV~-f8-v}L@xV`
zJpxLR;${IF3~{Fc&3I*dPoASjy@NHYyB#5CDn@C=0+R9&cH44RMt<Q(Ipi>~5%s;P
zb#d99OVFzfWOhaHq?2y|=h&sOY0B$w+!^HS0P=>Xb2CK<vBfYts6)w7vRvH#!s7AW
z23iM?M>-it@-qsNy(d(?Y|2cWob*mkoN<-SD*ycn(X0vK14U`wcN<CFange){y9gr
zJ8aZKD(YIjGW*y3JrK{RyW;Z5!?7d*>K}>_HV_9yuv}j}D1`F_HRyr-Lt99jXcwpt
zhS)t|YtX~g_yD)vxi^Q4+xtJmNAIk(MaB$B9SsD|vza?l;jNukBLYAf9H*DnjyP>%
z5IWX9SbObQh*E?Os><uf;W6~yQoI(j+@M>P(L=*l6wd*TnotB|mTk(=Lx4;xsD_9f
zHRz@m=0~j{c@Qp5B?@i)gQL1<8Du|;s>HM)?2m_&7{eheEpg|Bigd~ALIh8`(Ca^y
zu@b)CI*fUg0N1c)Em<ukx=$XL3lm*N{H6M5M7t6n`3b-2ILh+N^Jl<Tt)OGY2XH4t
zmTbl>dlj|Vz7~^>y05{)w{D#mKeT#IzBHzHKA&XcSKv@6s)U}eBfe))Y^(x2(eB#&
z`|&5vLao=?g3Ej1v)L_n1R8laEo}E+z6lU0&IlKLu$u^c!@=1EyU&T^QDE~@_(6vs
zT9&)HoQ@G+$IY_1d5}rXuusa1%$S{b=wFq@K^bT2xcUt#$dXmiabq8Uv6AkAG#!LR
z5wBH2go?CN@{_={M7Cz9k50_hgWZQQ4Nn5ujz@K3T1J`jm?VQkL*`M$9RX(`=D*lb
zs`T481y(SiaWRp6*t_c4Jt5|qX~F{;QeffcKh!9K_pk#H$WeUSyc9~0aXIbxr_L~E
zt0<sOF68f<KdB8o@`f7NHR$I3Ex8*WoJ-7mQrL3#`f&{|;dp|NWov#Jw#c#yVFO3X
z<IzQ;77c}SNU+tzSsg3!*<y2`(2@(%O(4;n3h<VU;-h|LT%-0Rzahk*{{6A2B3LmB
zdAE~z5F1Gk7RDsrCTtE+`_djv;bz^FB;uM`*6}Cq2t0h8v&_gLt;4h^%lAdOC2zUt
zopz2OY$Q{=K3yIs?zjQy0<1KZ`h*Bp-m3)haSqEn6`OW5Vf9^#m(U*?zgEDcj>FZ~
z*RAB4XLL|aS&pdv6!He0WUYZ9?N5zdx|U{~{AT!7v-c{Asg$Xpf1mdWoiO$;maol!
ze_G3Bi;xJV1!W_rWe^yF2UYnlk9pJx@;Bt5oP!3K?dvW$gP#U!^)(+oHAX`+?^PCj
zV!D;FOwayY9X_3io0trGoc}QZ$6(JC7Gh4}=_%wP%Zv@AS>4L7Tr%EB*j*n<J>77}
z`EJ==B(f>~7k7BRi3l}9*X1+$$gtrl)5ujmOkY!|K$c$~rg^_%f+Q74*;d-{*z$^a
zdQu7#kyYNd73{J5FVY#2K&oU!2D;S<_+dvgBPHRwFp-+A7}F<v=(3g5b3b!M=nHi@
zDXu4=Nh<)rBcB0COQyiM%*QLUE#qKem!K7OGw-^ae4X85Pw_iRFuEoZ@_-lrzf4uG
zM1NeTmYP&^?lK8;ldmVM0dg;g1P<jKF+gHHSw_B@CS*1GX|#SdBh2%d|H_n)Qu{A<
zDuOUwAZ+H6X54xF4$tC;`@TJB<i7NC`p+c4F%*C88q_88ERvQHr+@*Zp6Y&)vH6?-
zo3K?5%%6_D;UKc%qP2yO$6nmJaYH2U<g~M<J-0q!700w=FaBME-`5)PK@CU_r|0V^
zTe9_6rOg7QiWoUSf<wB<1|k<h##0^}EQ;J#0M-HaE37o)#Ypk+KJLai3f6%U;Pm}n
zDUqtLB}%4$GLm+%wmkHTW~~AL=t<g@UEr%%?95!uWX0$EOMm(%rkdP^mH27nYghtc
zvx9eD`hY3x!#_+}BG`XL#CWKpfKjvi7|DUgfX*m(L=Uv*iZ0e=4M8B39cE)&2_s0W
zaA0Lp`bJDahYWWhu=<(V@@B(b<yOrXavM(HqO~00S^H+5!xXWzg0Qu#h&^Usotftr
z^X|uQ)!NhH<$v;>^ZyBY^MZDv2in}a=ey8nFXiip0!2YAK|(v0Z|CWfCVKRuqLPnL
zL*%<Msxl6g{_x9`d?d0BsBr9<MTMV0*nvqWkLmW4<8pvk<~;<h`e5AAVdXn2J5P%E
zXhnVprIaC|0Rw77DDP`*-hmuqu;|h0QhvRi^&YN}AG3p+N=&knw*r(NeX%PC2+XaA
ztfw^IM}qi_vAVC}QKfhPThR{rgb$*RZpZN;kYYC>`lWx44i|)maM@8r0(tTG16c5y
z2E;Qq8bpMxUwl#iV7*m9UY8?Er6ezL`_+9Uuos}8fzT<PyG&IUsC6<+$e?u*pWph-
zfFZo7!Qy?;!~eG&MT3ZU;AN#+Aw3oW1{M&me7-agdv9vLgNB}Co;a5_wrY;7th|?z
zjR|KfR&YQqR$e~<&9hNArSL^LItc*Bbh2NV3kt(wNhMHz^~*nX0+d>|OGQ;=$qKo$
zJQ>d;(y|(v2F?{RfVN@?hS2FC(M*9wppr2Tjfst+a?dOIb@F6o_;BjKE@O9W5GLDP
z33yH00<;dxwfFknyNT#1Qip%Ur{!G_u3X0%=IG7(L!yB5y4z=gh_=-a58yq|y0P$6
zheXLJe=<Md!hTCwBmc=Mb5p!tHX-_%zVt@!$z?h$CMGznmC$W<sM^a)=w!FBQGyQ3
z#!>R6T+awLFXpcv|Idy)lJ6O!rOvIAXZdKQpJ6W_cBu>{$UoI36C$xr$skIsDx<Qk
z=wp&(xhtLnS0ah?oH(q5(%AL+uRNseyEK9hd?9kp4Of1J0nHkE5OWq{$dxq*4Wiaa
zX=Ea!++dNEq|3QgQjEe>otWrlc;H|LDkf802{bBK`IeDK8?At#)A>rc<hzb1(pu9s
zW)yDqbBx$HV@S*~{>oZlS^7UaZ-Vz-`wZ%}s`orj+OTxTNgm(BK27aH7heh%-4Q8w
z`=k4Q?}J^ZH@8AxB-3pqfJ<)j8OujN-<qu5>BuC}5QyrRyN72XMR3(lXkFXN?XxqF
z1V&pTTGE*G6UQ<RWfnlCwWuQCct=9LZ&O}SM|ZaYh0I9*i|9+g$7%87CA2{2P|7H)
z?lc?Cb_t*v1*0a|h$a(Y$#{8tO95KS%?;Pl;qyEaO3BTNMLX<x%(u_z(BNm7fb0%h
z`x*Zc$19cpSAtfIk|~_5yA-+nYzF^$G8A$Q`8n43FuXkD?z?p!A|G~8V`~4Y##~(6
zY)L*N0#hvW>BDf~dU>8*@1OZ}-aQ_VU7=s5QyQ^X$dz5hIRLWlxb+gu1Tmt6=IWgn
z30xN5(_ba;Dye8N>Q0oV=8%LVPgu#yA+Hai!y0nt_B@`AE84o*NgRY;I_8<l4J)*V
z;wJB$L~w8$0Q4Q7mKMa8e%s(R>XE7T`45J?t`wuh5E|BgnY_{fPmkL;($3*5CTj5q
zW77~wG|riJib=v()(Ap}ckf#kqBIjM9^9XgI#=K}A$2sSDbC2}^zIA%Z=k@Rzx5}%
z_J@tQ>_H-Zv|0D~mrqV7vg>@fhNGV8@SZnqV&89y8hiH;S+7SK)M6)-WO34cHg8hr
zv|&su^X$%7)#8BxHMsy!ATZR`PRmFd2aT3J2NORxL5mUiy^j=~tSIO+)|P;%Ag=EU
ziR7_kj~E|ElL|778Baqy6ek276O?CIvG}4!`(0}u29pIQwc7DvWgD5Dr&IJeGa1=>
z{B=nXqTr(D&U6d43fUwG?GVZ%@;H}h+A|@n06Pb~Ht)DP-`3{R_wAfBU_$@<pBkU&
znWgjO^$jWw@a-}XT+w%#7hZ$(ss}(iu7CS00n6Fbe`iJnWEPbW>hD;@<KCsLvpLsy
z@tsf5qfBNJ5yvw!iXBi>SJ2Uy7(8k?RN<RQc~Lc-nZmNRQC1`-d+(LvG*sVSeE@Ey
z16G>Aw|bZ(@z~Zj9B_?5L{b_aTbUgzi!?X6f5V?r=3*+0zRLj$yJMVVZ5{_>Ki#-L
zC8lL|udqMIB;leW-#XO$9i!n<-C%gzLA&~?W_-$5I);G&uJAv(wcFZ-I`YC}|MLyN
z+wWzJr>ROu{k&XcKGIEw*L2y9670yXA>aP(h|$7P_VEFt3?Xg(l=!0+G%3CjuozH0
zrxpoOyrc&Adn5+ZQHxs5&rm-?x45}X8^W}<ZU=UHSs&)Jp0mATXB6ipHu9$i2)xD4
zDDR+R6OTn+avJItlpC~BPBINg!hOEeF){vk2n$)V>uk|J;_cpjgnQ3%I!A7HL1PZA
zaom6SPf%aT4Rj~rcJpMCSrAw<3M1Hy4E5_mSM5<__}El_yTxf%+Gj<r({5+}11_q4
zj)=??SCvx-l)K*52J4{f6XK4LBcf!xN=9yK0JQ;*Sh>c}A|;DyMvx{FjzAQEjsO(I
z^v8Tdpi;a_vAS(ocvvhZ8YfaR^W!uSZFux+K(V24jR6X5kYU+ywl?dp!GTB0o=R!Z
z7tgP(U{kw$Xsxda1RA!(GJk;ZYk|n7(j0K#(SzT_|4+Xu=mrX22U{CoAQLZ(&ciTm
z25Jo;Wm5#6>)|;e<@UelhU&P|o3kH!K(0YL8ylbLd0#=hB;sgICn9oD@o5098jz))
z9&=p$ovl51=1&cj5{&%Z)Z)~K04z~(^Wi!=7D@+QMKC;yKurxt+tfIwWCd-Fa*c^Y
zl*a!uqc~z5EnSL7p%|J~@S=Qxb6BaXCx|TWS|}Zi$^(?Bjraf|Phlc*hS;$IqB2|7
zuA4pHs>S#8!OFR%@85G8Pu++0e8rm~uy=I5CBh&5r-J9{y+zr~kO^MI(V(iWE&E$c
zqF*fG5H>}B?087mj_dK`?U2IvwBIn<<t40sj9hxi?i1`En@`ny3AZINDEVDT+uE}&
ze_pi)^s(2U)z;d>9Rz3Gu@D$#ky3~GL0?9{XlFVCou8sORM%d9z`$M&5EIY=`6dz#
zqQ#80(i0tGwZXU)y_Cr8GRIza6r;2#zG5<t4$lZ%IK-&L#m9=zGtM(LEd#kUV2Qr)
zDe)p*%(TOY7<ZCK!C+$)FfIuco~k$?*%t4g(9#P~7_-L+fgOv5ONJJ-;^a?Z5i<qS
zobn;&?0guez0KcW-2Ox-<6ot#F?)s~j^uMAbluNO1M93-)%<9uld+1Q4}4MWGSC`2
zNCpI7Jf%gf2)B}zCMJ&8gid;HTzA_85SGiBWjR1uiy^)eTtK1%{___~Pss`&-qBcF
zvn#G-B_y=FkZWTv>s|esPt*MDoW0wcp7GJQEWO9XDk@(_<f5Pe5HN}~iUGN*Dvz;A
z#z{N|CuvAGG?gS#dt-GEVd@;hj(Bcw!9Epduq`9^Tp-($<S#yjT}U!1n=Iqz*3I%3
z;SIx^ra8)+!iLfPuQyg|zkJn<-b*ZC{g=lAl4x)Nv)bKW*J+PI+QAhXQMCisk4odA
zB749SvDgCH@u(R<W}Kiz_Vd*QtY!=F4ckKmlNe-UHa8j9HeRi~b$I1>xT_}>dWM}0
z4sUu^qHc?+yp_n&v2cn(N^Dy<EvWk?m0}h~9WXBF;ML7^E2Tj7w#3cNPE(+({;boi
zYe>^B{~$rSop<^SL4z#!v`b<`f&~W)_p4ot)LHhgT?4#X!y1$FF29}SzjSN##G&gE
zOL=I2ySs*m!#rTPC$amq<Hn0{ksojI5)V&cF)(WM+RuD)^<&$9%f>|ujBk(foNimB
zD)GCs0c!9ex6SGA@AWlTMnpgQ-{8Ns{(K~9bC&oicG{~p@e<cPRx;~@FCj^$@BH7I
z@7yK9U*ZfqLhR<+Llo!UAB{A(2O}o0=IAM}H%l&-ZqAqeE?brR4(j>iH`>q158KY$
zm<vL82K>IZD)-0a6$lm!Dj6OXhC<n~uWJSM9TUiO8II*Le<xzy`JX>Wy4P5rs6XEC
ziJ)Y-dgBK)pozFOe?tCTtxv4H6aGYUb5zZbO5hG5+EsUt#iz;a`%%5U`hhNY<@wcq
zY;~?k?~f}HB~%k{{bfg>&HV2t^#Rv$hLe6jA|7qqto{tN(d!95?`@}<ME%BpYx8C1
zN&Wv}@6E%Syte<*p5r-Es)$Nu6rx3OW+?L*t4FB?ML|IYi9nS}z{n7$B()+cQyfrG
zhKQ)h7!Vl)M1>@xLXasyAdy6wAp}SQ2}$mLvEScUjpzP%?{lB~v`_y+-uK;W`mD7+
zYwt|}5T_#en6P}qGlHiMs`!LX72&w*Y%oIIT>s?iD*Y2{Q9IprP$kFLpo%;+99#)k
z$%S{QFNxfw%Wr}cH=mmFsH`53Fej5F?^$eH{CYJ5v@noe!;ST~k=g2UuA^{C!>A~)
zG`0F2m%R4)9Borqn+T~<`q7!h`y}yq#k&0W=6gGdo;oyG%<;OJWg%CEPEx-`Qehrd
zh+X9JlDU{&t)vlaqwzjG`6*G2a<2kKiC22VVaB*pC`yG7IMU>I3wTm;>lN9K%HVo<
zcZs`pVRy+@)A2;#gMLXWQ;brd^z*d(aqmXkK<}2?V`bhz>e`{H<4-S~*jXFn-(Op2
z8(ce{QvI=n=*^p<7;h>{+`Pk;{O6fB-n@d_$c`DJcPl%?r+bmrl)Pey@{%z`ov0xt
zgk+18BdJs>#evB$h-6wTPl_<B#`zNM(#E*KSy`d@jK=a>${QOKyNIz+onJqbIal=F
za4&r}(d(raTu^B-nt=zio`p-*@7c3#cbOo$dLjYVq+~9!nmUOIEIJ+@Q*^RslPjTG
z*R1G-j<&lQg$u)M#jq9-W&De~$|5B9j@L&VD>^~4>SmQ!x^vE~pJLToRt7WH3PWlP
z%$O*rbJeK5MJKED`rJ(872o*0cuMq^o^tS>%BrH)S#z(t-`SOD-CdN-a4i{=a@~o+
zp#!I?sBbfdDeFQp8uWo!^v;tP92P|)L{n;cgQ^b&rfm>K-@%FULG<w@zyItQ+40~C
z=Y_pMc4DolfLEr4*X7qJ|CrB;{AA@nx{sjGr{@$JIYtil+%OvI;HDTXGThiKeOq{g
zn9I6W-ip^W#@gxzdxo9rFPpn_yutfOfBm^5MSkF#OxInB#Tnddve+lOI--F~NR}kS
zyG2AP-&m9!E-Xsq3yW~!<8(@W##HhyZRMs*$`Z3mn^M!M(b_sMPCbYLksqSsr(pL6
zlVHzi`?vL+%K4VbM0OuTm8yhdu36z$G7=F=XAJKm8lN4g+<=MZgs|;N*&3n24I5Qi
z6<f~^Ouv~QyNkG1g&BF`oj(0EQd(9|A}tF}KP=puGqD@C<spXik57`GwCBI@imzNl
zZTOifBy%`ga0}fA(@BOAKeL0A$<5LL7x#4>k7Ov7om+T$yfW;UogoPI9ccrq<F1v|
zq2zII#<C2b@$Qx~X?A#-H~;nVDlhc8o}Nh8xESvs4#_+yNvzYeHp5!yYB&)0W1uT_
zrXtY0x32;@oN(Jnmcrn~Z%*Whtkaq@H&>A-^yvfP_WgC<QT^>_nC;fY689h7n9)8S
z(T%3O#uIQHf*E=0t$UcuVr=U%9vPeUW3G&QgU3cS{IsgsE4{7g$e!MdHiv$dP?miX
z?^(*&rN))Vu8>!-Y=2%A!+J8`<1bV9m5ubL&tws(qrMV{#rHS;jf6vBQ3?CXA|_9j
zMUyAXE-bL8yj~W3bntGhy^Um{wb<Fo<31nzZkL6TXQtBpj+c9(i#$pCuHh$8EJ`(s
zap$<~`Oxu7ub}=)?-TuH0cZLv&Ia{2TsZdj)YDCv@(+S3qnCuVohgBL`c>&wEDiI4
z`klcZ-QY636F<5c)LE9C1VV+qsH>i<4P+P805~@vSnbwYvt2I>^IP3Q^*bPBO23`s
zudz6Vw}xKxi?W39tQM`!sit<moY1_6$2mLsTFRe<ZnG*-US(`zD;u~%zACKyc|STb
zEH-+q87Y0F*5r0bb|H4S-<UmyK&+YEp-QdKaHXJ6nHAOV=q{?Zb=^g>b=7xY8|Y?$
z<7>H63&MB!jJ`gDqz^c7rYZ`37HTWX!dk|T_wiYUW&>K|l|k2T3qz8~fv!!9k5Em;
zZ_>zruG_)6^ASZ2nL55URCB6U7ZWaikd7XnG^*mwtU?*!#(@ruLmG@Q(H;HOV}gGp
z!^M+Ex#Kgm;JOC!36&C2OPA3?k^y-QN2-tWdik~(r|NQX2*pF=Hj%y|r}oc+Qj582
zdXqam+%YqjvW+*ef|jz4wHUBSC96?_P)toNE<%lA1;R~8NZ}W9ubQG*s)4Tk@A#}Y
z-i^Sh5d*KQiQ^=09np5`c$v+ekmE21Iw6taR-CkN$5aY&YG5^h>XgkOdbQH3sHRm0
zNeEWuR!seTWqO4l(xd+TFN;sdX9!cr*ORJ)e(_*n?E9Yv?Az&4^Ha%*NCywIVK*a*
zD3DEEF-%@}s_HIse6f1$@dB@K!JP6|>nxexn!Pr}R{A88*K>nchtknBusDo(Jvg#C
zi~OI{(*dzc`GlY+zkTJ^2a|m@BFPG#FxdQ<BJWYTQ*sDCW`8DfXq4Bn1H0bAZRfRv
z?%h<hkZ?_Dd@A7>${^v`8#QX*Z4{-p>y5XhZ(Z5rH#5f^e41y7>`G$VK5;xBNfm`7
z-y{$@lt(CXaiA-O9n(#%JBFf82sZQHY)-7*nNp~_^Qy+|F!juA@Kt=ji8-e7{p0km
zaLHl`V<3tf4?M$;Zg4^!UMq0p$x0+=7O4m6<jpHWQXGa@>hQbP<|Vbf_<8bTFOMLT
z7Do;r@Zq$f>VL|PaWrzY5R?WR#M0hyE>lz1*EYt<7)%Y;aUsh{_`)87YwqUt`n`+4
zI#!E?<VWa2Uiia?)|~C1Ili;<pT=`0@rndX=|Nt;@u3s~UY49L8UR=pN>VY#BKJLG
zp<{|z)!VyE5Sl0hcl9@nkZZT!s8a8(Ls9PCMv?CU)$s2TeHIGN3^c~=dy_`V8mE)l
z0JN9`Z`kH2{cANSgXE|}x2B0RD2f5Mu9#6Iux9%vs!}$G4pWp%GE`*9fr)uuofi9^
zbqH?XIKAoht_&ak9Wndy8B~)~aJsfy%fdbn3sl77`=JsB>pa?HF(Z&OGfqfGFk^#a
zhZTIuNEr*sYe&`poGn(q5-e+oUa7LbPI!B7V7g(f6dR6P(;Ys=TAkyYd}Pl8Pl(Dq
zzri+e&zQN+Cjj`)uX3ZYV<a|a_>Wa<HTml|rwV*CuZtr09vdtD3-|B$#y1%bvc)>v
zbVRV~2=kwM=*T@j7B|2z9r^5+mZQ_#CXVd!M^F_*PR^a5;V7BrH!lD5l|RmEb$Y#w
z5=xzT=Vi|;%nhna4LETWu<a2&{cfI8E7NJ$z1->b7ZUF-SL!nyluTxk_ez%0nMrGZ
z6N}y|o=MJmdEgBezu0qWTX@f3t8Y7Z>1RiEI~j@n!d_>GP9H2?a3=L|9}BrBmr}mf
zq=C>$g{}O28L3_`&FM-L4i*!%tvM#E!6JIRpVZau{albU>4w*7cMMnO#<lNGR*8RU
zyyf94t-n5auXwaEE;lVqRe6w$U1s^)R;4BjuV$=t<3d@+=gT?>)5}D0pVuYr)rMY^
zq@oGa<(Fk_{%qyoS0M4(Ibt+{cR>+yUi5EhW@@MSjwUU-6ka?YyK}+v>qjQI%cPQR
z{3w^PDWIWd8sl^E0!4;78jtKkejDgoxS$eq)9T#+SlO&GPBXCX+`h3{v(II{XXjX*
zd)f2#AW)wJhI#f^ZL`0gFmdm=qL}Qv<_WQL_@Mv(e7O$|*2T#3B40H$IOEM9O<~40
zRQ~5%NFZ6A|GP$Vqkw-~RJSdh9vN}9b2upeGYgvnZ)uCvyJ+QrZcVKv{ywYwl_*+E
zmg*8;Sd~w^>x|-ta2>x~ALIT?70qs0Lxb-@$b+WRtO8@P`CQ@&=POEot)#`wuMv2e
z;)nh?B5<6BFLZgy%+c?Oc(eBMLTl-)ax=H;?&l9aX^e|^FoQeWWa8~f=nrL)3&pbA
zi=JQD_>!HY->Z*zRWx(s%y&gwEZCa4f96IYXmTJYE|Usz>y&x0NK?-SudOyNdp>uQ
zy=T)82iU~y;oMbNm8$8Jpoo&$E!3}k{F$bdYP|O3^KO1WJRpWc;6}tJpMKJ*e$b@h
zF<4-`WT$dI0loDz!!fRF>y=_Z6Xt9>*n054!KI=1`}WV=FkbQQZ~trc46i+p|9<ql
z?{=5}t)@>=WWp#teCN^s0$_{b#*QyN#Q!sZHFkXU>`Pxe`B_Y?RQY+=m#%l`3y5-b
z_|g;q{_K(sT0jK<e_qL}=Dl9l3z=GluEh!eUZL7lbfUc0RNveNY+WZ8pCo+d6x!DT
zDyt8^-a$y}IAp!+kln(Ef!dJ=iM94Cip`uqN3wsII{K|#+iK0N`|zEYQ^VXNgK&7p
zp2F1sb(+d$XVcpv<KsHtpAIq2K9gf~{#E^FpHf=z+QZqbF5rXay**M8=N?jDF~2hE
zGq0=MXzS>gw6;Mq8M1}?Z`-VxsT$<yqqGRp(Y<>)7dUs<I9E+>8%FN=%!}ja482ga
z)HZdU{;1)s0nX)yfV$uJ`jlRembJgWy(7_m_>1Cc!f?<RE<SPL*S7N#K3%P948f=j
zN;Gb@&|N6gG#GD-j0yhC&x3wlrJ_j-yRNy$G25m_93(!8pk<$bw@+<d)vLVEC2RKD
zgD@B@h3CdSrF_4UUltm3;L7L!4$=mKcBiz;fYs)v3|~B-8PVroZ-)t{-Mjvo`Zk(7
z&iB$V-GL@;>T6g2Ud4<KB+%ZqsqgyW{JF&ah8y|kmM3p~v725*Gd}*sdX-c}(Hd8*
zvYR*X->$T*&-}c;i0?G?`uO^GJ>{Rn*)hSq?u*5D@qvb)<r~IzO+6FO-*T)}tLNGs
z602671Iiia*Jv7WuRDB(D_YKtaVN^p@-<hOloS?@4F|k>^qH_4dPl!p?_Zyd|L>m#
z^%+nI*<GO$@(KFz&yIf;4S!rTeHgC@X^c>|yAgP_d%yR4$UURc*;WX%2fbejulZ>C
zvNX?Ve;}McSbM#l{a)>5S6_UdV)q88Uyr_aIi8Y!`hLJl?E)Bf=0h{teRk_-SO|Wn
zEz<G|<=6XG>*obkq*MLgvf9~J>lpMze5NNQ=kroKzTYtCh|i|$n#zkK_VDe&Wg(`X
z1%nO<kuw3uP~T&+e2wMp&uwh!=oi-KpU2l+ZQ>F3ZlBoXCqGb!1f+rrY~gDE7uL|$
zGszy<z`walDf!*YKmWZV!F{SBh*231@7OcOF)@i>W%)(ySUdW4M(SPaOuyf$zSSe@
z<L$D5u7at5>^Z@?;8WeEuKkcn$Qvh+8X^gGUwlSK&!n%z!t1_?=THuBr&7)55xo)a
z==bcx$Jcg<ZZCvM_JJIpqhi;1sOINY_~akzdL{vNNB3eq?|Gp#Iq?)@!_-H?x87K6
z{7k2S{>UlfO(@1#9|kaTw&mVB$gI30m2`T!;RnM*UqFqy<JQ2Wm6|k6oBFm)2nB$~
zCe>9<^vz8u-xIaeO_+DazkK_Iq1XM_;hCR@(;b+e$!p~2KLUmKgMNzQXRd7QICA6=
zC+<u4eSPMuhyG)kr1C|YZL?PpeewNxMWR2769*u)xVo!t=Vx-qE5>X!J>=&hNK*XE
zsY`w16(vuGEe3b1Z#MbN5}@3J2PUo2G+>?IkhE6wGuQq*&Hr!n`gc+Dzg)ooUrM+?
zoPB+_K+(~u$45DT_e{SrZmIfA)rtCWgDq3Px#<A-zx}y@C6ln4Nw@xaKOA%fP%(S|
z_w)_WzukOT3zEkD*MHT+n;ZYN^wUjUx4r)P-7(NsK}fk(L+{t6SLK(iRQc}7&!5WU
z-OYx-eELB6ne^{=-v&(=6nekKD}MTE!H&j%eOK_8st^C{cztOx&Zji=zB_#8Q!V(%
zO|L(fetL84wc$Vh*#1AuU#arxr$^o56~8<B_mY>i@wb-0Oa0q1{tmyt5#w)M{yUZY
zov@)o<?kBf|F)8Q7>2VI9JiT%z6?zI`;OO!`S(XKg38VhzC8ZS??^vJJXv<G_i0b5
z*It|X_m{{qpvud?_jTd-&~*|k9maa|*kp)QqR<-z74xs`?bh1sMEzbjPc}#)%_ows
z!bQ38ggo0xf1WvZ)|~f1t$yn&>xt4x@(#~6y7nnaY4sClcrVo&45cE1%qn~F!xkz>
zM_3&>WQ^}cXBhcVnzq>Su31RtIyiZ7iNEj(C@%Al8Gl`zc5XM#o_wDf+ImeHJ+gP8
z@lVS1`?kHx$taR|fOpv-c4LFEo=YZR_=VMgc<_01XBRn!Pq2rfP0WJ&I&H(M#|jE-
z;k2e}Dx8sAT*}^6mYZ~B^HgdeLfmJ;Yf|C7TwySQsr)ynj9p)SU6VSOnCdLT$BW9@
z={_l=n4wFQ?+4>l4qaWLUwuF3VVk;&N#PZp=&eJoSz5l?QD>91%=P1yVNA)n6<n-L
zOuX`gkQoz#bO7(JldHN@UCal<T!iH?ll|Y*FoJSUE==O*aiMwh0PU<{=gBhfeZpVa
z^;~p_4L{#0nO;8?&Pxu7Z7kpR>&=Hna&HT}j}i``sBR=tS0pbHv0it7(p8s3mYauY
zctjolc|PywwJLg!S1&2sRmVJ>&f>=U5)jUNZf^2_B3H|g?w)-5Mn4g<ka0FIX}&Yh
zZr$uq%MRv+_b7gs8&v^5|6-m4mvW}kVycJk1twGhH_|>XSLRz?4_I9Ih>-CSoG4n1
z=Xzw*8Xio>gOS~5CSEAOCNM*BjjOD8dd}XXHwj5H340d^aE1MH9l)x%%HRP_#3Eab
z)9i$~AT(C|FgNkNWU`>JWDFC3b=Xc=GvEW+Z`nq<24rsy1@ETkl)YX-JM3^~So?vp
z*o7JPu3|=iO(<u8i$N{6uoe%-3W=;;CAT*n(7Y!ng=Rrk)X-gClCO=fo#-8olCW+^
zLg29oPO>@0n#oIvY<BQmo91!h0J0b~UTtz+P5a?O-pzl5c$mPnNxI!bc9k4MNEV0T
zI4G)+ok#2!qQJ^GrpX0iPCF%8i*pIKc)D(6pFsk+cbH197z;^G3y*y5lD!r$TO3Xd
z1Zi<OTwdrL|BlCG7rdy`;$x~yLf|ul0Tc_(8&t+xcmgd_yW9|`Ja!zoX%Z<f?b|7m
zOlmPz&0$YBD&wMNht|;puz@z5rgFSwBHaS%jZ6k|zW@E}2GGB>mYuMJ{r-}2ib3q)
z3bLu@E1lTs7gK4J$d*7)l@)6GJ0S%J1k0Isd2aKqEht%{6cUMST2KhW*dSg>OI6L|
z{4dbgoM+{{z4NW6EI|fh@>(v=&dMx=>ETgW>t1^Lm%7D|1~@15m<!nyI8#frTtJrx
z)MY+T1kN-MjDYaSJu48iWOhtpgOJQ%3MzYQ!vi5pI@imoCSX5Y+)HC$bBcYCrlS7O
zgZE7GV>Vn6PgbeXf&X{OTd+36Ea;@oLbFo}<hwv46Rozw{zy>=UWe`xdT>cJTe;5^
zl!u_l-FcbAHd|D89zX1no73)unYpEIcJ8<bX5r(KT+8rs?oulf(MdW0jQom@mp!Q7
zzqc3=&*7vd%H}i=XAfOq>{9tp<!*C%-1O>(u1{mU5?EisDXfIgOU<*{tlu!tW@`e+
zy!hs84YuzVwuE4eJq1tG$PmvA7RXdG?_cc{-F~YU!2}@gzd;VfO|EB!wp#N3RIA(2
zU39|5BX`-O-g4iid@$cdz#M}X_)I3ZK+Pxm^>gj>g(Oj7H7CV7Au#NIIC-606JEA}
zr$Rh06aWGPxd2%9E}VW!4G5LUS{yPbn_3-)OtJpLtBqwy4rNI&G)+!0DS(;UNI^MX
zTMZG5L?*AAJ7ml6v|wM=O>h9tB+iyox?~clB6Nw8-~6$iM}@(*WS-xH1ciz}OU`GX
z!LtDV@N$&VPpTjGYnlce%>=+g-VC%{FTa=u0f2sk{VD4;aCRa;Wb%c=`g+^!`KB!t
zAe!5fzEE=d>Px$`^Tb;0%ogn0s{RWO;HgOf({MJ~0D}}>`T+<LlOZPrdK9D;pUm*?
z6XeYwnQ@9LY#*5TI3z^AyhZvL4Kue1O++TYgnjd54t#M?nRhd-ZWCmuCA~`J(usda
z?danSHU!;82vS4!{^E8pWm`fD1z2op^?sl&n!G3(ng)aCr3=GqX`KORP%~V^VHY_|
z+OqZi8hr9e5Jp32x-P9V!BpqCx?ODK;+Pw0d&!MW2l~)*m?P>KYRTZxWQ#PPcMtZF
zO#`|fJK=Q*Q7wT>rY6eS!x?J@)e50_u?I3gJZ7SdT3Zo~1=l#iSduAJBIF-vxn=n~
z?K)n%0i;mhCN4%uWZ&za7)QvW*N3(JGN36Ncs&q1+ro3h^4)j<YnM>ukzC|F%LA%L
zku`iXk9Y*sOQ)~En~30(01kTeQ)~r2Y}Mjkd${Obx=qvt$Y$HJ<P1CzK3PGhY`n7G
z8N5Lg>#AuTagK0gPyTJH;ttoMt8j5SJW<RgkswRtqvUiIL4Rit(}pHt$oz_c#myok
z>iS#jH_dgv=EPwasn{f_Sk%`C-{!Y|9aGMTmMm-YY+>E-y0nvpt*xz*RBPsq^*=p)
zHfG_~Rd9?4t_QQTva(tV<cJpe$F))YNaWC6U)8jec@+E&rc_Y1Md5p5M*$}ehndUe
zRS?j{)c^#Z1<I}4c^9I=CR^|<j||K#`;q=^zq0Mz;Uo=XJxiMroa_26J5N5_n$0ir
zU6%%4@Qsfb(9lz%yagJzp}JF30;XHZRrQlW%`aXs{m96d5W`B{r*`V72e5tLT{*1>
zELGx?Q70Uc+d!<=lL0Z=yF#KIF8GxioD9l5zwj}Qs>Pb8u)*`~t#Ul;vB1SQLXh2R
zt5dPi=v5bI^RuO453Jbi5aiuEJ*rEW1h<t#zb;MXc0@$)8S<k#21sX%11!m@3&#yt
zpY+c(jGFl{Kj8~^Us|+0pk)`Itk^DP8%Q5ir=o-`EEX!-ntyq*JK#2QPfSeAdXUnW
zVsKI}V8?pRNee9ipzMx#ysi>Y5e=AgkIgq<`B%8#ZRAVf_o8aiSm9En08`EVwK?Du
zAw1SQcm<|IlJW_O4pEf=q9Yr%nAg{@?#y3DQDlj_tJt9E-U_)G_-q@r+voof&-q2X
zpI6ahhm66K5j0z2n5<CRdH%k}1=yA%kD8ywV!%T85#_Xd8x2anWL)eAUGk&FlOzl}
zNz-(@rMlnaeySoNx~q^~J6^H_eDI;GJQVaYSW-dt#M3tk4p9}L&5E21L}aZ0VP$+K
zRdK0GO?5f6?Y}&mOVx?lxzvAD1~C!>XZ&04neqToSy`@DcdSWK!OUi`ngT&d40%NW
zUvAx|_l$iH3E!;$&|Pq>kbRpFd**_=;g27z4n?w6@apbW!Mm0|EFXxuP>1x%nN#hd
zu)m>eucz>jy$4b_agbXhL0<h41cLJpIT*`-=MHQD4cuKltft)?Vxyoc%W2g=;-MKK
zXrt(CymS-~A!q(=IRNV%@pp<jd5&3EQyq4?Dr~epviFVWo%P?XJY$)x0!=qVfXt9v
zeB;??pjnM%&StxLSd|jTONeaH_#L#H#%_3ZCbJX_cq7|yWiQ2Fai(1SIT+NNK+w4^
zZPi4I&CIJ0%cKf7Jv{$8czy4xQH<b+12;SS<eXT<RwJn+7GAq0I4A_uJ!67qE=rdj
zS*0EN)`-V}$*Q4<J83Examl_8^Y3t&qaIf08P!OinO~&KT6Tcfu&u31tw9OS?sReb
zWEeS!YX<}H_Ug$>2KZtGRU73o^Fq3;844vf(Xx~XCirX#7mJ0Uy&^yk<FcpYwtUJh
z?H322lqQq1WLi#GQO7N1k@6YrLK;?MOIDaOFKhk$!1+ccUM;=8<l92zR|MKzXGQMz
zSc8zxq}{r75sd_CLlssL&VcBG@e{P|ORgm+LBOP{zK1&KaCYifM8qpDW5FcvU}P9b
zRZBih{@RBhb_Wl={VQ3Yv~G;M$c35)FuTrMTA0z-+Rd+6(!ySt=rBk?#*XUq5Y2GB
zE)uz<6z{K<;v9&gX!te(PGOYo9O#>FIIRsp%*eMj^S`ZU3fSs5;j)>Z9iQZ4=dWO|
z;-zzW5X^NZ$azTdK=T&-`aC=)`me=+E=>(8gO*%)ot*ohBrfI1@>Z+DYRKVM#=;g}
z*|VYS@n4@%ak4RzPO_xM|MLXs{Wx{At)Bqt?blf{U=r{sPE9q#+3p?zlPgu0+X)R0
z+?4CkDk30H07Qx<`B8VPq1StBKxMH1T%ivE>qFOXy%uCIY86Q``IjEi^Np~S6$W)n
ziRa`u3?&A1&+5Fah?9Tm286D#=~0^JNhsYEr+*!zoqcG@I06S~ibsD*>O)DocGnnW
zdkBcmeqH8OL=0z!9)h~~za)qm823~Q9d7*;Ti=jC)2Unrdb_H>UJrGqKBMiA{;O>>
zEkoH*N&naSm#T+l$~K4p+xktVrs5Tz(|>+aP^n}XEA@rSyQL2l$U9G#0uFUH-LO=n
z9*{WlOh;c|ADbD&-KAdv+T>*n(5Gud&ES%w$TMSepyRVEAx_n@oyVb<W`Ppk$m?a?
zlACvZ9d?`0A9P{YW@&Izb!RQ`!*h@~2rWRr21V^xL)Qj=aR@ojj-UV6tLsM(h9m@*
z)I9z+b{r@_+mZlb7Wq*Q-2lT&gopwm-Qs#dm#yQkXeDtjkFS67*^~K^y#$CcECam4
ze_3Tv@6*lMGpTt$rp3E1vs4|)=RL<%)Lv1{XG{{d*bFKTAcTj<r-<B?VVhygB{jd6
zQ**!^)Mx?@BZb}G_5}1z3#S)%Ta4Vllt6o0{zjj;Gm3#;Y6NqGLzs0@%?zbfB`vcb
z@&0}$Xwh6Gs0>am8C*W8?1wGBSK!33#GXj-%K%b7T$*0?$VheM1c$aCK%gTtB4j;!
z?j6~_zZaD3H|i#LYXs!pgc^9G@UhM%nLo-|#?(r-ILlcAIDfyeu48VSVR5jn$z{7M
z_~UdG{E3u~mC&1oy;;My+$eNI(aSQbPDU*L;)U_zDpj!RPW8yQcwkp3Iaz)Up9hlf
z=%3Q0MYV<|IPA>@nuM~04=6jxzJ&ye@sWpivfY}$HO94Mm{V`z7e6emQNHlFZ7Zyy
zC4)VU<<%zhz^Z;w)#DypDY623y@`+6p!CS=C0Pxzp1MBNaS52C<-F>(os@^$f?%F5
z4WNK+=EnF1N1ld4B?VuxKUwqkV&<}-tx76(086Qs!<jBiCXg>tpF-e_{BXlwHElWR
ztdfeR=CVI)LWfoHQ8MC(n+`YedHQ?()2HA1+8MhRpFH;8-a^5zSSrKr>~c>CTwZl{
zmCA1rY0cwMsh8C})j{r`^jrWnzcv?emu%1kU}k*i{!dn_fZU}5)lyJU9snYKUC9FH
zV9o=~ar4mZUk=W>-q!@X93V!@=*;U>NVzuw3<7rPvt1d{0zmz4Wr8{v$YkCaC)EOv
z0H)TWkHPr%X9@%mOVeF)%H2b=Zl#JSWa~;5jC2-HWr4f_q36v?`7*9=iEp2L){)?F
z$t?x!W;OnyPSiz6fqvXvWp2Xku0F`A%&*E@dD-iw7>Hm2;C87WX2_+rqd`;sHAdb|
zSK00FZ!-)Bdwg)a8p^M(+u_=D#9q5=+$?!qGh|0#ofF?|+a+7ap1P>e#b*5CfkmLw
zT}w^5yk(DGfng$rF0;yKErA;d%5dvgU9ErvYa5S99xpIuRz(vFUg+q>a<o`iq2>>Y
z{)bs|(f=DCVos!kFlkP;iN!|~$N8Bwl1Ho?5inNu=cqxNvaAALgDu?k^=W05RhA#F
zr^B%r$YcG%cC)W=P*hM{yepU_5wM+S1A$gTvu884tNa&H1vz$(>2~PorRHksVY*yc
z^Sko%vNe|&q5K7~4u~pl5Bz`}ysRduJ;kZbHf84V8oC(Al!gok!wE5`-liMnLP7kE
zW>ypcdO&2`FA8L6uWsNy_e0>Qhp*l2U2w=uMBsJcrD0XHKieYRIcCWWGv=1?jDi=M
zdeGiRUV&SE0rb{RYdL5jj;}z>@A;HahXkNUvs&0-Q$GMkMr5brn)-#7kQAtnTT(Pr
zPSJg*lx?PW159?h`p6+om2N`V>~LojEH1CAB9N{MuNdlzT!@DT5=+mA$thRR3F!wW
zYP*u)&{E7Zm~l~VD(^s2wcQq<-E;Rzod$695L$VvTszaA9X<EXo&#Ev2oKaiIN7vi
z64XbDP{WRVsko)B5iaR~gO0}I$YSa}ca9@-=1vvV-P{z(y~-IY0<Nx>Ymd{{&%YhG
zs4;_;5M=A1ssi%UF?p>?ig{0eSR$tn6!(Em(!J0aX(>*>#Tx}Ku2>)~z9A(k=tSNJ
z<d-YQZW;^|8qvD}VIu$Xm=0*3p$^ByA`cXILv3?9pe~m@>{}$64xpmcjlBQk0rmT)
z0k^?Wfk3uqEwn;N<1CXWA*&=H8j>Jz0c__<kKF#iy12$AKJ}hy;1UfZl4r8fqtI_~
zKtkQ2zfA~{l4_ojUeKbxMG?}O^AvMXf&<3NxdaXR(yJ<mEOM3{`x}%c9ZFA5v1}Ee
z)=W#X>FZ)pqQE_Q32seGEd)eD*UOX8(y->uX-(6EjT~?R|BjL)`b*tS9#)hG4Lg^Z
z$ZwFUQo%<9tmP-5P2H<y9-IUzo7V`Fexf1eNn4ZE==aNDS8?bGAM$4H5vY^k0m8as
zNgA$PNN7IMR6dIxbwF&_AKe|)UyX#kamn$%#Zq@kge;f=denjQ)|j@l<lV*YE%1eA
zi=lL%PlU5P%YX6Q!pz5|c`Ne8ztw)EkL+Hu3=z*~1Cs6&u*&3sNGU8)byos}77+HR
z(*2&810F~nFwp-8jy2Qep6YtHtEzu3U?;KbIvyX~QS$C^Q2WCJ$oau5)b3*UPqQa|
zIY<ew0Q#RFCrQUGz}Pe)(4<oY_$I9o!031b2>Ud?kSC$70z*P@YjpQF0Z0H|nbU4>
zH*>>c>`Cr}^78T}iuk>J4njc|BuN;gWJMLdAL)LEVldHD3&DyRoTZ)_njN9#Y!t8`
zV$|zEEp-i5$I<42?o*Xs(7Wrz(@jugX|eF(){<~za-FWTu^0@ESd#Wt(Bu}Ge6Q<F
z+9_4Ycfsk~)@=}66v$8GTBg8+39|@BpU6zCbOB7bh){{lBgural%<*);$1D})~T5U
za^Gq~?8hBTli@|FkYrHOwYv)JRgYvan5s~rwM479g2$ueAAhUon7-N|`duO$VDK?y
z^6~<Tv5U><0jC6r7Fb$s07ed!mlL4^8KOnW5VV1gPkPv+5FG@DDad{#m0XCBj8*d}
zT8V)m`!3%iXaBu9DJdxseAT`Gx4~6VPi`SsfT01fOa$EF<%i}^x&>9v*IPyR<`<UK
zZeJQJf$~-hpt>8fSkMV|WJ)2Cd+R>#3b_VO==@0k?dB#W`E|#8%VrE@YoQ^*5=#$G
zWE5Asc0u3*n}4prwxmTJ@`xO~Os)XHxc3|Dt~i)sMuy^VDoxLV-|0e>O+1(cL$t?Y
z-`Th=HXE0a)dy$mTDK3B3V!lqjwf^`Q-=z}#xBk44X_Ju%FevG17^)`9F4VytVSQs
zcYv$}jU8bSPsjWT%;ZeuoI31&82uEA4Ao?j{Tv-NXlTy}CKg9JNL%h2Z%MJ9s*ddI
z$W~n<8l_^c!P0x4ed9gh+<IXIJoIHbFlOW%cjnrTJSPL@O*d_AZ61gdb5DMO8yRAs
zpg^sac_VIa8r>~FzrNTHRrU7T;YY#X-6Z?q;Jv_-ZD1>a8R*rgeC?sRB(&`Tv+xF?
z8UiA~YFaMor0y+rBcz0Y!VRK2YsGhe)YQGagHSi~M`ZlPV)v>D`1oF*xao&mY_~~i
zvIRJj88sXZ_W}F<+AqqNiTx)l&z*40vNGL|)6rg<bwNEc=UK<0M6KZJjXI$HV!K|V
zLlc)N-(S*A{GIEj!U>6&R#|OLu^dXTe*fnRZw@7y45qv{XXJN$w3_b!&dme44^YbZ
zQH<GTs_higD4_F6x}nz|;p)?%Un~R6BfEPU>+!)1Z;-QiU^B%QfL6FBuPfTBSb;BA
z6{0kKz-gHib#rk*h$%UZbpB=wPM9GF!s%p)U$KzRb2z4isDn$>;w7MlU^WKuC-ZZ8
z@Bzuw)c}GMp=dG+>@J$}b+7(-$K4zJ8SR1KhJ~SOsc4*|Mw;ITRRIGA^@gC+nE$W^
z-Yc0<Ab|Zi?0hG5z~V=Q^h_PtLXvdrkkz(UxNNbS-|e1A_zyr2Q@I5`9Kk4F)(WXJ
zS{lH1S2e7Y%{j?_<WO<AG9N1ChP{pk0V^3N;*mK<7Jhy!HJQ`#=3ykqCMb4c;vOm+
zkl^4J8LJ&<-=@E_-5vS<qCx@=5KR#G@b`A`e}U2o{Qvi+`|tifC-C)#x4#`@>A3aZ
zj`9DVV=R03pnN}dM7OXtWgnU0UU<-G*7zEp>U%b#9uzzI6>{led131a_-`_Y-69#y
zX%eGGcQX6m4rN*Gj8`hWw{|w6s>(IPTT}*KQr02FrnlX-o3)#4kw6_|{MVhv!(Rd#
z=hWTN?`sD;rSMd|&a$NBQLO|m6Qjns>@IW1o-key6NbX|c{i=`@p(Dg;Uj2&p-}=D
z^lrHlUGpJ0e-!*gNoD&g4J{Y_5L^n|XVqOz+IQZj8xJ@~$mO0(sa|_1N<6PUIg?ip
z7{xg7E-E}~9XW1g{TOezk)mT)Hy)^&DJ}tpy<ElWe;u5Qs$S-k-c}ss;fQdFoz|-y
zeXVHppjgCL2j{0!eZDTP+c$t?P{<aTUXmp;z|j`%cI(i3tz6|G2PQ`C3M#8(FL{ff
z^C-6w9J6}3Uaq2BzO6AXDNt(Y^?H%L{F>FB)}(x@ndkN#q4@UxhykuuBEwv_5hRwA
za>ebZmFAc__IUa}!(wAsI@NI_TIyNQDm$&@b#nC96-8|aU+RSAu|#hd99ECyFP%&8
zixkOp!Ci`8K}PJW?~DXZ`9^5zV_RvUNTQ_|MtMv&_U;>1@mai)>od1T_d)Tv<O+<*
zXW6IVuH`u;Y9tC5O^=Ryly5zYu-o-G+ooZh6NWCDN;6g0$<>sh7n>}^>?mTa46rLJ
z|J9-AbhYdh%(ru6<Acez5*%tG&O$sU7#We~b{$@?cP@la=G0;<^KnK);=)AtM+_3)
zm#)F8ooz7pnHllKV|z;98I3;87Sejl{Px0NrRBPP!?dJP--8K?)Cjb6N_-BfYM+v<
zcq@<m_K`L@dGmqx({l4BCFg@WdJ}?5?L_v3txXq}vF4Vy;Lap$M^}TAidC*;W}A`V
zdZ$>>9Whd!ZsqHwJc(+mE7xmE$6(%qzbVHdv6!THjd!AAVh_Yk!t=WJMx?h5`#spW
zz*p*|g_`40<`mDmaktKzu>FovEWzM!V}fHbwXEN6J`~AYKR@SZ$KzH8kJo2MyYFFC
zr#C#F|KqkGI(_=pSUP5L3iiP%F!5bpOQM}@S1bZfw^i@rhq5LuyjqgfSuBNN+&y&_
z*4=8%uPaq*@4+A86a-r+6JyF=&oSA{j2(Yv(FCW>h3*G#9>vOgt?8zCMO~lH!Q?!S
zQ+=^A^$@O8Mams8^a^*BJ@>>g6YQ@1OFX|jn&WG91_SGeyCU$eE3@8vCcQ0u`>nOG
zT%Pj}9j-r`Z__Hp4{rb$1uTE>3kH;eU0?Tr{?|3vHpXT3rKrPhHNbVx(){a!WlicR
zKSblj%5;-F;j;M{W5?s@7wJtwS^y2idX!!@+JwmJEH#Pil%QzO=NJ8eUa1(d=)v%w
zQ*`kWWeR#a-*F<cwBKR)sp=iBlE_nsGVey}#kU?0>wChinyi$JXGJVbq@($B7UB<O
zMmABzm)AA)=4{uF@R7?WhHJK+#heXGvU_^H4jV49!Hpm@^i>EGfY*Q31O7HP_SCb-
zx)BQ(y5A=H%hIg~H;lrONU)c&v-2Au_X-g8(f)BM>5|KKI@g_S{$O#vUf$6#YM@VQ
z?k7JyF{<WwTyhCHnx<nBaFXjA(qpKrhdCR)v(MRMD!287iVDn_=e#^d%rCjC!*Nz_
zH(z<zYY$6!h1nSQ4{4O@BNA>7UFAD_2&UySe6COOCOB+h$7AhdlAcEIxqs&mi@;8p
zaJi{~k~3r{s)KQuux%neJ@z`<negHl>=i|{ZK>VaDrHSn(_0l6*-JZK`!Y#!`FPAY
zl{!8?t{qN!z*emvuk&(Fa;&5M!_sjiM6wulXL8&yhg95fI*g0M&P>>@)C3fawmiag
zURT$9h`5vEJFAz+B#eg^Z3{l!Ndgy=p1xjtiY0s6Pz31U7!s>g9d$wm8{<xlTum`e
zOh_wCU+uFN7D87T5{sg#)T9BsOV1yqi)Na{N;zp4(s>t|FrN^3D^q5JGJD?`XK$68
z>*Ml_hSx4Pz^mS}Gkhwf!v4UMz?LBKHw*+~L$B%WT;Z09CnV)KVMu!E*2fRCFXq6N
z)wz?j@f#_Ar0A25R>*TnU;`FcINn&oa)T|qgqj*Up;tTZ(`NX4ydpXlaqUHO%JL7x
zHRTd*y|CbPwGP@nbq;R5wyr38G-*2o(aXWH+b>Vb`d=>yN;lL$yGaK<$vS@@wanTv
zUCkwW;IRX#VOZJ_;`<C?W3*o=d@+`A3jB#xyWtPE*!pHX^)T?&aF#>ef2hx;HmFKk
zW8AJ;@3AroZ2!bdD2@Cgw*y-9;Y~s_$y_;Gzz<s-E&C;KrHU1?A>~MYh|9Z(kr8$D
zhbQXUk?tgL^nO12%8a2GUr5~qr<AJrQc9fe8@0h{S(6=xUM+L#Rn&`3)9T^B)9Imp
z?%XQq%&-YKIs8?3cj*|+Ah4}+xa_$=6lo>Z&D)GPdm3>?vCu$0E0R>1ejyNf>48L3
zZ;qq!{R5w~j=$E2;BM?5UfgE%lpfRv!q}IZL@vzD%`LA!lRHOmCwRToAu;Xt>+bbR
zp9U`5L%k&C7!cmBUA_oZUJuhdmoQ3o4Pkw}US{->-TST~UpE?K*^CPT=bzibDd?{b
zE-Sfe8_hPoUY0kopVCPtAB>>0^RFo+6EkhBqwPFA?&z#?yn2^tr+O-%KqwQw**6Eg
z0_Br%YK$Y+7GIjN1$PrS*R$*{%+!6L`2eMHGaIfDs&L%UHXx#pNMS8)Q(K$awzJ%s
zO9dQgfjW9c>8iOdRS1{5`lZv*@?@0%^lxyj7XpRDDA{uq2k{{fb~^j~nP~s(I;)<M
zA6kxHxl3gHtV<C_<Mj>jnu!S+sr-=i>zLA*o>C*Dpnb?&cE6DpchlC`k7;a;FrN9<
zuI&_hLp=`r2!1a1ZI5&RrelcQ8!y)D*>vi3q`<OcSrp0X{xe;Kq7?jas~{r1bo42*
zK7=*?Xv=F+HU7WZwI{*f=&|U%zgH)`^=enX>>&Ip0LvT#UBVUrV|MQR{WnL@w(h_9
z(!~$qv{xO4<#EzgDz$cFnFro=k4H2<D8~6lkCYJ7U_ECx+z_dW>k${cW0JW+zkjS`
z9E1Ie2-`1;K*K|ykJ(XHo&uB9>9(<Sq2!v;&~+x)B|yA8;CmMut$e2hJ!c#HD^t46
zxE)SY>hL+ouT+Xx1RNnam;U86ul;m)WPY4o_DNDs(0@P)<_t=(fT{yqQm(DZR~T8L
z>oht3VrBs3$7kBMo6ibkn<_(H-e%fZS#B5^?^P06JC0!PUTSoy3Q971OntD6h*8i^
z%%jHH1;YjGY($vo(Z8IPYY7dJpuXy_%%LwEMcayBlvb+SMx8x3w(v3dp0*2aa{Pp1
zZKs8!I%XaUwUYglXOl`B2T%FV9p=42<v+&z9--b=xmgt=e*VnQ0Ka_jMoQ1oepow8
z_@o4fsG*|5&;#~!yujE0CAdFy&+!OEVfULYeBO7nf(TGOL&EknUg|~Ly6l!+QCyQX
zo7eQjvue^mGReyOKd{EDlhN?qO!cTaBx5hNDX!VYXY7Nb(NNpK%3o^dFdNXyoE!0b
z_wz&kb(mx5g#fxSRpm$F^T&rzbW$DQM<><sbC#<|YYFS&Wii#py!j*ORht0G^KCF-
zjqTVnF?dmA>!+pGLH`j}glh-6-H*{?Y&h3GOe$d!z4qjGso@xt)2FLM^Hx^A2PcHo
z_2FZw>DR067Ius7jT!|LFo#sUUs|ismG64JoK}w!)osP6-zNl!qav!T9&2}r1;?!M
z>vFMG+;0B3*Xs{WH0b7nZ(*Tkcz<+6(j<*kOYpPyc78iS%`F}seBP6e*(=it3$gb8
z?Jc84w>Jjd81qatbIm9^L}!}&^Wwb!JLWKHh6_rM{hL{Yyh&Q+bxi)B4LZMaJ#0)V
zbHjnQv3uTnlv_Q9WkfRWE33p)(^Y6p{u%2CcQl-~KdWkL^se!C1Nu9{#aDtf)UVP^
z=%BkMIPU)Caqov4J9RU>nPeC5ew}O`63<aVHzj+uFReE<_Qd6*=v69aVfDy2b?MO)
z(@hNGv*Z+>Z%fL-n0ejC2H?lz$F+@}V5BQ)dqzYUmigw#G%l_2wRYJ^5z^?Vh3_J0
zzCZMc7237ndF4*(1Jv^GSK^GeIN=0S$_@U0D^>dTH;w2LTe%;~p+ah8gqw9HEnqP!
zqN^q>+R^3#S1hi6vmIV$YV6HEGAa&WtWoh1wNJ7hLRUO|1Jtex4%-)b+E|~Z^>#fl
z)qYo_EE6*)4RfA^I>fv!#FVguTAq~qgzp^VEocqx8h93>`~==!GfoTPjiF;t+a2+>
ze@5f-%7(e)m!t}Z!=*OFvHoY{;1){bfzDxLpQGHu*#~HsDFp!Y7v%;_otk{#r3)~W
z<mS-MflF6oJbWR?1qUQAT|Z!=E|=7bZ!Fzou^cp9U*>U$N(FUR`)RpEQvtSn0n$%1
zNzTUIuYh-MI&kTR-2OPtrF%I14dwdpO@EVz(3O0Db435|aYVpF|Gvsa>ASx#%=rrO
z022K_;~1KYR@PPzavg>*PP<<i77!|+7GO^cq1@t9iP#aD9DiJS9qEwHc6|rU=Ml3p
z@9iXgvoQq`5xXUK9j*TO3Nrmqk5BacI5`WET9+3l%bJ(D^aag&zb^GMDteQ;Kt8s&
z`k)$aQhM@=wzaL<I(mL{P`Zu}k!$SZBZ_BRDdi7k)k^4dr7pwk=zcPHZR11gYl5yM
zUCXbEp08ea65Ttm{?5#Xl3C_QfAKsRquif|h`~5@rK~D~-#Vb+{8cHc<d`v(sjZTt
z*#Q31ZH#~78g7QBF+Fm_zWkLc&(fk+2fgx5KfKp^N{A^be)l*Xa}LRDd>kghKedln
zEC95L+_g-7ELZAV?D`6c!3&}i?#L(3EjfP+4)^YgaQ%*zIG!3opXs#S*s;yiwegZ)
zmV2MikZ=oS&OLF*h0&ZYBQCpyg(K+n#EfTSJmVF^JNYukEh=S1cT3aUx_C@+8>rhT
zJ-YVPI9L(5pe+DMB!>pi;6Vu*)U?J+mFl5=6FsTXMZb5{SrDm{lv6{49{jh4zr^(Z
z0l!)~i_jYu>~>t(@UbURVTM^dj8F6W^C)0Tzb%^3sw8uD&;T~7SCu1}cl=)4h|^R#
zi9NwI+jus3Tge@uGSGX@fvr^A8ds_ub^q4N2)h#U4OmjOw4g1mNh$2U0RhxGR!Y$O
zvm2gNUqFTLx3{N8#LO0T;pQSqm^z<f&;H6v!Azjt;V~91!a%yVu6!{vW+62C2~})+
z)Ps(5M#y%rREfa!r;MzBB~#*8!}v0epqe{LrakuOTg+;+T;Q<(jLT?I23Hitk=U{F
zdI_I3D~PFH0GCIutPuO+Li-b02W!V0Lil|C1bLyeo5{>u{@@?d@G=(3X=*-Nz!t-P
z@TKGL0pd0fp&Kr_x63XVfp{gAMl)OJ#My>wobZue|6PGNu%8DujLRDY*MfJpaew(L
zy|%i#1h&fx{Xsx749Qq)D|;PFSBSCBm9VcF)v6nvqpRk^jg2FObt|(Re4A2*$y*yr
zbXIAobWCjx?#w+H`j}F^H_;b4f@EA%s}hP0Dp0X-`-6-YdSz&n5wg1kRukItHocEZ
zw;3#o8hv3;r!|Pph*76w@jB@}IGj7W6vO|h;`4SRIcP;Apdvfc9%G)yNFAnA-1Epy
zx$)c?o@U=FZ8aBt?s%Qm1Hi1-)2baffp^OuSD$#0qU9`1E<*~6wacObbD%wG#OXvD
zd8diP8>iJnXOfQBr^&4Q_&cq<nWH{FksmoJwt+`j!jJZ5<cW?LtZh9l-}o%PP7)N<
zEbeHn%2Tlac&R5SM!11`C+ZqPi$TguF(qo~O|;OWsF#rFo{!=Iq1Xi<j>h@oEgoM8
zKDA1tD;%*SxTCy;8zgG(VpM@YJ8>{MifFT<r4`|FASQ;ji;;+T!*cQHls*w!HeuhT
zaEktRL?S+bCkeSCvCD9~zY&JY4I)ahMXcfSGy9Van3Y6?N@D9)sqDgH5_|aBFl&bb
zN48Dd93aD55*%FmI8h=KPo7aOPPuz6lq==OuzPQdqfUGrZlAB`^o%V;NTl_V=E4k&
ztgr82R=JNi57Ru`%M;j1`;Ri6Vip&k(C{&cy?>TOpj^?F2-5z9fJIY=QDA#3T*Y-N
zPaEaPl~hzFUq$BSO>w&rv{T%e|Ik<+TG;OX1=+O}>sm~`8-@nMbf%`)XNa4a63vOL
z8q^SMj7Ub<mzYO+HH9;_JA2elXeT)#$eEL%#lnVRmhYxX5Q1NH-jnXuI)+gTo%J#A
zy{PxS!W`X?c+k6A6)I^dqj@P~A};09>RaAp(C}c;TJ|!k(|=Uqq#MF=<NkG3{x(m{
zE5+z7<TxMua2a{L1Y6VB%je^^HKYt0jm=*>Rh6ZAE81qa#aL$)T6Q^@7O{ASmwu`N
ze`<S32jE?V^Y^<5QH#wkRoE7L?&6BcN@Yi`ZKOK8F)m6CH!cz}yqn&scsuh#2Y9>p
zF-9zi^kqL2#SYWizFq1+C0$dz1=^$6lAppoKMFg{e%i0*YZE>bmv!lyou~8f;X*oM
zAr_$QkpZsxcD>B%kjA(fUpk=C*hQ#%8ee?ZJ|QBL{9)8cvMh)JyXqg?PZ3sgeo3eV
zye7La*6)a;ZEf{InwXiQfZ89PMQ+9Wan88c5}YK_+)V9TuV)X|*8@&$M4GOGs*!Ok
zO*?%0Q^aqtdXm(VtdK8tV5#GD$Y~MiktjHe3Y<15y^IrcZsj>WGds_;Yt%dpZ7kp*
zoyMD<5m;ljCn9$jfMFHOD5Ym1aJ#<&muTGw99K4lZ}fiq?`sF>o=mv#I8DJaqzA+5
z;~5t%Mr)v5`8xd^VO}B1Zk!fj3$E!5x06H}UKqX~^z!yc+X$hu`YMXd#q5-;6w@2I
znlv9?#-RtCsV>WlUR+ry3t=wEu4lukPqW}@2jE+2C@~>_Ox*fp5%X@qGizXK?CRYu
z!7>Y7+K6OJZmZ?32yhkeq%{U_8zo4Q;raYfuIZ+cBY18(Q7}&b)h%3lFUxcnB}|6O
zp-RGr(U@Vp<>TzS(Dm=cQqOuqs*sV52&9X{vtB$dL8v9;PBGJ7voeSwp^+khNi%_^
zoEaV?1fek@I7j_HA`QM}f`dN;8xv{FiNY&q1J8|%WaEOtqavY({DFUQdn<P1z^p-=
z_!v&1GFp0hI7Ok7(f43-`oL{$a<|58E%k?F;kblA26x=1VU28ulA1`*C;e~&9W0V%
z8ORopUIhW(W4kPY5IWjRrkfYwY5yvND2r-Isd`rQAH{_;2C1k8`iC8_Nq$b)mSpU0
zcRRn{z@##m@=st`5p}*`c%dR0lpU@OxhXo>HO%qe%_2-`5cP!V@_(cWr3+@#b!Vak
z^*Vpx;ii;qJ8QFJ*bJ<nr{i{=X{}kwYw83N{8^A+_z10trU`>oVWWP`!uwzjBFQPM
zLu_jmlKyNM>F_|?_Z{CDCcWtYuHlW#VFFWn`aEY6T!);DATe0o-QyB4{9+}PPb@rt
zlY;z(tMEcom_gA<j*a4A8phv0Il@^<gnN>%s3m#_ENt*15D3F}^GMlfv89mnXuaU0
z^jXGG*2(N&*caKY2h04^Si=!gJL9nx<vZZil=P?>0n5jkb7uzgv!B7owrZ@-KsZ7q
zV-Pv88m_ouc2W5uY$4;<=Op#-H;ByFSnPg!Qg-^E@xSNTZTM&G(>Jw$Z2Z@6DP79T
zb8ekpP1$!^*F10i$-T+WY9>EaJzGDQs(Q%3_I%1zm)cC<vkM&?es?!N<5E=>G!RE#
zXb%}CTI0EIV+FRx*5%@bsVN3-%k*yV8y@<@#^uBwZlJF!v#dru+>nTL?}yV(W0|9H
zyM3@wlpZ%E>Kt8&>!s#7z(}2^7HcvFBr_MistwLz>>Brt1(cfoxT^0Cw#X)%w(XYC
zR~U52&Jz<PL(ASN!=_^3cg%@Ov?(IVQN&Xiu>NQ_+gUL+^%8zMf|zL=gOO3EYH~WH
zzTv&nvUR3OEB<6$;2OzpG*e$U6R<^68w+u5fZ2aeL=c?<%p7}$jM`_6#1kKoD9nts
ze>CDXr+ASyXooE$ii*d=Fmo8x86jt;_c9|?+OT#Ygo~|M8PtjzhMUwlIR-Z_z$JcG
zt|PLEt`Iln{vg<;#%hDe4C3t@?QBAbu;Ls%dmAoS6gWv1mQ~Ho<~%Hh3T<$h``HM%
zW7N>`dZmBFk$~KgSfP@dg&Jwl(xb<2_C_;SxxvWV=HC}27rrkSe$*vse>kz~tlCC>
zY_7DD=h=AuecCF)uDQXy0l_K1ksWHmL!=v|j(S;WM+lox-D5=DD!U+*E{0e&H^v$B
z_x({5AJRv6@~NNjW+)6WI#zV5xn+_i1dl#3(L7R+B1S|-aA`aHrMOymM0)m{_7^P^
z((+M5uc%v93uX0HHFK3KG+TetTrqbrnCpwe)@!a<IlMY4He!d`FGDncfkh#UTJP%=
zRY??(>sMUh|9hH<=p!Cf7KLP-Dy?rHDA=#~Xz*>Cgwg0a(5j%WSG2VOUK}70)=zgy
zV5Vd7jNg0Fv623NjQNlsUVvLxdcE8#q?xuTvF~8tX4zJ%!3DHauj<Yh6IIaBeM7O`
z9!WbJZ<AMyzOi%F=@?F3NaQf;Qky-jNxYdkB#C|39%B<jNgz|F`!#kgn6JdW8=IZ;
z(sB1{404u9M<oIudTk()R&W@uKXr)YdVDeTjoq+*zrfzgv4?HcK5wLjADx&DUtGH<
z6p^uOF=l9LN29~Tx8@7{3to1ybu@HK%`}p2h<i~(ux(f!J82XZJ4M2^wtD9h&Ri)!
zX&0ASk7J<iQlHrXwhNp=K9C4*ef^>FU;Y_qS?L2KS}gQ<yrM6Gtfnv|ZN0qWT%$Q5
z4SOp*XVAN6Chs<vw2dfkSU>l2b~>a_c2GF@fV#-g+^;(B8HyDs&0IF3TmvJK=vry-
z<w0^{nS*|%l5fQjd-go6N7f<of=6|tr*DsniR7A<YSpWvWyjrQx1QrCWXu+Q9c5qM
zXodq5Hy@$Rls)1k_Bu_qlHfvYxYhFG^PwHalLqOjp~tJYy0pf&Xsl|HPTeL{q1(ii
z*c~Cme6_x`$_eSSr@fb-Nb^YCoy3F+#)R1kLD9}~MwCp9nA$=$b5tLd);z6A%uH;o
zB*&A)w`OVHMlo<sS)X9oMSuMiN&SSKzyK?Vzs!(S+vTm~BuL;V8crz0_B(CAluQ;=
zCn6LPwT}slT56u2?DM$d>Siw9)6lazIS*DY5+7cOtuZXw3hO8^J}$6CMd0m3UN0Zx
zZgk^%&v%;j54*)OVRlY$@e30TYjGdX!Wef3)Y5EKSMk+7W^N+TAMHrfNj8n7pz<X{
z@oNI+tJcri8I0bC2um>4KN$u%GO){o(FtES!8G5MhSiNe&}g1Pi=a2aB_X%sS1=q%
zCo};+9a5*SZL-tMrsaaniU*4^Ged*vRzpj3jFGsx{_+a9@(_x_;(r*^N*}}F7i)h1
zzuNoqxTdc3Z>F`S6a-rxqjjN+1v^&24OEuYl~PoS3sePChGIm7s2~ACa;*p%MU52|
z6d^7sDg~4+z$Li?B7=w#5Mqc)WRVb-B#@2d=HC0eVSZyBW|;Sn_x<brz=wYV_vGC3
zoM$`Fd7kfc`tW)DdW$FYls`KeP*QARL44)2aox_y7B<Tn-#>83aW}J@2#Nd_#!;b>
zZB(Sa2_(B$cf>ElP+QsPU#T(m#O`ZY$9*Oh%z`?h0BtWr*CLGEM#u{C2mv0q3@@h*
z<sg>_7(`h&^EM>&jo~C!()-wnfKjSr{Ix9TVHWt~_Jv9~gcr8EB<+~5vx}~vVlhYb
zn4|s^(<<u~yJJ=5aL63gWKg6yOZpWi7vugtjLxwwRP-)F-gQBLizj}9{E`lL7+vQ$
zoIZ*LwO7HuaxhRYk$p{A-eX<PT+RNP@KxP9QM9jD$ATW&1sF2c4vlE96RILfV4LTw
z{<PyfVUHK@CyyoQ7}y>2-8b_c<|`hQ{eEPGiq`5RK-O78Q$jNx*@*^gn`*wL&o8L(
zD*}c-h(hK>Aeq6-Jr=XF8ylXb)kry!V=sx=8oHPs-Wc{vp-SgD6n3b>Ie>p{`DtJ_
zKO8Iymwou%5`LEj%h6}AIrkh#I*(`xegC`A%+I3)k~R)|&BIMS_qwt=(+B`(fy2SX
z-{Qv*%jLkI!nVH2>Q!^R6C6^Ej<0L27x{YwpG1U*+03jZ+5<C9ugUW;gLnFh%9TN5
zO9@+@sDYAdNid?(pzc_?UA$A4M5aqznea5|3iutP99DTdD*5E1Sb?;<le?O`x_oH;
z_b-eq&PHbl=>y-*mgvyD9tpT~AMuMS{ozm@)Ks7ThIl53`q27IU*{W9G<SY<MmV5+
ztj|l<r~;_Sj<z)K9Wvt40<cn**i|Cx{d@q}Tgg6nM7l?66h)d~*(uw^mQ|)X_Z40V
znd)zd!9-x$5!YA|Hfz{dk`abp9aV`0AmKJ4N3YqCl_j?H4LKDK3`Y0t9%+!iZ!)YR
z(`HyQY=+^)pA$j5n}za*)-E6;4Tha-ow;IGWafQkoX72kAr)i(Ba`SSDrxuVoXbn9
z>2~m(t2=Wp?nmeDO#N-Uws>Y;&+G4W?Dhe8w8oxw-ql=_bR~3s>y5e(kKtGCoyE$n
zBz0W6z$RyBPE{3ZHJ~NXdMJ0iOvW@CP?#w051$TUK!rps6w2cQu0X3R3P)$MOGG!T
zc_Tal5oJ^k2XKehNzYTh%%B{jBzzJP!Hfqj!Jn}Eph3<hLh;wvq7Gv<eAh8GF%hCA
z=)W8lDq1t5hhZy;3R&Rv?eAV0$;e*9p!0KOjzQ=vE~60VJFOi$`aJD}v;tyJe_VEU
zO+Be<<}2Bf_LTbN-*>u0^7>K(`NvE~&|?1lV!h4WtfbSKTz-Dewj|-E{9%erSd>Ee
zYk4Ix>iJE3i99-KBt>CTKzFGRKpkF3c0rIsY~z}=%#k(C`4Y|%qOg}&##_Y^<n~km
zPQ%9t!9A=sBMo`GneJ@5^KaX8HhaeZpueSTIyrWA^bT!|J5O9@xZb=&Pz5fLvRj8-
zgA5hyS;UTx?3UcF9@v!9Q<p6see$=+wJS5v&XTd$g(~HSriRDv;&zm7otooI(T9EJ
zFa(^Q-xd~E!G~UnO`$6N&VAk-1+N%YAC89?>XyCK)0D9pvn87oQj-UE&vJ;;mkkXm
zLp*1DJnZpl4Vb7YU1=ZpmiUr3R38}ItVY?>f`L#<qgxD<@*znPjqW!}1CG9?ETn`~
zJ}h<HxL8R!<X`TQUfXG1PQ6XodQb4Jt@Hh-ZeW3HmCND3aon#;pOBv~#&V3^ZRaV+
z?flxS`wk{s&l%i2y$gwocV7}xlGv*HOlHaZh<osy|DGe#khVAi!;$sMd8-uLN=m&&
zS?jU7jo8=p!3zUt2P#qwdm(ckw3OU?8l85Lu<Z%_;84qe3r9U9xM96)ub5pPK|Zq2
zZ<uRO_QFt&<1YdP3<y*>X1(HcaE!VE_e-#>IJH4Hz^beZqjf1mMn@w056Rk){(aO0
z{jie0?JxZ0#gymeqVxJUC+ORR21S{oTZ07YTWx5~A-12RRQ_McD}mntWbW3Fu`^5$
z@Xr_ZC4hlBTD(!s);|G62rZ92cmsf28(L0T>j&*=>zMJUOX*LY<~n#(3%M2WU_Ozg
zq#!*B2H8n&tw?{LRU?k8%3_OYB2a&E)b%F+`d-FT_iAPSvjY{wfgU-FqRi0rtLjeV
zZo+${rGy?YrPuj6ve$^XV;AiLfDOh5`&eaCqOo}}Plvl5gZP!O5hPdnT2|h7F`*@v
z)jt=nd-L87Y-{gmI;AUekk2zD#nrnpVQT`&p&;ocb&+#Pr=Ty0LH$==_SQc7LZUdt
zSi<{;2tqH4YO06U*Bmqz4;)ki;l`N_F&wQ5J;6~9=}6U3Gs?R~XkaAegUh*4)bcHh
z^<q`tud-;tvQ(?O9UIu$SL@b2_E1k}v6qOODMQf94rw`!Fo)D1ofN4=OXL4^m_A5y
zxQ%_&DsMP{AoD&lTy5k>Q-RLH{;)394sA`tv!fT`#+H!y;F!IBYt5V2=Ty3se$bA8
z%Ad4_MEyd2NVWpNDn|>LzU)?@C|ud(179ufRclW@Sz(jChIR`q8_FACvLLPBxgz4t
z@SZfYM5V$b`EFnqNYsV|f9FKVc4Vpd;x6;(U`xutpIJBe;a~vvUq#uM_@-K=^K~@m
z<W}O2ItRybsp8F(@|a0B+vY8_Oy`UoTD$#YVjRU#c5tp&!DmUfq#aworF-~v;#p4b
zdWv!v&jCf!NVjHQCK)6HU7M*?soPpOkIKjcM_c46k4l{5+!k&DA_@zbR_O|$ENB+=
zG8j4Y+K*V>F1`%DJfhFItyK5Y6J*rmlzy}~lseel2!$Job|DE-DpssBz);dVvCq0_
zo<@e@WGmOdfhX0z-mLG|WIc0c6ri^?*<F_{y+%6=>WOeVEIzZ`LLPM_w0j1H>bfN}
ze&7H#pJ#7_<@A49J8Jy_;&)HUm<corpD~u}yL<W1cuhpp30SzK{i7QYbb+(RUo1^c
z+lff^1p{`^SdqtxV71WiO%A=xG|N;v5Vs&K^S=ATOmrtI^n=>yKO!rrSswcefmMmq
zF63Q8zo*K3kYm>tUViG76od{*KDZQ4D9mv6$kz7qMpFhDs|_%5?8z&`uDTOu9n3!C
z^dyP~?2+__Jrz)n+9f-$Q!padKAn@ZEVT922Q_J=c%muB9g4Y0bB{l5Lu?M>ZZ?Fx
zk#e!SwSnVep99Oz_c>LivW5+C1JC+J6VYq>g6<?^sx?TesM23RyRH>b4DgX}**hF$
zht#4PNbX1lBinux{OWwg>x${}F1${baTU>=8PKhHICm*6;BOmp#H~+C8iFcb1P@Ts
z3=Qh&@JB}zMjsSo(1;(;zQH9F3Cm@iJbQ^H46i%E<*&-;dG%9z?0C93JzDEr02Uo|
z-?AF_ayngBXC{3+wjKjAPNz6TY0HLGx;*l(Gsoh1M~}vrr%Sq{RvJw3nL%;8Kq6Pn
zZ@1WF{APmjU3<`7Tp-BF27ipd?GCTt$MG4zQx7XvwK;mlR?=jaBlSi2)j*Ykp-1@_
zf8B`bNdqB_;Y2FZoG)%-kEs-JxvII{n+kG>{!XP=hrLWEGr)X`4#E<tZ%W?KLv$B9
z^5;3Z4DUf^fqKXlytanMc2w5!6<j_7Kx6gp!5W9z*if4IB8mXRZxpReBgIwPA3ek?
z#~wKl92}zHIHnrzZC7+E>eJNXTYOSMbX%in?2)J@@@!Gnk1j;Unzn3UU{nIXU=KGm
zMP9a)_Oij3v3(n2^h}h0m5!U-&SF^9y{STN&z87_&>z5uFkMbnhj>gX5C`hrz+=nx
zu1t}34^^8^gIsZhpCkDa8VK%;W|!D6d^TXr3ao1K+e#69O`+;$X3T`0%D3*RcvvQE
z+mg|zWR|GGR_@peL>M$wNFN?n-DnSumI)WXIjA3!8AUxm=xejuL_ZJ0ZN<0k`Lm<p
z$_7+O208iUg$ZNxocT_3=>31C8rr$U5?CqX26qDjQ$wzrT~!P-NN9?@NqHeH&zF%%
z-L-^L<kN5zv&jPgo}f*m=?jEm@Db|aBiKtAblMuNH>R$BxJl_F%Hc=H5UgoUA4UyI
zGFpgvTPOkp6scRCZPK&0c}e_$!C$Y;yT9}*%`hgy^o<R#-QgVnE(>yL_nW`i9*y(O
z;(Bay{%rrZHpWY)3&>dA2`%flL3p8Au{GhzjSVe^B4i)QK1f-}KwC9lJ21mh{l#HC
z@s*iiquBLfVH@<-{0mz}qme@I^oH6)0Q+it8mQNguG)oZU!39o`r=V-)XM7Wjz_1+
zDd#4hjph404cS@ZvP%45)$iPk95J=k+^WNyjv3jf!D8Ac;a$p{)!4`qF|2hfrvm4P
zQtG=l8W6u=MWHwGgT!n5oGlOOtOudPpjylUo8AZ)^=SyHh4irYrZA5I1$qQQ**Ke4
za@og31GSH$MO<-;dXT;Qb&QtYl|+@d<MAhz*L}tMb=ZzXK^394gkDL5!4b}|J@PtR
zF{-(-{jlz4VAVWge5nz2O~5Fo+U}S8^sZw<4hW`02N3$I1EZ9&_NJ1w$Tp6U*z|Wx
zVhHfXq65g>qRL>H{WJ1HbG+raT3pW$QARJN1*QsGLv*M_DKWg%L~W`|d4E+U*MTJ?
z6QYZ_4fP@^ySn}sp};oAU8;;`#Es}y!?%Rt?FCA|P+WGMLrxLVq=7xun8suISmSV)
zkld#wyhqA5&8wRS`%`7U?TDeREp&b5^V0TBx7#^_2V&yQ*$U?t`Y%fLqvqGbqz)kI
zN}1R&bX4CgUlIQM{FA=}q|M0H;+TBU)#g!OwlZX_yID-+941z^agj=6k8&gMr=xO_
z$HTb=a~w1a`i3s8ns<0<WrbJuiN~YlOFtP`A6UKRrD@r6oI&G^(pnu)mro4T7(*Xk
zENbTm-B<9ARN(p7I#x|YoyBVl{x2nO1jX)BE)R>m*C-eW<h8IoGG1*)WT<>NX(WS~
ziF*5FnbU&aIl3)4jK?>IhdhS7#v}y_MRIl{1{vNI*6H?>^FTnkO<z;jD1{WYe^<aX
z2!w}0G1AE13&VndR4aRVVIXm&`W{M`Bpl*^>SSh?cpm~|s$|78F6^!1@yohI)19B9
z@o~1Sv;7;Q0C!JQP4*eZUX~H=;o4O(vm^xj=Ilm)ae13atMKPF*SW6AL!}A%#KFd4
z+D`J(m0RuEQjyAJd)Qs+HJe%wNIUd-cf>3r($Mriqxwp~LQ9&xvjf=`Q7#$eCm1P9
zAjl)yjZt}~vO^rgR4i7oTQ)sI%ZURG`qF}DvcHKuGah*5X~-|I%i;$4Cv+P*5w&F@
zUnOQ)_Ak0BJ{ciu`1^yTZFjvcl^f$r*X<>K$cfO$H1?626bvs|B4Mw2POE?`Om`VO
zzh|0jmESpmwfehyL>XHJAMu58{4YP3k0rhkCze4!LE@&UtM_b8Z2~+~0r%4WC0rRB
z+nn=Wydm<048NXU8zmkt)cQ4XwfKk|=G?E=W#)wWcQ#CZrwVH)=yR5K-HjBB*~R3l
zRa?$&+s!h^Iq3NyT*p3chwlwuN}Xvg;dm!78heLFEpW$Pl!!}+F>%UT)8#aAnG(@(
z(ZbN<xihoO31w^h%*kPM9TqJ$H(|nGV8K0QaF*?-&thdn#w~Cpna%TG&;BouSFt??
z78-uUOUoDed^};P=A1E~?fDaCF#qxViwXYP_#YRQO(3xONgT#)>T9QnFvSVuDVtN`
z@PDln<ok_oz*vJ=iK4(Xi(@n8-dI<QW=WvWXXb*-cWxWxhY<YwQHl)7C{#aSlx|_M
zx&<}mL+$iIcP!C(@%Ke%(kI3V0x<#RZc25wu(<jQCL$quZAJUPzDvi`6Y%=1^Rw}`
z&9hRiTI0CdK(w*ejY=FLBK!1}OP~)dEF7P)VKyW}?h@71**pf<fF>)X7!zBI3<R9-
zoEU+XKx17j0`D#?;GwiO5)OlLe%n4k(?FCnMuvfvNz-8y84J<-0aUUv|NILxY|cAN
zi%_GFi=_6v>mulCGCnkRPvVH0!z>ENYA6^e2QKE07p}e&(9d$imfK3>u^J78!=d;%
zLT<ntdplAS+Cenc$pOz`&v?}}xX%p|a&fE{Fb79R59Z^L4=O27ovn;@chC7`{aULD
zv9ByW0UK4?NS`8F2zJiHcH!T=*3b+rg~iyQbQ?C<sFZMNVlzAi4*nJ+#(_F*hiX%-
z#bz9J<=MH57Whq|Y~OhPvb;2|Gt~;xMMC8xb3!kpyZ;<5<$2PZ&Oe@DU^?*&Cfvki
zcWi5Nxo$sWe*5RwJM3n8Bp+Xir=<PMmc9MrR;HD^yF2L2oiY96O9^IJb{v{@t*LAm
z&kRwF1^6tTV1`}!<C2?KmhQ2o-`k4d6!ekzw0azIwS_zSWC_T>^BhC(pu!nA3@2BC
zLzi|gSh|st7mowcXYdyIcu!=pgj6dtl?t+7;C-ZN`YIg%#bdsm#z@}S5Aa>JU!^#*
zla?oeT!u~P)z8Z&99#cbZ?{Wm`{L{Ni!}VRhh}kc@NAG@F$3{@Y_)wHy7~`~Ah&tN
zcz$`raPIV(0o<ew*mfK5U`($KWA-a)I8gd+$hXzsT*>;5(-M_4o~^zu(lIJiWm5B*
zU;T;-r|;=tExZBgwE_|wwe|6y7-W09vfR*GyxH)d&%VEh_2~~dXv+^{di!=?@9#S&
zUCk4h>wr&+845?b_~$nAxBcEWl<JYkKezeU#qd2jN_szO&-&Jui9+=DEyFWsa4)PU
ze)_W1jI-MnrNL<F#$O+$`*~HWToc1QBa=04=O)Pd+t)pTH;4W34%5O6zpj5>lPp)R
zj%O}#&}f~-E|_bg>{)gedM~$bQWAw`K8J!fcusq%`Fj2j-pjW1=S#QYNIH`7Q4`$v
zKM0GD!b`SH7T19C6X@<)<?=hJR(geaZ@fDfedDlbf!9QdAFnKk-Q%tZt@XFFIC~)T
zxxP2u&#Af~C%10g=M3Ea_MTP!&co)u&&qBrG-P?#Vy0CK90%)$<wZYqy8f35%tSxb
z1p#xJb+H%fPV@`shv1Dq{!gP)tO>K7O!oV}e@?{*@eG`H_oq!;Cg1k6w9NaBDHz^q
zhC{=<;{>dwDqhu!1~IAz?$c7JyS+VQ&*AiYh+{YX`^qBfPV92N_#<oD*(diN{2F>Z
zRx?pKjhAoawp|IW9eA(~v$0h}dc5HV8$Fd9HHnDhA>-zpWnCv7b29(yQ`4N^e168<
zHsReE^FR4%f!?JH$Fy;8wZ^<X7p=zE^c&C9jQcr6{v+w(ty1Y(=<s60alF~h79bov
zo0#c>l_%mN^F$N4D9T)uGRX|rq30x$2~fSR(FD3F@qINjT&-xptO?pk7fYiiQ-81O
zPTb^;0r>uyn|XIdX>C_rIag{1@S9|&P%@b;t;fvW9E|x{=8ZLrX;+jd4q=Gft=MrG
zxc{VXD)7JvG9#2>@f@VXI{Y;UaSlgHT%SVOAz1Ou^R{NNy8kq3HZG7W--{ntO;x+i
z0>!TH!p(;@B~KPUKli4&K=qwd+#foz*5!H;aQQ^yNB5gc9Ol=%bm5wc6hDt^UMX%w
zZ6~8=HV)~a`Q3^|M8^~*goCZ2%yCH^Q>T!9&z=1@)B8`^!qF9Yi94TzIDFmXo)b}n
z4VK898w~pp970pBJQq7_5`OIT!An>90V*5-b$H%~X3(h1=-D`J50f#?9}^?KTTZV!
zeU7nb!lCIc!=$g=(w`R6dR;Jjd8TpQBOJ)K7UE#5e7q-JT`3-Oz_IOTot<VJc{nWM
z<Uxb`6LEg2&7kyazq0c2o^&G~;UkL=uP*{3n3i^Uv|SdbeV>W9EpteMJ#S9C8$214
zZO6Ah|FIR9aZP2$#x{Zi0W@nP2E#-Gms`7d-GNEpwHKdp+;&*@X`Snzv!+Ypd(J4D
zNazv0xdYjAOo>9cSEBA>XY0)b<dr7pmQPA(dk2jeq!!V-Pde5vtMN}T%Jb}y(HZA$
zFHOdJsiEuTIM+z&0m=J+x?jLhz25z4?%_$)MOhoZ7b2lhNKatac_r84=0-R!fSkWX
zJpI>G^B-2Ybo$J`K~nd1V0H=`KIg_rO*MLXb`B0^FZNH)IC$>t1SdA)ttxM>JAtRX
z%)&h(E-N>EGUFTmnZx&M$Lr_cb^hL}!sEFQCRq&LqrfL@LhRzLwZeUt$7USVcg3!n
zl%P-U%Blp02eBk0$qcQVF2zRsC}yvcjj8O+{@}m+y&M#U7V<8`C2UBH!~WCe<Nlkx
zSI<{sd0m4qC3=i)zZKZ@9|p2b9IS(@B4ZNObTJ=Ekq$)D^>$TK1i?PSz3=BhC*7)8
zMpS3j3GPEGBe9=>kxVjqV8y?C&mcps#A;MqgYhH5)*NI7)PlU$$9bqTa{VWWUAYn)
zYX;hJSbHQ}mBqENIP@vP0#H#%?YdSx?DDTUGx{EE6g!IbvJps3!uCP}y4)GLsVF0v
zqVS1+V~hl%hxKllE?GfBY5@5{Pa~E`YWZRNII_SWDirtU@!zQk%M=l&IKdp!oDzpA
zBK#w`J;e!AoG`@+Z|{1h#9>Mtro>@N9NxYEoze+YI$=sD{6EqO{Y)z}T>N_$_>cEH
z@elK73yWh9{`buOkB@WkE~Dn${(pR$`sWmdrZ{0r9Hw-F#gt;i6~Pn{rikzldoaZb
rQ=Blx33$5l6cPTvB!a;HD7N6@!Vg}ljt8RVJ@(nOZDZm3pO60!xLi&F


From 5432fa2d1ae9c84b018aad097845612205b2c3d5 Mon Sep 17 00:00:00 2001
From: KP <109694228@qq.com>
Date: Tue, 22 Mar 2022 16:14:36 +0800
Subject: [PATCH 042/126] Build docs for PaddleAudio.

---
 paddleaudio/docs/Makefile                     |  19 ++
 paddleaudio/docs/README.md                    |  18 ++
 paddleaudio/docs/images/paddle.png            | Bin 0 -> 5043 bytes
 paddleaudio/docs/make.bat                     |  35 ++++
 paddleaudio/docs/source/_static/custom.css    |   5 +
 .../docs/source/_templates/module.rst_t       |   9 +
 .../docs/source/_templates/package.rst_t      |  57 +++++
 paddleaudio/docs/source/_templates/toc.rst_t  |   8 +
 paddleaudio/docs/source/conf.py               | 196 ++++++++++++++++++
 paddleaudio/docs/source/index.rst             |  22 ++
 paddleaudio/paddleaudio/metric/dtw.py         |  14 +-
 paddleaudio/paddleaudio/metric/mcd.py         |  33 ++-
 12 files changed, 401 insertions(+), 15 deletions(-)
 create mode 100644 paddleaudio/docs/Makefile
 create mode 100644 paddleaudio/docs/README.md
 create mode 100644 paddleaudio/docs/images/paddle.png
 create mode 100644 paddleaudio/docs/make.bat
 create mode 100644 paddleaudio/docs/source/_static/custom.css
 create mode 100644 paddleaudio/docs/source/_templates/module.rst_t
 create mode 100644 paddleaudio/docs/source/_templates/package.rst_t
 create mode 100644 paddleaudio/docs/source/_templates/toc.rst_t
 create mode 100644 paddleaudio/docs/source/conf.py
 create mode 100644 paddleaudio/docs/source/index.rst

diff --git a/paddleaudio/docs/Makefile b/paddleaudio/docs/Makefile
new file mode 100644
index 00000000..69fe55ec
--- /dev/null
+++ b/paddleaudio/docs/Makefile
@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
diff --git a/paddleaudio/docs/README.md b/paddleaudio/docs/README.md
new file mode 100644
index 00000000..d53f0be7
--- /dev/null
+++ b/paddleaudio/docs/README.md
@@ -0,0 +1,18 @@
+# Build docs for PaddleAudio
+
+## 1. Install
+
+`pip install Sphinx`
+`pip install sphinx_rtd_theme`
+
+
+## 2. Generate API docs
+
+Exclude `paddleaudio.utils`
+
+`sphinx-apidoc -fMeT -o source ../paddleaudio ../paddleaudio/utils --templatedir source/_templates`
+
+
+## 3. Build
+
+`sphinx-build source _html`
\ No newline at end of file
diff --git a/paddleaudio/docs/images/paddle.png b/paddleaudio/docs/images/paddle.png
new file mode 100644
index 0000000000000000000000000000000000000000..bc1135abfab7aa48f29392da4bca614f688314af
GIT binary patch
literal 5043
zcmV;k6HM%hP)<h;3K|Lk000e1NJLTq007ni002S=1^@s6D-<Yu00001b5ch_0Itp)
z=>Px|ZAnByRCodHoe8iVMY+cbn;1d107fAM5|&a3LJ*P7Ld`R*M!^N#5=2ErP!zR<
zA_!7S^daytPsEDif&wLr7zGqC2&iDt(hzpCh!7y4kbpu=uD}1`&Y@4AUS`hpOwPT|
zSM|^Ie0%py&-8Tnxf!v>Fr<Njft{(=USJ$}63hqJj2JQEd5(Qa<UE4J-UN05uLO(0
z9proq$A+2~SbZ(P&AATv23Q4bnh!%aZQl`jZ3%9&6COOjlH<mGmvocRe@_c=1D@`=
zIWzV`3LErOi;&Nn;N~nr&+(Fe%DKtqzo-ScA=d{hJU6Dv{!%|R3yB?SCKo8bwx9BD
zatXJ<nql#fl8z$VwR2>=Zw{S0Rd&xAY^R)hjanbHz;NnVFVESY<8$iOrM7j>K$_5K
zXo$4HaOzl3<!tkNa_ZHkwjgJqhjQvQYJJcG!>MCEPJ8B=PM^~U?QW)Un|Tr_|2ZIQ
zL`@4+*8+U_e<ASkFVTHqWOW%7_e?$`!1W+R7Zvy1STrq=s0Fw)*8*pOm5#e~F1#HR
zg-Ys%ytV+paO5o34?tfn5hShrFm<NY{$WSlh}%)AJ%PFN;zo8ctnBB?lJ{QZHvyPC
zFaFriIXAgAEzq<;(*m`%z_5H{)z*4Vpq^WRx3kt9?fY*fql#6wQ*_s{dFZXMdA*+F
z|7F<P!|efLJld;IyJEfiY3%yA1-NnELVBOAC`H|TALk}~I<J57(W1C!L+$(TeEfXE
z^n>nJoG)_deg@tDHFZ@+|05huw)o#i_Y{lQIz@k73U>l~eN&sZ4{+J!;pwTdT8IiM
z_F949&r`7hpB($0k!@nh|Hb)Yuq0$_+2lSYrfvj1J?$GJ^P#7lcgi38m!h~wraX95
z9km000k+RrvQ($KX(pv|X8BQlPq28gQ|Y_f@CUXbZSq&zZ8rRaE!k=CFG2p0#ovtc
zF&0nXtUj87K<#7GZ}D%(v~|pLUB~(4lXqRxv`tec;myE5L5TDb?<9;~0sa**eiqqm
z@F9?QM}{+&$;8*GAViNMblos5H^_7|gh-o&&jtIKam$wRcT6Z~N!f)(@*~!0@Vq1D
zAAEk$-3_>Yv2Zo~j<s0MPX;#4vVEhVWscbOwEe`+m&&gltG083$9ADVjLzD=gphp+
z<Rw-1MDP-yzdU^G*8}P}A+m=cZPysd{;`m;CA*Px`|Z~DSEPWSsydc#_8ki5;=A@!
zLw<<!ZQ!EwOgj{!9Sfatz_+~Nicy5OdO8*vjs&{dL-aYbvCJ|?vVS6E?2+9L-yR~E
zu7hQ4KkD(Dv7ez}4^7)XbJ%Bo%%j%hP64GZ7WP$jtjpUK%uyf@UAxDDO?u!2AAHF5
z>jf?FyXbtF`c~36mbYUe(_2dGg8v3p&<dQku9f$#kfWV?Rhf$6rUB#S$Sw%Try=v+
zzyUjbFl&)`664@M1@yW^gN=1!m!Oj|3%~kS>Y_pWPjw063=MP602A<W1pa$nhttRT
zCGl`w8rr3z?+jnZKC8<j%+SQbD2K{S>h_$2-)13St~P*iE3jt?KMrp%<fn4%7#ebJ
z(|c`3!t<r%*_s#dHCVz}FS?@Qs`R};3%5S{I>|Y2;+*zAzZriCg*OmmVaA9Zi8}?I
zD*}4D;kU=<gDxHH4~2h~OICn)0sfCEpkG>*ymfy(23(!tCh~b>-XQqe2wh!!O)i-9
z3@2OT>j{(TE8|D^M%Qehp|8o5*4xYAeFy(X4c2Fs!Ox0!QJ9yV<T$R6&Rc=qiEks$
z>vPzrRiR>sfe`6cy0tGVvMC|s0y4e1LNk&bDWDtsh3};u2G$AoH-Z$Tw{>kq!l55w
z9#|w1e0|T;0ADNON4og=qQ{qyJ!{ZTLqU`Dji_OyW@`%(HkxR%Db(Q$`7fkQ7Mj8g
z(ZwN$7>)vaSa?0>`+>v3*T91y4}CHeE0f`9&m7KzlML9u2H1IP=rRd>1nBQt#AZoR
zX?>fz-X2-HNsq+4NkJX-7kIfxJLrDJ;X^FV?2hgwY3z;%uGvM{U1WDGbtp|<>}tm5
zd@R_&<hjb|90FXVkx9Ko7W=p|v3R?s$O@nP0@oA84EJ|VqE9MxeqtIq%5xIstdH|E
zfG&E7W~7K0^Fac75cuf*F?-1|UKgC{a{~XIJbrvS*z4h>P97t(V`)}$Thk}!F9Fkm
z9&YMz-nj>4Yl3;eCf!dOBGyX;)l)e{*Qd;gKzh6Rp14^a9}LArNz<~fo${Hzj)#PN
z6qVsm!>`cWOEYNpH$>4eQGH@u3S6`pragJk_J>T%3cCW$W<CIXBtOx&L))>ZfRDAn
zQQ-H$N3D)!yFgh(!y|#6=MUl8-%l-b_{Rbl)$+jWr~_GS4`NT^$sI-sPKdb`&|<NV
zt_y|1Xr|9_kbf7)+(V*JEK`Qh(<5;YGWAoJiM<W*O-Z+g(8wnOJ-P23gl<l)^Q{Ci
z(hzR{3i93?zpmk-uVZbBzTPx0x(w!eW{#X62s961lXzR1y0w%EdV!sZb*>AP{03F!
z@Ef{G|KW=ZUyZJ{sSq7qMFSG>Wk8GMKGGC?Wkk^b^8|hperveH@oJAwjI9T3uAyX+
z3gUA-xX%;Z*|Fpw{eyw;p?@o~?FRKU?DX9aUoDbr5?zwn9E658hzF)73FdHu?}-W!
zpz#hfHTbCk8;O>H;G(9>Z(bh0zpo%3nw{5AQ9a^q0+xH?o-TjLw8f|wT87bkisls4
zmrtNh?;~%=8mv#~TI=4`weW4By8~I}L-VdSAZX}~fq39=RVIe-4HP6m6YZqY)pvrv
zWkTfLzGVCB<($AL2ZG<+i_bhZfw&q{x7MIIUJ=O6Cz^v`$2>XY)xn=0+8VOODA=(S
z4+k@1P8@Q*!3u5Z%3HUf9zjQfrH+J5mUsJ7(0KtN!8GLak*i~s+P+jEHLohkPW2@W
zzmkrHU(Jp^;)(kWC5vnV@C}`8)2Crv4)*p|v<CtHHUsJizBenP4vKcH!MdG<Ri1A|
zIe!j#r`Ol+p<YEyrlDMB`%;luPF*UmZ-Nb-!WWpUOd0x`4RY@MF!a3Jmkgb<#a9L#
zbmSb+D=K;)2Cr}!t%R2?e(Fyi+~Ckz2JZ==ZC#ImTfkg)BFMHhaa}YMsR@t6fbtv(
zk`TVNHn=<35<CW6Z2-GX%PDD#+0WXXx(d2M!TlgaXIGsM!4D54pqVfIdbCZPPYLKt
zB%6+PkhXEc0x^E0U>^Er!cZ+&#L?9Ad{4X|r;b;|A2}TjLZgl>Z4KYA%}$JC>y5>^
z>UaXs=gEcOI<N>7p?gq%O_BVQ>Jt30NVb}y^lo2DkqrJd6jej`=ngv$ilw$M+5UPt
zCx=+u1=rttjCqi+k>Dp;xbGz|*%1G{KyI06Hia#K>)X`qSebs&qo(n-*xfM&@2gXl
zL&1}LK?nJl@+w^v(J3qbIu5oEIQg|HqtP_XiG2?g-rHuzh?U7f!;#SL{sH83Vt#s%
zf5v0;E3Q-P0Yq-cq4O4p-v6U>t%F|&OT*7oi9b`p$T4-y=WzI4FoPHmrgmP)AC=n&
z#Hg~TAc4n#Ya$6~NY}B;p7UC^FZmji_(y=pL5RxklJ5#nu-yXz+Z_p=$sWB0@LTI!
zUCt9Q@a=kIX;SP9pkwz2=c(RUx8duPV2fU=(0*g#U<3}{2DH7uEKQ3zxFLHAN{`qY
zvrdTE#Kg*96D8UZI|alp^yTF60D3JVhw&6-7ozst3Vls-915cQ4MPr*X)SCFI1uFd
z$w3ucXMArf{g0BrBCi%v*28KpCld!**<maN%^D^jqGkb%<E@l00vD5yCJ|E5(>T#j
zT2!|=KtGB38^`iQgS`vXMU%6P38*!r*iYV5=k*j0?YGs;NRW>0OXyCE=vN`WIBU05
zN0J;1^!9eqyX`nA?dMV)<oP<N?WWK<zIAV&2=wp^5<2=m{iTIZ$OxV$FLnVtf^EQP
z&@p)xbl!`Q0KWru&|l2_2pC&4sv@V38-@U*z_XD+M`wAXrwcSMqMurPFAh6~Jy||w
zj`%#slc@IoIe)VT|3@Aht#2J)CZ8&8$!#N`j%TB_@SQw1f%XP<(bTL01k?>0nzgGU
zr>=c=1eqQQv;=v3`x4~?Bl@YuTD09MYqwNKeCkG9=1KNgUmWx|28%s*Vr%xy*(rTK
z>)?~q0ZPt6N2HFg?-IQtFX!;~)Xp9L9)`CK<5&HBT5<JFccscT&?;h_K<x9)SmdDT
z)5Re4)H(&bGc!AEZlJt>c3$yOWB<v-UywQu_4qtKzwGe)I6VDa{+|w6i-qle6x0sB
z4)hyj@5RSoTLvZmA=bH=Pb%ReGu1cv=GlpHTuNDWE=%*M@4H$~NO~N|yNv_YQuH}9
zb3BC>K}!X)jOTOyW;1qHh94V>YCqT}hW<p$cU7mlCf_n+XE~%e;#WUH^zEQOB~J|O
zKL9lR^O4%_ET9{@XTtvHz5sp3fAKSi#LllNISo15Gmez^tRe2*#5f+bXFtuW3tziK
zVnfmMEF0hV1+J3Nb9lK5bo^}G%ppY?|5H8LE{ETKki=r4tM-kNCX2oev<TfEQpbE}
za%_fN$I=bAGq^jeM2|-e2eymYUdU_mABIkfxQ4qmv?uKmplQ<k!Bxb2@d*OML7FZP
zEg;M1IhNh4I4|2?rb|s3KLnKb@4-*W`zel9)2BTFbe*2`$hz3*YK`d`1e$L5b()tW
z`_LfNasNnJ!|z2v^<Dw4#g}-)f?D5LWotr19gdI70)GCDY+S!J9C_bNUZI^9myz=e
zW9C5RhMI_aA3)AOcjT?>v@LZm{9T04*J1k;u$9Aa9@bRiUgT25_fO#;;*ure=|N^b
z^m}#tXX>DbbsD1Lv8nnWriql!3pwW*itdpvV*hxe?$-kseXmTGI2;AO30AnAM|j@F
zW$a!GHc5c@dvZ9kOuZ|yCDDC7$(rolDhXb~DaozY?Mo$6FJ(%-cQr4JQPw0xk|}-s
z>;UvTtRm#?Sfwg#7?aFoPeRG3+npcvMlN}ZvDNmD?MrzaR;@$hjwi;HNSyZerHI~8
z6Z@{$FBrzmOHmp8+=HKY(9Sz@)MtXQ3#gQcH<|!?>#l1B(l^|}7O(3R`zv7D6E|9>
zIgUEr*=0}_gGBUX3Qe~n9%yf0sw$77e#vR35(`kr_NAi!`&FO%jsd=5+8e9Po4DJ7
zW5L-#v!3(7vmm3=Zw(yjrz0+!rMe*qzOj&h(Pa#8E?B3^_UZ6VFs-wMC^jqC;jfMp
zlGS&Bc|l$IA6-W-b;qEhGhCI0r&+q%iMZN%qwgo7)I=Kg`Z1)7^v7VvsB#=*9ZP>7
z<uR8Nc&FFWKm^(xDCrPYK5i~rS(2R|N*LJ^;F~rs8m_POsn<eJ-8&ZU`zq-i$xx?5
zfbU`1t{XbRj-^oO90^>cjmp(Oa0zmTE4sm}^sg`FXzwl+a$67ho=92hCwCbR0|u3w
zkX6ueJZPv^k-B^s_;%;tj!bK;EutLlSW-k!LscJ5f5BS?{}OQ|SnYp(sc6;6IktUC
zi<kDtYtb-$sZYJ$1AKH^UnU($ii+q#ajCCD@b!byv8D^o^L8vv?cWc4bRB}w`sW(G
z&I%qy!kd~&Tjgb(UtsZ8>nRPfCJCGvNurLwz7$EJGV!Ydo~P43W0Btq7K2rwZqnlF
z5?@gA(>K^SpoM5H+P@9>zRR@9`yKdO$gwbC_ayPOApKjgp~Jx>c$3h%k)sy1ZwGf{
z*L%C+Y%TBr2+`RD>MJ7IDWQasX*jV>UnQL|8R|AIQvcO0KFL>I5<3(4{>HkUw>JL1
z)#VF*t~XX@i@r8o$Np3TDlO7p*jbsap5jI6?HFtuFw(Xddw*R(y*82e!VPAipXx8D
zEt~oS{i*(t$s%e)@<utl>8Bt6!*#qihi+5_KXXyq59GNo)<;io(-!s8v0^u<{`!)J
z6MaKGN!~y*dqooZYD7My#at6jsoyfBD-llqYQ#HH&!lz4-(C3K1$-wQzEfJLV{wD-
zi+ODSTtDtu@a;CpT0?(4MC0V)PWm=^aF|u{4(g-tsY8MOifwYj?=o-{j^7M^ohLw{
zj)lY6+v<7}(37!mh=)O*?MuV@ZIxXNuFb=gcLvjep2S0ywEO;BK*PR?;EFmbLL6<+
z+n-n({hSN#^TqYes&wgdgYE!R7b9eI0D-URGG2`g@}5cfj|EcDZ>wHS-U9)hVJV_*
zHlt#r!N#E18RArJqo1`u>T*V&+UKWP=!d{U^d4~O#d!GH2c0V&dLP8+YzIHe!si&2
zwgp>)mH7El)W}r8L9>0yY>=khv_RPwpi^xDn(a$v8?v!)TEJ`pI#siM$!w6O+_XU1
z7NApU`_dC2MD2}^W#gr?C)r~{2_qY}+m}+cVdJO57T^XvE^>oDjeNrjQz-13eDv3s
z?v3PqUSY?LLDK>$T7Vm~*}jybAsZh}3z#iH$9iYvsjpw7YPzwnh@5VT<b6qBCEO%4
z>{=kUcA*{7q51te%-b*Sr&gguX`O9pAn#SkP6G`!El^nt{0HuA9HmP~`H}zt002ov
JPDHLkV1n`1<Zu80

literal 0
HcmV?d00001

diff --git a/paddleaudio/docs/make.bat b/paddleaudio/docs/make.bat
new file mode 100644
index 00000000..543c6b13
--- /dev/null
+++ b/paddleaudio/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd
diff --git a/paddleaudio/docs/source/_static/custom.css b/paddleaudio/docs/source/_static/custom.css
new file mode 100644
index 00000000..bb65c51a
--- /dev/null
+++ b/paddleaudio/docs/source/_static/custom.css
@@ -0,0 +1,5 @@
+.wy-nav-content {
+    max-width: 80%;
+}
+.table table{ background:#b9b9b9} 
+.table table td{ background:#FFF; } 
diff --git a/paddleaudio/docs/source/_templates/module.rst_t b/paddleaudio/docs/source/_templates/module.rst_t
new file mode 100644
index 00000000..d9a50e6b
--- /dev/null
+++ b/paddleaudio/docs/source/_templates/module.rst_t
@@ -0,0 +1,9 @@
+{%- if show_headings %}
+{{- basename | e | heading }}
+
+{% endif -%}
+.. automodule:: {{ qualname }}
+{%- for option in automodule_options %}
+   :{{ option }}:
+{%- endfor %}
+
diff --git a/paddleaudio/docs/source/_templates/package.rst_t b/paddleaudio/docs/source/_templates/package.rst_t
new file mode 100644
index 00000000..7239c11b
--- /dev/null
+++ b/paddleaudio/docs/source/_templates/package.rst_t
@@ -0,0 +1,57 @@
+{%- macro automodule(modname, options) -%}
+.. automodule:: {{ modname }}
+{%- for option in options %}
+   :{{ option }}:
+{%- endfor %}
+{%- endmacro %}
+
+{%- macro toctree(docnames) -%}
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}
+{%- endmacro %}
+
+{%- if is_namespace %}
+{{- [pkgname, "namespace"] | join(" ") | e | heading }}
+{% else %}
+{{- pkgname | e | heading }}
+{% endif %}
+
+{%- if is_namespace %}
+.. py:module:: {{ pkgname }}
+{% endif %}
+
+{%- if modulefirst and not is_namespace %}
+{{ automodule(pkgname, automodule_options) }}
+{% endif %}
+
+{%- if subpackages %}
+Subpackages
+-----------
+
+{{ toctree(subpackages) }}
+{% endif %}
+
+{%- if submodules %}
+Submodules
+----------
+{% if separatemodules %}
+{{ toctree(submodules) }}
+{% else %}
+{%- for submodule in submodules %}
+{% if show_headings %}
+{{- submodule | e | heading(2) }}
+{% endif %}
+{{ automodule(submodule, automodule_options) }}
+{% endfor %}
+{%- endif %}
+{%- endif %}
+
+{%- if not modulefirst and not is_namespace %}
+Module contents
+---------------
+
+{{ automodule(pkgname, automodule_options) }}
+{% endif %}
diff --git a/paddleaudio/docs/source/_templates/toc.rst_t b/paddleaudio/docs/source/_templates/toc.rst_t
new file mode 100644
index 00000000..f0877eeb
--- /dev/null
+++ b/paddleaudio/docs/source/_templates/toc.rst_t
@@ -0,0 +1,8 @@
+{{ header | heading }}
+
+.. toctree::
+   :maxdepth: {{ maxdepth }}
+{% for docname in docnames %}
+   {{ docname }}
+{%- endfor %}
+
diff --git a/paddleaudio/docs/source/conf.py b/paddleaudio/docs/source/conf.py
new file mode 100644
index 00000000..4efe85b0
--- /dev/null
+++ b/paddleaudio/docs/source/conf.py
@@ -0,0 +1,196 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../..'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'PaddleAudio'
+copyright = '2022, PaddlePaddle'
+author = 'PaddlePaddle'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = '0.2.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.napoleon',
+]
+
+napoleon_google_docstring = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+
+import sphinx_rtd_theme
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+smartquotes = False
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+html_logo = '../images/paddle.png'
+html_css_files = [
+    'custom.css',
+]
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'PaddleAudiodoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'PaddleAudio.tex', 'PaddleAudio Documentation',
+     'PaddlePaddle', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'paddleaudio', 'PaddleAudio Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'PaddleAudio', 'PaddleAudio Documentation',
+     author, 'PaddleAudio', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/paddleaudio/docs/source/index.rst b/paddleaudio/docs/source/index.rst
new file mode 100644
index 00000000..26963308
--- /dev/null
+++ b/paddleaudio/docs/source/index.rst
@@ -0,0 +1,22 @@
+.. PaddleAudio documentation master file, created by
+   sphinx-quickstart on Tue Mar 22 15:57:16 2022.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to PaddleAudio's documentation!
+=======================================
+
+.. toctree::
+   :maxdepth: 1
+
+   Index <self>
+
+
+API References
+--------------
+
+.. toctree::
+   :maxdepth: 2
+   :titlesonly:
+
+   paddleaudio
\ No newline at end of file
diff --git a/paddleaudio/paddleaudio/metric/dtw.py b/paddleaudio/paddleaudio/metric/dtw.py
index c4dc7a28..662e4506 100644
--- a/paddleaudio/paddleaudio/metric/dtw.py
+++ b/paddleaudio/paddleaudio/metric/dtw.py
@@ -24,11 +24,15 @@ def dtw_distance(xs: np.ndarray, ys: np.ndarray) -> float:
     This function keeps a compact matrix, not the full warping paths matrix.
     Uses dynamic programming to compute:
 
-    wps[i, j] = (s1[i]-s2[j])**2 + min(
-                    wps[i-1, j  ] + penalty,  // vertical   / insertion / expansion
-                    wps[i  , j-1] + penalty,  // horizontal / deletion  / compression
-                    wps[i-1, j-1])            // diagonal   / match
-    dtw = sqrt(wps[-1, -1])
+    Examples:
+        .. code-block:: python
+
+            wps[i, j] = (s1[i]-s2[j])**2 + min(
+                            wps[i-1, j  ] + penalty,  // vertical   / insertion / expansion
+                            wps[i  , j-1] + penalty,  // horizontal / deletion  / compression
+                            wps[i-1, j-1])            // diagonal   / match
+
+            dtw = sqrt(wps[-1, -1])
 
     Args:
         xs (np.ndarray): ref sequence, [T,D]
diff --git a/paddleaudio/paddleaudio/metric/mcd.py b/paddleaudio/paddleaudio/metric/mcd.py
index 465cd5a4..d1852b4b 100644
--- a/paddleaudio/paddleaudio/metric/mcd.py
+++ b/paddleaudio/paddleaudio/metric/mcd.py
@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import Callable
+
 import mcd.metrics_fast as mt
 import numpy as np
 from mcd import dtw
@@ -20,29 +22,40 @@ __all__ = [
 ]
 
 
-def mcd_distance(xs: np.ndarray, ys: np.ndarray, cost_fn=mt.logSpecDbDist):
+def mcd_distance(xs: np.ndarray, ys: np.ndarray, cost_fn: Callable=mt.logSpecDbDist) -> float:
     """Mel cepstral distortion (MCD), dtw distance.
 
     Dynamic Time Warping.
     Uses dynamic programming to compute:
-    wps[i, j] = cost_fn(xs[i], ys[j]) + min(
-                    wps[i-1, j  ],  // vertical   / insertion / expansion
-                    wps[i  , j-1],  // horizontal / deletion  / compression
-                    wps[i-1, j-1])  // diagonal   / match
-    dtw = sqrt(wps[-1, -1])
+
+    Examples:
+        .. code-block:: python
+
+            wps[i, j] = cost_fn(xs[i], ys[j]) + min(
+                            wps[i-1, j  ],  // vertical   / insertion / expansion
+                            wps[i  , j-1],  // horizontal / deletion  / compression
+                            wps[i-1, j-1])  // diagonal   / match
+
+            dtw = sqrt(wps[-1, -1])
 
     Cost Function:
-    logSpecDbConst = 10.0 / math.log(10.0) * math.sqrt(2.0)
-    def logSpecDbDist(x, y):
-        diff = x - y
-        return logSpecDbConst * math.sqrt(np.inner(diff, diff))
+    Examples:
+        .. code-block:: python
+
+            logSpecDbConst = 10.0 / math.log(10.0) * math.sqrt(2.0)
+
+            def logSpecDbDist(x, y):
+                diff = x - y
+                return logSpecDbConst * math.sqrt(np.inner(diff, diff))
 
     Args:
         xs (np.ndarray): ref sequence, [T,D]
         ys (np.ndarray): hyp sequence, [T,D]
+        cost_fn (Callable, optional): Cost function. Defaults to mt.logSpecDbDist.
 
     Returns:
         float: dtw distance
     """
+
     min_cost, path = dtw.dtw(xs, ys, cost_fn)
     return min_cost

From 7512b3079151b7edf455ac012326888e05677183 Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Tue, 22 Mar 2022 19:37:57 +0800
Subject: [PATCH 043/126] update the release_models.md, test=doc

---
 docs/source/released_model.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/released_model.md b/docs/source/released_model.md
index 62986da0..05ed59a0 100644
--- a/docs/source/released_model.md
+++ b/docs/source/released_model.md
@@ -8,7 +8,7 @@ Acoustic Model | Training Data | Token-based | Size | Descriptions | CER | WER |
 :-------------:| :------------:| :-----: | -----: | :-----: |:-----:| :-----:  | :-----:  | :-----: 
 [Ds2 Online Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_online_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 345 MB  | 2 Conv + 5 LSTM layers with only forward direction | 0.080 |-| 151 h | [D2 Online Aishell ASR0](../../examples/aishell/asr0) 
 [Ds2 Offline Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_aishell_ckpt_0.1.1.model.tar.gz)| Aishell Dataset | Char-based | 306 MB | 2 Conv + 3 bidirectional GRU layers| 0.064 |-| 151 h | [Ds2 Offline Aishell ASR0](../../examples/aishell/asr0) 
-[Conformer Offline Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_conformer_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 284 MB  | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.056 |-| 151 h | [Conformer Offline Aishell ASR1](../../examples/aishell/asr1) 
+[Conformer Offline Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_conformer_aishell_ckpt_0.1.2.model.tar.gz) | Aishell Dataset | Char-based | 177 MB  | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0483 |-| 151 h | [Conformer Offline Aishell ASR1](../../examples/aishell/asr1) 
 [Transformer Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_transformer_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 128 MB | Encoder:Transformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0523 || 151 h | [Transformer  Aishell ASR1](../../examples/aishell/asr1) 
 [Ds2 Offline Librispeech ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr0/asr0_deepspeech2_librispeech_ckpt_0.1.1.model.tar.gz)| Librispeech Dataset | Char-based | 518 MB | 2 Conv + 3 bidirectional LSTM layers| - |0.0725| 960 h | [Ds2 Offline Librispeech ASR0](../../examples/librispeech/asr0) 
 [Conformer Librispeech ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr1/asr1_conformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 191 MB | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring |-| 0.0337 | 960 h | [Conformer Librispeech ASR1](../../examples/librispeech/asr1) 

From 5d3c760eae17fa6ae4eba3e8864b3567c49f4487 Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Tue, 22 Mar 2022 20:26:16 +0800
Subject: [PATCH 044/126] Update RESULTS.md

---
 examples/aishell/asr1/RESULTS.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/aishell/asr1/RESULTS.md b/examples/aishell/asr1/RESULTS.md
index b68d6992..b00f4f12 100644
--- a/examples/aishell/asr1/RESULTS.md
+++ b/examples/aishell/asr1/RESULTS.md
@@ -4,10 +4,10 @@
 
 | Model | Params | Config | Augmentation| Test set | Decode method | Loss | CER |  
 | --- | --- | --- | --- | --- | --- | --- | --- |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug + shift | test | attention | - | 0.059858 |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug + shift | test | ctc_greedy_search | - | 0.062311 |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug + shift | test | ctc_prefix_beam_search | - | 0.062196 |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug + shift | test | attention_rescoring | - | 0.054694 |  
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention | - | 0.0548 |  
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | ctc_greedy_search | - | 0.05127 |  
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug| test | ctc_prefix_beam_search | - | 0.05131 |  
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention_rescoring | - | 0.04829 |  
 
 
 ## Chunk Conformer

From 59b3de6a6db154c29088b807cb5f660ce7d25720 Mon Sep 17 00:00:00 2001
From: zhangkeliang <zhangkeliang@baidu.com>
Date: Sat, 8 Jan 2022 00:49:28 +0800
Subject: [PATCH 045/126] [NPU] test TransformerTTS with NPU

---
 paddlespeech/t2s/exps/transformer_tts/train.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/paddlespeech/t2s/exps/transformer_tts/train.py b/paddlespeech/t2s/exps/transformer_tts/train.py
index 8695c06a..9b1ab76b 100644
--- a/paddlespeech/t2s/exps/transformer_tts/train.py
+++ b/paddlespeech/t2s/exps/transformer_tts/train.py
@@ -42,10 +42,12 @@ from paddlespeech.t2s.training.trainer import Trainer
 def train_sp(args, config):
     # decides device type and whether to run in parallel
     # setup running environment correctly
-    if (not paddle.is_compiled_with_cuda()) or args.ngpu == 0:
-        paddle.set_device("cpu")
-    else:
+    if paddle.is_compiled_with_cuda() and args.ngpu > 0:
         paddle.set_device("gpu")
+    elif paddle.is_compiled_with_npu() and args.ngpu > 0:
+        paddle.set_device("npu")
+    else:
+        paddle.set_device("cpu")
     world_size = paddle.distributed.get_world_size()
     if world_size > 1:
         paddle.distributed.init_parallel_env()

From 5221c2797f0e27f0e92893c7b2864f064a3174e3 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 23 Mar 2022 15:01:00 +0800
Subject: [PATCH 046/126] add voxceleb dataset and trial info, test=doc

---
 examples/voxceleb/README.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/examples/voxceleb/README.md b/examples/voxceleb/README.md
index fc847cd8..a2e58e00 100644
--- a/examples/voxceleb/README.md
+++ b/examples/voxceleb/README.md
@@ -26,3 +26,31 @@ ffmpeg -y -i %s -ac 1 -vn -acodec pcm_s16le -ar 16000 %s
 You can do the conversion using ffmpeg  https://gist.github.com/seungwonpark/4f273739beef2691cd53b5c39629d830). This operation might take several hours and should be only once.
 
 3. Put all the wav files in a folder called `wav`. You should have something like `voxceleb2/wav/id*/*.wav` (e.g, `voxceleb2/wav/id00012/21Uxsk56VDQ/00001.wav`)
+
+
+## voxceleb dataset summary
+
+
+|dataset | vox1 - dev | vox1 - test |vox2 - dev| vox2 - test|
+|---------|-----------|------------|-----------|----------|
+|spks    |  1211       |40     |      5994        | 118|
+|utts     | 148642    | 4874   | 1092009     |36273|
+| time(h) | 340.4 | 11.2  | 2360.2  |79.9 |
+
+
+## trial summary
+
+| trial     | filename |  nums | positive | negative |
+|--------|-----------|--------|-------|------|
+| VoxCeleb1 | veri_test.txt | 37720 | 18860 | 18860 | 
+| VoxCeleb1(cleaned) | veri_test2.txt | 37611 | 18802 | 18809 |
+| VoxCeleb1-H | list_test_hard.txt | 552536 | 276270 | 276266 |
+|VoxCeleb1-H(cleaned) |list_test_hard2.txt | 550894 | 275488 | 275406 |
+|VoxCeleb1-E | list_test_all.txt | 581480 | 290743 | 290737 | 
+|VoxCeleb1-E(cleaned) | list_test_all2.txt |579818 |289921 |289897 |
+
+
+
+
+
+

From 13ac21b7052174ecf80fdfb55c7743e1c5b00157 Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Wed, 23 Mar 2022 15:07:44 +0800
Subject: [PATCH 047/126] Update RESULTS.md

---
 examples/aishell/asr1/RESULTS.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/aishell/asr1/RESULTS.md b/examples/aishell/asr1/RESULTS.md
index b00f4f12..d3f9743d 100644
--- a/examples/aishell/asr1/RESULTS.md
+++ b/examples/aishell/asr1/RESULTS.md
@@ -2,12 +2,12 @@
 
 ## Conformer
 
-| Model | Params | Config | Augmentation| Test set | Decode method | Loss | CER |  
-| --- | --- | --- | --- | --- | --- | --- | --- |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention | - | 0.0548 |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | ctc_greedy_search | - | 0.05127 |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug| test | ctc_prefix_beam_search | - | 0.05131 |  
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention_rescoring | - | 0.04829 |  
+| Model | Params | Config | Augmentation| Test set | Decode method | Loss | CER |  paddle version
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention | - | 0.0548 |  2.2.2
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | ctc_greedy_search | - | 0.05127 | 2.2.2 
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug| test | ctc_prefix_beam_search | - | 0.05131 |  2.2.2
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention_rescoring | - | 0.04829 |  2.2.2
 
 
 ## Chunk Conformer

From c07d248afdfe44424ab01a8d2c586e24e23742e9 Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Wed, 23 Mar 2022 15:12:31 +0800
Subject: [PATCH 048/126] test=doc

---
 examples/aishell/asr1/RESULTS.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/examples/aishell/asr1/RESULTS.md b/examples/aishell/asr1/RESULTS.md
index d3f9743d..5ebcfe50 100644
--- a/examples/aishell/asr1/RESULTS.md
+++ b/examples/aishell/asr1/RESULTS.md
@@ -1,13 +1,14 @@
 # Aishell
 
 ## Conformer
-
-| Model | Params | Config | Augmentation| Test set | Decode method | Loss | CER |  paddle version
-| --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention | - | 0.0548 |  2.2.2
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | ctc_greedy_search | - | 0.05127 | 2.2.2 
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug| test | ctc_prefix_beam_search | - | 0.05131 |  2.2.2
-| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention_rescoring | - | 0.04829 |  2.2.2
+paddle version: 2.2.2  
+paddlespeech version: 0.1.2
+| Model | Params | Config | Augmentation| Test set | Decode method | Loss | CER |
+| --- | --- | --- | --- | --- | --- | --- | --- | 
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention | - | 0.0548 |
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | ctc_greedy_search | - | 0.05127 |
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug| test | ctc_prefix_beam_search | - | 0.05131 | 
+| conformer | 47.07M  | conf/conformer.yaml | spec_aug | test | attention_rescoring | - | 0.04829 | 
 
 
 ## Chunk Conformer

From 5c1283289ed90056659a8fa6928dbffde2902f5e Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Wed, 23 Mar 2022 15:16:30 +0800
Subject: [PATCH 049/126] [Doc] Updata doc

---
 examples/aishell/asr1/README.md | 25 +------------------------
 1 file changed, 1 insertion(+), 24 deletions(-)

diff --git a/examples/aishell/asr1/README.md b/examples/aishell/asr1/README.md
index 1226a4f4..5277a31e 100644
--- a/examples/aishell/asr1/README.md
+++ b/examples/aishell/asr1/README.md
@@ -168,30 +168,7 @@ bash local/data.sh --stage -1 --stop_stage -1
 bash local/data.sh --stage 2 --stop_stage 2
 CUDA_VISIBLE_DEVICES= ./local/test.sh conf/transformer.yaml exp/transformer/checkpoints/avg_20
 ```
-The performance of the released models are shown below:
-### Conformer
-| Model     | Params | Config              | Augmentation     | Test set | Decode method          | Loss | CER      |
-| --------- | ------ | ------------------- | ---------------- | -------- | ---------------------- | ---- | -------- |
-| conformer | 47.07M | conf/conformer.yaml | spec_aug + shift | test     | attention              | -    | 0.059858 |
-| conformer | 47.07M | conf/conformer.yaml | spec_aug + shift | test     | ctc_greedy_search      | -    | 0.062311 |
-| conformer | 47.07M | conf/conformer.yaml | spec_aug + shift | test     | ctc_prefix_beam_search | -    | 0.062196 |
-| conformer | 47.07M | conf/conformer.yaml | spec_aug + shift | test     | attention_rescoring    | -    | 0.054694 |
-### Chunk Conformer
-Need set `decoding.decoding_chunk_size=16` when decoding.
-| Model     | Params | Config                    | Augmentation     | Test set | Decode method          | Chunk Size & Left Chunks | Loss | CER      |
-| --------- | ------ | ------------------------- | ---------------- | -------- | ---------------------- | ------------------------ | ---- | -------- |
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test     | attention              | 16, -1                   | -    | 0.061939 |
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test     | ctc_greedy_search      | 16, -1                   | -    | 0.070806 |
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test     | ctc_prefix_beam_search | 16, -1                   | -    | 0.070739 |
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test     | attention_rescoring    | 16, -1                   | -    | 0.059400 |
-
-### Transformer 
-| Model       | Params | Config                | Augmentation | Test set | Decode method          | Loss              | CER      |
-| ----------- | ------ | --------------------- | ------------ | -------- | ---------------------- | ----------------- | -------- |
-| transformer | 31.95M | conf/transformer.yaml | spec_aug     | test     | attention              | 3.858648955821991 | 0.057293 |
-| transformer | 31.95M | conf/transformer.yaml | spec_aug     | test     | ctc_greedy_search      | 3.858648955821991 | 0.061837 |
-| transformer | 31.95M | conf/transformer.yaml | spec_aug     | test     | ctc_prefix_beam_search | 3.858648955821991 | 0.061685 |
-| transformer | 31.95M | conf/transformer.yaml | spec_aug     | test     | attention_rescoring    | 3.858648955821991 | 0.053844 |
+[The performance of the released models](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/examples/aishell/asr1/RESULTS.md)
 ## Stage 4: CTC Alignment 
 If you want to get the alignment between the audio and the text, you can use the ctc alignment. The code of this stage is shown below:
 ```bash

From e2684e71f226f8777941f9aef5eef788050fb065 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Wed, 23 Mar 2022 18:25:32 +0800
Subject: [PATCH 050/126] refactor the data prepare process

---
 examples/voxceleb/sv0/local/data.sh | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/examples/voxceleb/sv0/local/data.sh b/examples/voxceleb/sv0/local/data.sh
index 42629c69..a3ff1c48 100755
--- a/examples/voxceleb/sv0/local/data.sh
+++ b/examples/voxceleb/sv0/local/data.sh
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-stage=0
+stage=1
 stop_stage=100
 
 . ${MAIN_ROOT}/utils/parse_options.sh || exit -1;
@@ -36,4 +36,23 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
     python3 local/data_prepare.py \
                         --data-dir ${dir} \
                         --config ${conf_path}
-fi 
\ No newline at end of file
+fi 
+
+TARGET_DIR=${MAIN_ROOT}/dataset
+mkdir -p ${TARGET_DIR}
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    # download data, generate manifests
+    python3 ${TARGET_DIR}/voxceleb/voxceleb1.py \
+      --manifest_prefix="data/vox1/manifest" \
+      --target_dir="${TARGET_DIR}/voxceleb/vox1/"
+
+    if [ $? -ne 0 ]; then
+        echo "Prepare voxceleb failed. Terminated."
+        exit 1
+    fi
+
+   #  for dataset in train dev test; do
+   #      mv data/manifest.${dataset} data/manifest.${dataset}.raw
+   #  done
+fi
\ No newline at end of file

From 4051e7b7623cc4b782efbd20e0a6e1d9f0162011 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Wed, 23 Mar 2022 11:07:40 +0000
Subject: [PATCH 051/126] fix compliance test bug, and format

---
 .gitignore                                    |  4 ++-
 demos/audio_searching/requirements.txt        | 13 +++++----
 demos/audio_searching/src/config.py           |  1 -
 demos/audio_searching/src/logs.py             |  3 +--
 demos/audio_searching/src/operations/load.py  |  5 ++--
 examples/ami/sd0/local/ami_prepare.py         |  1 -
 paddleaudio/.gitignore                        |  2 ++
 paddleaudio/docs/README.md                    |  2 +-
 paddleaudio/docs/source/conf.py               | 27 +++++--------------
 .../paddleaudio/compliance/__init__.py        |  2 ++
 paddleaudio/paddleaudio/metric/mcd.py         |  4 ++-
 .../frontend/zh_normalization/chronology.py   |  4 +--
 paddlespeech/vector/cluster/diarization.py    | 12 ++++-----
 13 files changed, 33 insertions(+), 47 deletions(-)
 create mode 100644 paddleaudio/.gitignore

diff --git a/.gitignore b/.gitignore
index ad8e7492..e25ec327 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,4 +34,6 @@ tools/activate_python.sh
 tools/miniconda.sh
 tools/CRF++-0.58/
 
-speechx/fc_patch/
\ No newline at end of file
+speechx/fc_patch/
+
+third_party/ctc_decoders/paddlespeech_ctcdecoders.py
diff --git a/demos/audio_searching/requirements.txt b/demos/audio_searching/requirements.txt
index 9e73361b..6eb3fd80 100644
--- a/demos/audio_searching/requirements.txt
+++ b/demos/audio_searching/requirements.txt
@@ -1,12 +1,11 @@
-soundfile==0.10.3.post1
+diskcache==5.2.1
+fastapi
 librosa==0.8.0
 numpy
+pydanticpymilvus==2.0.1
 pymysql
-fastapi
-uvicorn
-diskcache==5.2.1
-pymilvus==2.0.1
 python-multipart
-typing
+soundfile==0.10.3.post1
 starlette
-pydantic
\ No newline at end of file
+typing
+uvicorn
diff --git a/demos/audio_searching/src/config.py b/demos/audio_searching/src/config.py
index 72a8fb4b..70ac494c 100644
--- a/demos/audio_searching/src/config.py
+++ b/demos/audio_searching/src/config.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import os
 
 ############### Milvus Configuration ###############
diff --git a/demos/audio_searching/src/logs.py b/demos/audio_searching/src/logs.py
index ba3ed069..465eb682 100644
--- a/demos/audio_searching/src/logs.py
+++ b/demos/audio_searching/src/logs.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import codecs
 import datetime
 import logging
 import os
@@ -124,7 +123,7 @@ class MultiprocessHandler(logging.FileHandler):
             logging.FileHandler.emit(self, record)
         except (KeyboardInterrupt, SystemExit):
             raise
-        except:
+        except Exception as e:
             self.handleError(record)
 
 
diff --git a/demos/audio_searching/src/operations/load.py b/demos/audio_searching/src/operations/load.py
index 7a295bf3..80b6375f 100644
--- a/demos/audio_searching/src/operations/load.py
+++ b/demos/audio_searching/src/operations/load.py
@@ -26,9 +26,8 @@ def get_audios(path):
     """
     supported_formats = [".wav", ".mp3", ".ogg", ".flac", ".m4a"]
     return [
-        item
-        for sublist in [[os.path.join(dir, file) for file in files]
-                        for dir, _, files in list(os.walk(path))]
+        item for sublist in [[os.path.join(dir, file) for file in files]
+                             for dir, _, files in list(os.walk(path))]
         for item in sublist if os.path.splitext(item)[1] in supported_formats
     ]
 
diff --git a/examples/ami/sd0/local/ami_prepare.py b/examples/ami/sd0/local/ami_prepare.py
index 01582dbd..569c3a60 100644
--- a/examples/ami/sd0/local/ami_prepare.py
+++ b/examples/ami/sd0/local/ami_prepare.py
@@ -18,7 +18,6 @@ Download: http://groups.inf.ed.ac.uk/ami/download/
 
 Prepares metadata files (JSON) from manual annotations "segments/" using RTTM format (Oracle VAD).
 """
-
 import argparse
 import glob
 import json
diff --git a/paddleaudio/.gitignore b/paddleaudio/.gitignore
new file mode 100644
index 00000000..1c930053
--- /dev/null
+++ b/paddleaudio/.gitignore
@@ -0,0 +1,2 @@
+.eggs
+*.wav
diff --git a/paddleaudio/docs/README.md b/paddleaudio/docs/README.md
index d53f0be7..8e4fccc5 100644
--- a/paddleaudio/docs/README.md
+++ b/paddleaudio/docs/README.md
@@ -15,4 +15,4 @@ Exclude `paddleaudio.utils`
 
 ## 3. Build
 
-`sphinx-build source _html`
\ No newline at end of file
+`sphinx-build source _html`
diff --git a/paddleaudio/docs/source/conf.py b/paddleaudio/docs/source/conf.py
index 4efe85b0..09c4f312 100644
--- a/paddleaudio/docs/source/conf.py
+++ b/paddleaudio/docs/source/conf.py
@@ -5,18 +5,14 @@
 # This file does only contain a selection of the most common options. For a
 # full list see the documentation:
 # http://www.sphinx-doc.org/en/master/config
-
 # -- Path setup --------------------------------------------------------------
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-
 import os
 import sys
 sys.path.insert(0, os.path.abspath('../..'))
 
-
 # -- Project information -----------------------------------------------------
 
 project = 'PaddleAudio'
@@ -28,7 +24,6 @@ version = ''
 # The full version, including alpha/beta/rc tags
 release = '0.2.0'
 
-
 # -- General configuration ---------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -75,7 +70,6 @@ exclude_patterns = []
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None
 
-
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -112,13 +106,11 @@ html_css_files = [
 #
 # html_sidebars = {}
 
-
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'PaddleAudiodoc'
 
-
 # -- Options for LaTeX output ------------------------------------------------
 
 latex_elements = {
@@ -143,20 +135,16 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'PaddleAudio.tex', 'PaddleAudio Documentation',
-     'PaddlePaddle', 'manual'),
+    (master_doc, 'PaddleAudio.tex', 'PaddleAudio Documentation', 'PaddlePaddle',
+     'manual'),
 ]
 
-
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'paddleaudio', 'PaddleAudio Documentation',
-     [author], 1)
-]
-
+man_pages = [(master_doc, 'paddleaudio', 'PaddleAudio Documentation', [author],
+              1)]
 
 # -- Options for Texinfo output ----------------------------------------------
 
@@ -164,12 +152,10 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'PaddleAudio', 'PaddleAudio Documentation',
-     author, 'PaddleAudio', 'One line description of project.',
-     'Miscellaneous'),
+    (master_doc, 'PaddleAudio', 'PaddleAudio Documentation', author,
+     'PaddleAudio', 'One line description of project.', 'Miscellaneous'),
 ]
 
-
 # -- Options for Epub output -------------------------------------------------
 
 # Bibliographic Dublin Core info.
@@ -187,7 +173,6 @@ epub_title = project
 # A list of files that should not be packed into the epub file.
 epub_exclude_files = ['search.html']
 
-
 # -- Extension configuration -------------------------------------------------
 
 # -- Options for intersphinx extension ---------------------------------------
diff --git a/paddleaudio/paddleaudio/compliance/__init__.py b/paddleaudio/paddleaudio/compliance/__init__.py
index 97043fd7..c08f9ab1 100644
--- a/paddleaudio/paddleaudio/compliance/__init__.py
+++ b/paddleaudio/paddleaudio/compliance/__init__.py
@@ -11,3 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from . import kaldi
+from . import librosa
diff --git a/paddleaudio/paddleaudio/metric/mcd.py b/paddleaudio/paddleaudio/metric/mcd.py
index d1852b4b..63a25fc2 100644
--- a/paddleaudio/paddleaudio/metric/mcd.py
+++ b/paddleaudio/paddleaudio/metric/mcd.py
@@ -22,7 +22,9 @@ __all__ = [
 ]
 
 
-def mcd_distance(xs: np.ndarray, ys: np.ndarray, cost_fn: Callable=mt.logSpecDbDist) -> float:
+def mcd_distance(xs: np.ndarray,
+                 ys: np.ndarray,
+                 cost_fn: Callable=mt.logSpecDbDist) -> float:
     """Mel cepstral distortion (MCD), dtw distance.
 
     Dynamic Time Warping.
diff --git a/paddlespeech/t2s/frontend/zh_normalization/chronology.py b/paddlespeech/t2s/frontend/zh_normalization/chronology.py
index ea518913..ea4558e2 100644
--- a/paddlespeech/t2s/frontend/zh_normalization/chronology.py
+++ b/paddlespeech/t2s/frontend/zh_normalization/chronology.py
@@ -64,7 +64,7 @@ def replace_time(match) -> str:
     result = f"{num2str(hour)}点"
     if minute.lstrip('0'):
         if int(minute) == 30:
-            result += f"半"
+            result += "半"
         else:
             result += f"{_time_num2str(minute)}分"
     if second and second.lstrip('0'):
@@ -75,7 +75,7 @@ def replace_time(match) -> str:
         result += f"{num2str(hour_2)}点"
         if minute_2.lstrip('0'):
             if int(minute) == 30:
-                result += f"半"
+                result += "半"
             else:
                 result += f"{_time_num2str(minute_2)}分"
         if second_2 and second_2.lstrip('0'):
diff --git a/paddlespeech/vector/cluster/diarization.py b/paddlespeech/vector/cluster/diarization.py
index 6432acb8..99ac41cd 100644
--- a/paddlespeech/vector/cluster/diarization.py
+++ b/paddlespeech/vector/cluster/diarization.py
@@ -16,22 +16,20 @@ This script contains basic functions used for speaker diarization.
 This script has an optional dependency on open source sklearn library.
 A few sklearn functions are modified in this script as per requirement.
 """
-
 import argparse
 import warnings
-import scipy
-import numpy as np
 from distutils.util import strtobool
 
+import numpy as np
+import scipy
+import sklearn
 from scipy import sparse
-from scipy.sparse.linalg import eigsh
 from scipy.sparse.csgraph import connected_components
 from scipy.sparse.csgraph import laplacian as csgraph_laplacian
-
-import sklearn
-from sklearn.neighbors import kneighbors_graph
+from scipy.sparse.linalg import eigsh
 from sklearn.cluster import SpectralClustering
 from sklearn.cluster._kmeans import k_means
+from sklearn.neighbors import kneighbors_graph
 
 
 def _graph_connected_component(graph, node_id):

From 115798b6ab58b7ffbb920e3de1217a674007325a Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 24 Mar 2022 03:06:21 +0000
Subject: [PATCH 052/126] fix audio search reqirement, test=doc

---
 demos/audio_searching/requirements.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/demos/audio_searching/requirements.txt b/demos/audio_searching/requirements.txt
index 6eb3fd80..95c6140d 100644
--- a/demos/audio_searching/requirements.txt
+++ b/demos/audio_searching/requirements.txt
@@ -2,10 +2,11 @@ diskcache==5.2.1
 fastapi
 librosa==0.8.0
 numpy
-pydanticpymilvus==2.0.1
+pydantic
+pymilvus==2.0.1
 pymysql
 python-multipart
 soundfile==0.10.3.post1
 starlette
 typing
-uvicorn
+uvicorn
\ No newline at end of file

From be3a2a50f84ef87c3a1eb6655e44db958862b781 Mon Sep 17 00:00:00 2001
From: Yang Zhou <goat.zhou@qq.com>
Date: Thu, 24 Mar 2022 13:30:35 +0800
Subject: [PATCH 053/126] fix normalizer bug

---
 speechx/speechx/frontend/normalizer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/speechx/speechx/frontend/normalizer.cc b/speechx/speechx/frontend/normalizer.cc
index 1adddb40..52412561 100644
--- a/speechx/speechx/frontend/normalizer.cc
+++ b/speechx/speechx/frontend/normalizer.cc
@@ -107,7 +107,7 @@ void CMVN::Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs) {
 }
 
 bool CMVN::Read(kaldi::Vector<BaseFloat>* feats) {
-    if (base_extractor_->Read(feats) == false) {
+    if (base_extractor_->Read(feats) == false || feats->Dim() == 0) {
         return false;
     }
     Compute(feats);

From 342b487383eee975c7efaac685508ffa3006a284 Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Thu, 24 Mar 2022 06:13:47 +0000
Subject: [PATCH 054/126] update readme for ljspeech hifigan, test=tts

---
 CHANGELOG.md                     | 11 +++++++++++
 docs/source/released_model.md    |  1 +
 examples/ljspeech/voc5/README.md | 15 +++++++++++++++
 paddlespeech/cli/tts/infer.py    | 13 +++++++++++++
 tests/unit/cli/test_cli.sh       |  1 +
 5 files changed, 41 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 62fead47..2782b817 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,15 @@
 # Changelog
+
+Date: 2022-3-22, Author: yt605155624.
+Add features to: CLI:
+  - Support aishell3_hifigan、vctk_hifigan
+  - PRLink: https://github.com/PaddlePaddle/PaddleSpeech/pull/1587
+
+Date: 2022-3-09, Author: yt605155624.
+Add features to: T2S:
+  - Add ljspeech hifigan egs.
+  - PRLink: https://github.com/PaddlePaddle/PaddleSpeech/pull/1549
+
 Date: 2022-3-08, Author: yt605155624.
 Add features to: T2S:
   - Add aishell3 hifigan egs.
diff --git a/docs/source/released_model.md b/docs/source/released_model.md
index 62986da0..db0b31f1 100644
--- a/docs/source/released_model.md
+++ b/docs/source/released_model.md
@@ -54,6 +54,7 @@ Parallel WaveGAN| VCTK |[PWGAN-vctk](https://github.com/PaddlePaddle/PaddleSpeec
 |Multi Band MelGAN | CSMSC |[MB MelGAN-csmsc](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/voc3) | [mb_melgan_csmsc_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/mb_melgan/mb_melgan_csmsc_ckpt_0.1.1.zip) <br>[mb_melgan_baker_finetune_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/mb_melgan/mb_melgan_baker_finetune_ckpt_0.5.zip)|[mb_melgan_csmsc_static_0.1.1.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/mb_melgan/mb_melgan_csmsc_static_0.1.1.zip) |8.2MB|
 Style MelGAN | CSMSC |[Style MelGAN-csmsc](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/voc4)|[style_melgan_csmsc_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/style_melgan/style_melgan_csmsc_ckpt_0.1.1.zip)| | |
 HiFiGAN | CSMSC |[HiFiGAN-csmsc](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/voc5)|[hifigan_csmsc_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_csmsc_ckpt_0.1.1.zip)|[hifigan_csmsc_static_0.1.1.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_csmsc_static_0.1.1.zip)|50MB|
+HiFiGAN | LJSpeech |[HiFiGAN-ljspeech](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/ljspeech/voc5)|[hifigan_ljspeech_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_ljspeech_ckpt_0.2.0.zip)|||
 HiFiGAN | AISHELL-3 |[HiFiGAN-aishell3](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/voc5)|[hifigan_aishell3_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_aishell3_ckpt_0.2.0.zip)|||
 HiFiGAN | VCTK |[HiFiGAN-vctk](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/vctk/voc5)|[hifigan_aishell3_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_aishell3_ckpt_0.2.0.zip)|||
 WaveRNN | CSMSC |[WaveRNN-csmsc](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/voc6)|[wavernn_csmsc_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/wavernn/wavernn_csmsc_ckpt_0.2.0.zip)|[wavernn_csmsc_static_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/wavernn/wavernn_csmsc_static_0.2.0.zip)|18MB|
diff --git a/examples/ljspeech/voc5/README.md b/examples/ljspeech/voc5/README.md
index 21082942..9fbb9f74 100644
--- a/examples/ljspeech/voc5/README.md
+++ b/examples/ljspeech/voc5/README.md
@@ -127,6 +127,21 @@ optional arguments:
 5. `--ngpu` is the number of gpus to use, if ngpu == 0, use cpu.
 
 ## Pretrained Model
+The pretrained model can be downloaded here [hifigan_ljspeech_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_ljspeech_ckpt_0.2.0.zip).
+
+
+Model | Step | eval/generator_loss | eval/mel_loss| eval/feature_matching_loss
+:-------------:| :------------:| :-----: | :-----: | :--------:
+default| 1(gpu) x 2500000|24.492|0.115|7.227
+
+HiFiGAN checkpoint contains files listed below.
+
+```text
+hifigan_ljspeech_ckpt_0.2.0
+├── default.yaml                  # default config used to train hifigan
+├── feats_stats.npy               # statistics used to normalize spectrogram when training hifigan
+└── snapshot_iter_2500000.pdz     # generator parameters of hifigan
+```
 
 
 ## Acknowledgement
diff --git a/paddlespeech/cli/tts/infer.py b/paddlespeech/cli/tts/infer.py
index 78eae769..c7a1edc9 100644
--- a/paddlespeech/cli/tts/infer.py
+++ b/paddlespeech/cli/tts/infer.py
@@ -237,6 +237,18 @@ pretrained_models = {
         'speech_stats':
         'feats_stats.npy',
     },
+    "hifigan_ljspeech-en": {
+        'url':
+        'https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_ljspeech_ckpt_0.2.0.zip',
+        'md5':
+        '70e9131695decbca06a65fe51ed38a72',
+        'config':
+        'default.yaml',
+        'ckpt':
+        'snapshot_iter_2500000.pdz',
+        'speech_stats':
+        'feats_stats.npy',
+    },
     "hifigan_aishell3-zh": {
         'url':
         'https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_aishell3_ckpt_0.2.0.zip',
@@ -389,6 +401,7 @@ class TTSExecutor(BaseExecutor):
                 'mb_melgan_csmsc',
                 'style_melgan_csmsc',
                 'hifigan_csmsc',
+                'hifigan_ljspeech',
                 'hifigan_aishell3',
                 'hifigan_vctk',
                 'wavernn_csmsc',
diff --git a/tests/unit/cli/test_cli.sh b/tests/unit/cli/test_cli.sh
index 9852b069..b0d18b3b 100755
--- a/tests/unit/cli/test_cli.sh
+++ b/tests/unit/cli/test_cli.sh
@@ -21,6 +21,7 @@ paddlespeech tts --voc hifigan_csmsc --input "你好，欢迎使用百度飞桨
 paddlespeech tts --am fastspeech2_aishell3 --voc pwgan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
 paddlespeech tts --am fastspeech2_aishell3 --voc hifigan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
 paddlespeech tts --am fastspeech2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
+paddlespeech tts --am fastspeech2_ljspeech --voc hifigan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
 paddlespeech tts --am fastspeech2_vctk --voc pwgan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
 paddlespeech tts --am fastspeech2_vctk --voc hifigan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
 paddlespeech tts --am tacotron2_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"

From 5347dbad3fbab117ff2f92cdbdfeeb22f35edd7b Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Thu, 24 Mar 2022 08:13:20 +0000
Subject: [PATCH 055/126] fix dtype of window of stft, test=tts

---
 paddlespeech/t2s/modules/losses.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/paddlespeech/t2s/modules/losses.py b/paddlespeech/t2s/modules/losses.py
index 93644e24..db31bcfb 100644
--- a/paddlespeech/t2s/modules/losses.py
+++ b/paddlespeech/t2s/modules/losses.py
@@ -489,7 +489,7 @@ def stft(x,
     """
     # calculate window
     window = signal.get_window(window, win_length, fftbins=True)
-    window = paddle.to_tensor(window)
+    window = paddle.to_tensor(window, dtype=x.dtype)
     x_stft = paddle.signal.stft(
         x,
         fft_size,
@@ -896,7 +896,7 @@ class MelSpectrogram(nn.Layer):
             # calculate window
             window = signal.get_window(
                 self.window, self.win_length, fftbins=True)
-            window = paddle.to_tensor(window)
+            window = paddle.to_tensor(window, dtype=x.dtype)
         else:
             window = None
 

From 62cbce69152baf953282573435f3c164dd0bee24 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 24 Mar 2022 16:25:26 +0800
Subject: [PATCH 056/126] add vectorwrapper to extract audio embedding

---
 dataset/voxceleb/voxceleb1.py                 |  47 +++--
 dataset/voxceleb/voxceleb2.py                 | 163 ++++++++++++++++++
 .../vector/exps/ecapa_tdnn/extract_emb.py     |  93 ++++++++++
 3 files changed, 293 insertions(+), 10 deletions(-)
 create mode 100644 dataset/voxceleb/voxceleb2.py

diff --git a/dataset/voxceleb/voxceleb1.py b/dataset/voxceleb/voxceleb1.py
index c6fc0695..d0978d9d 100644
--- a/dataset/voxceleb/voxceleb1.py
+++ b/dataset/voxceleb/voxceleb1.py
@@ -59,12 +59,17 @@ DEV_TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f5
 TEST_LIST = {"vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102"}
 TEST_TARGET_DATA = "vox1_test_wav.zip vox1_test_wav.zip 185fdc63c3c739954633d50379a3d102"
 
-# kaldi trial
-# this trial file is organized by kaldi according the official file,
-# which is a little different with the official trial veri_test2.txt
-KALDI_BASE_URL = "http://www.openslr.org/resources/49/"
-TRIAL_LIST = {"voxceleb1_test_v2.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7"}
-TRIAL_TARGET_DATA = "voxceleb1_test_v2.txt voxceleb1_test_v2.txt 29fc7cc1c5d59f0816dc15d6e8be60f7"
+# voxceleb trial
+
+TRIAL_BASE_URL = "https://www.robots.ox.ac.uk/~vgg/data/voxceleb/meta/"
+TRIAL_LIST = {
+    "veri_test.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7",             # voxceleb1
+    "veri_test2.txt": "b73110731c9223c1461fe49cb48dddfc",            # voxceleb1(cleaned)
+    "list_test_hard.txt": "21c341b6b2168eea2634df0fb4b8fff1",        # voxceleb1-H
+    "list_test_hard2.txt": "857790e09d579a68eb2e339a090343c8",       # voxceleb1-H(cleaned)
+    "list_test_all.txt": "b9ecf7aa49d4b656aa927a8092844e4a",         # voxceleb1-E
+    "list_test_all2.txt": "a53e059deb562ffcfc092bf5d90d9f3a"         # voxceleb1-E(cleaned)
+    }
 
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(
@@ -82,7 +87,7 @@ args = parser.parse_args()
 
 
 def create_manifest(data_dir, manifest_path_prefix):
-    print("Creating manifest %s ..." % manifest_path_prefix)
+    print(f"Creating manifest {manifest_path_prefix} from {data_dir}")
     json_lines = []
     data_path = os.path.join(data_dir, "wav", "**", "*.wav")
     total_sec = 0.0
@@ -114,6 +119,9 @@ def create_manifest(data_dir, manifest_path_prefix):
     # voxceleb1 is given explicit in the path
     data_dir_name = Path(data_dir).name
     manifest_path_prefix = manifest_path_prefix + "." + data_dir_name
+    if not os.path.exists(os.path.dirname(manifest_path_prefix)):
+        os.makedirs(os.path.dirname(manifest_path_prefix))
+
     with codecs.open(manifest_path_prefix, 'w', encoding='utf-8') as f:
         for line in json_lines:
             f.write(line + "\n")
@@ -133,11 +141,13 @@ def create_manifest(data_dir, manifest_path_prefix):
 def prepare_dataset(base_url, data_list, target_dir, manifest_path,
                     target_data):
     if not os.path.exists(target_dir):
-        os.mkdir(target_dir)
+        os.makedirs(target_dir)
 
     # wav directory already exists, it need do nothing
+    # we will download the voxceleb1 data to ${target_dir}/vox1/dev/ or ${target_dir}/vox1/test directory 
     if not os.path.exists(os.path.join(target_dir, "wav")):
         # download all dataset part
+        print("start to download the vox1 dev zip package")
         for zip_part in data_list.keys():
             download_url = " --no-check-certificate " + base_url + "/" + zip_part
             download(
@@ -166,11 +176,20 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path,
     # create the manifest file
     create_manifest(data_dir=target_dir, manifest_path_prefix=manifest_path)
 
+def prepare_trial(base_url, data_list, target_dir):
+    if not os.path.exists(target_dir):
+        os.makedirs(target_dir)
 
+    for trial, md5sum in data_list.items():
+        target_trial = os.path.join(target_dir, trial)
+        if not os.path.exists(os.path.join(target_dir, trial)):
+            download_url = " --no-check-certificate " + base_url + "/" + trial
+            download(url=download_url, md5sum=md5sum, target_dir=target_dir)
 def main():
     if args.target_dir.startswith('~'):
         args.target_dir = os.path.expanduser(args.target_dir)
-
+    
+    # prepare the vox1 dev data
     prepare_dataset(
         base_url=BASE_URL,
         data_list=DEV_LIST,
@@ -178,6 +197,7 @@ def main():
         manifest_path=args.manifest_prefix,
         target_data=DEV_TARGET_DATA)
 
+    # prepare the vox1 test data
     prepare_dataset(
         base_url=BASE_URL,
         data_list=TEST_LIST,
@@ -185,8 +205,15 @@ def main():
         manifest_path=args.manifest_prefix,
         target_data=TEST_TARGET_DATA)
 
+    # prepare the vox1 trial
+    prepare_trial(
+        base_url=TRIAL_BASE_URL,
+        data_list=TRIAL_LIST,
+        target_dir=os.path.dirname(args.manifest_prefix)
+    )
+
     print("Manifest prepare done!")
 
 
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()
diff --git a/dataset/voxceleb/voxceleb2.py b/dataset/voxceleb/voxceleb2.py
new file mode 100644
index 00000000..ef7bb230
--- /dev/null
+++ b/dataset/voxceleb/voxceleb2.py
@@ -0,0 +1,163 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Prepare VoxCeleb2 dataset
+
+Download and unpack the voxceleb2 data files.
+Voxceleb2 data is stored as the m4a format, 
+so we need convert the m4a to wav with the convert.sh scripts
+"""
+import argparse
+import codecs
+import glob
+import json
+import os
+import subprocess
+from pathlib import Path
+
+import soundfile
+
+from utils.utility import check_md5sum
+from utils.utility import download
+from utils.utility import unzip
+
+# all the data will be download in the current data/voxceleb directory default
+DATA_HOME = os.path.expanduser('.')
+
+BASE_URL = "--no-check-certificate https://www.robots.ox.ac.uk/~vgg/data/voxceleb/data/"
+
+# dev data
+DEV_DATA_URL = BASE_URL + '/vox2_aac.zip'
+DEV_MD5SUM = "bbc063c46078a602ca71605645c2a402"
+
+
+# test data
+TEST_DATA_URL = BASE_URL +  '/vox2_test_aac.zip'
+TEST_MD5SUM = "0d2b3ea430a821c33263b5ea37ede312"
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument(
+    "--target_dir",
+    default=DATA_HOME + "/voxceleb2/",
+    type=str,
+    help="Directory to save the voxceleb1 dataset. (default: %(default)s)")
+parser.add_argument(
+    "--manifest_prefix",
+    default="manifest",
+    type=str,
+    help="Filepath prefix for output manifests. (default: %(default)s)")
+parser.add_argument("--download", 
+                    default=False, 
+                    action="store_true", 
+                    help="Download the voxceleb2 dataset. (default: %(default)s)")
+parser.add_argument("--generate", 
+                    default=False, 
+                    action="store_true", 
+                    help="Generate the manifest files. (default: %(default)s)")
+
+args = parser.parse_args()
+
+
+def create_manifest(data_dir, manifest_path_prefix):
+    print("Creating manifest %s ..." % manifest_path_prefix)
+    json_lines = []
+    data_path = os.path.join(data_dir, "**", "*.wav")
+    total_sec = 0.0
+    total_text = 0.0
+    total_num = 0
+    speakers = set()
+    for audio_path in glob.glob(data_path, recursive=True):
+        audio_id = "-".join(audio_path.split("/")[-3:])
+        utt2spk = audio_path.split("/")[-3]
+        duration = soundfile.info(audio_path).duration
+        text = ""
+        json_lines.append(
+            json.dumps(
+                {
+                    "utt": audio_id,
+                    "utt2spk": str(utt2spk),
+                    "feat": audio_path,
+                    "feat_shape": (duration, ),
+                    "text": text  # compatible with asr data format
+                },
+                ensure_ascii=False))
+
+        total_sec += duration
+        total_text += len(text)
+        total_num += 1
+        speakers.add(utt2spk)
+
+    # data_dir_name refer to dev or test
+    # voxceleb2 is given explicit in the path
+    data_dir_name = Path(data_dir).name
+    manifest_path_prefix = manifest_path_prefix + "." + data_dir_name
+
+    if not os.path.exists(os.path.dirname(manifest_path_prefix)):
+        os.makedirs(os.path.dirname(manifest_path_prefix))
+    with codecs.open(manifest_path_prefix, 'w', encoding='utf-8') as f:
+        for line in json_lines:
+            f.write(line + "\n")
+
+    manifest_dir = os.path.dirname(manifest_path_prefix)
+    meta_path = os.path.join(manifest_dir, "voxceleb2." +
+                             data_dir_name) + ".meta"
+    with codecs.open(meta_path, 'w', encoding='utf-8') as f:
+        print(f"{total_num} utts", file=f)
+        print(f"{len(speakers)} speakers", file=f)
+        print(f"{total_sec / (60 * 60)} h", file=f)
+        print(f"{total_text} text", file=f)
+        print(f"{total_text / total_sec} text/sec", file=f)
+        print(f"{total_sec / total_num} sec/utt", file=f)
+
+
+def download_dataset(url, md5sum, target_dir, dataset):
+    if not os.path.exists(target_dir):
+        os.makedirs(target_dir)
+
+    # wav directory already exists, it need do nothing
+    print("target dir {}".format(os.path.join(target_dir, dataset)))
+    # unzip the dev dataset will create the dev and unzip the m4a to dev dir
+    # but the test dataset will unzip to aac
+    # so, wo create the ${target_dir}/test and unzip the m4a to test dir
+    if not os.path.exists(os.path.join(target_dir, dataset)):
+        filepath = download(url, md5sum, target_dir)
+        if dataset == "test":
+            unzip(filepath, os.path.join(target_dir, "test"))
+
+
+def main():
+    if args.target_dir.startswith('~'):
+        args.target_dir = os.path.expanduser(args.target_dir)
+    
+    # download and unpack the vox2-dev data
+    print("download: {}".format(args.download))
+    if args.download:
+        download_dataset(
+            url=DEV_DATA_URL,
+            md5sum=DEV_MD5SUM,
+            target_dir=args.target_dir,
+            dataset="dev")
+
+        download_dataset(
+            url=TEST_DATA_URL,
+            md5sum=TEST_MD5SUM,
+            target_dir=args.target_dir,
+            dataset="test")
+
+        print("VoxCeleb2 download is done!")
+
+    if args.generate:
+        create_manifest(args.target_dir, manifest_path_prefix=args.manifest_prefix)
+
+if __name__ == '__main__':
+    main()
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
index e30a50e4..ec24be51 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
@@ -28,6 +28,91 @@ from paddlespeech.vector.training.seeding import seed_everything
 
 logger = Log(__name__).getlog()
 
+class VectorWrapper:
+    """ VectorWrapper extract the audio embedding,
+        and single audio will get only an embedding
+    """
+    def __init__(self,
+                 device,
+                 config_path,
+                 model_path,):
+        super(VectorWrapper, self).__init__()
+        # stage 0: config the 
+        self.device = device
+        self.config_path = config_path
+        self.model_path = model_path
+
+        # stage 1: set the run host device
+        paddle.device.set_device(device)
+
+        # stage 2: read the yaml config and set the seed factor
+        self.read_yaml_config(self.config_path)
+        seed_everything(self.config.seed)
+
+        # stage 3: init the speaker verification model
+        self.init_vector_model(self.config, self.model_path)
+        
+    def read_yaml_config(self, config_path):
+        """Read the yaml config from the config path
+
+        Args:
+            config_path (str): yaml config path
+        """
+        config = CfgNode(new_allowed=True)
+
+        if config_path:
+            config.merge_from_file(config_path)
+
+        config.freeze()
+        self.config = config
+
+    def init_vector_model(self, config, model_path):
+        """Init the vector model from yaml config
+
+        Args:
+            config (CfgNode): yaml config 
+            model_path (str): pretrained model path and the stored model is named as model.pdparams
+        """
+        # get the backbone network instance
+        ecapa_tdnn = EcapaTdnn(**config.model)
+        
+        # get the sid instance
+        model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=config.num_speakers)
+
+        # read the model parameters to sid model
+        model_path = os.path.abspath(os.path.expanduser(model_path))
+        state_dict = paddle.load(os.path.join(model_path, "model.pdparams"))
+        model.set_state_dict(state_dict)
+
+        model.eval()
+        self.model = model
+
+    def extract_audio_embedding(self, audio_path):
+        """Extract the audio embedding
+
+        Args:
+            audio_path (str): audio path, which will be extracted the embedding
+
+        Returns:
+            embedding (numpy.array) : audio embedding
+        """
+        waveform, sr = load_audio(audio_path)
+        feat = melspectrogram(x=waveform,
+                sr=self.config.sr,
+                n_mels=self.config.n_mels,
+                window_size=self.config.window_size,
+                hop_length=self.config.hop_size)
+        # conver the audio feat to batch shape, which means batch_size is equal to one
+        feat = paddle.to_tensor(feat).unsqueeze(0)
+
+        # in inference period, the lengths is all one without padding
+        lengths = paddle.ones([1])
+        feat = feature_normalize(feat, mean_norm=True, std_norm=False)
+        
+        # model backbone network forward the feats and get the embedding
+        embedding = self.model.backbone(feat, lengths).squeeze().numpy()  # (1, emb_size, 1) -> (emb_size)
+
+        return embedding
 
 def extract_audio_embedding(args, config):
     # stage 0: set the training device, cpu or gpu
@@ -83,6 +168,7 @@ def extract_audio_embedding(args, config):
     # stage 5: do global norm with external mean and std
     rtf = elapsed_time / audio_length
     logger.info(f"{args.device} rft={rtf}")
+    paddle.save(embedding, "emb1")
     return embedding
 
 
@@ -116,3 +202,10 @@ if __name__ == "__main__":
     print(config)
 
     extract_audio_embedding(args, config)
+
+    # use the VectorWrapper to extract the audio embedding
+    vector_inst = VectorWrapper(device="gpu", 
+                    config_path=args.config, 
+                    model_path=args.load_checkpoint)
+    
+    embedding = vector_inst.extract_audio_embedding(args.audio_path)

From 0bb67d8b8e09985d2275af497d8053ba479b455e Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 24 Mar 2022 16:55:23 +0800
Subject: [PATCH 057/126] add vector cli unit test, test=doc

---
 paddlespeech/cli/vector/infer.py | 2 +-
 tests/unit/cli/test_cli.sh       | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index f1a0e79c..91974761 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -39,7 +39,7 @@ pretrained_models = {
     # The tags for pretrained_models should be "{model_name}[-{dataset}][-{sr}][-...]".
     # e.g. "ecapatdnn_voxceleb12-16k".
     # Command line and python api use "{model_name}[-{dataset}]" as --model, usage:
-    # "paddlespeech vector --task spk --model ecapatdnn_voxceleb12-voxceleb12-16k --sr 16000 --input ./input.wav"
+    # "paddlespeech vector --task spk --model ecapatdnn_voxceleb12-16k --sr 16000 --input ./input.wav"
     "ecapatdnn_voxceleb12-16k": {
         'url':
         'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz',
diff --git a/tests/unit/cli/test_cli.sh b/tests/unit/cli/test_cli.sh
index 748e5608..ec9f8d13 100755
--- a/tests/unit/cli/test_cli.sh
+++ b/tests/unit/cli/test_cli.sh
@@ -28,3 +28,7 @@ paddlespeech tts --am tacotron2_ljspeech --voc pwgan_ljspeech --lang en --input
 
 # Speech Translation (only support linux)
 paddlespeech st --input ./en.wav
+
+# Speaker Verification 
+wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
+paddlespeech vector --task spk --input 85236145389.wav

From da4a30a36d12388780713eda0abca06cc1582371 Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Thu, 24 Mar 2022 18:20:50 +0800
Subject: [PATCH 058/126] update release_model.md, test=doc

---
 docs/source/released_model.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/released_model.md b/docs/source/released_model.md
index c5c65c82..a6092d55 100644
--- a/docs/source/released_model.md
+++ b/docs/source/released_model.md
@@ -56,7 +56,7 @@ Style MelGAN | CSMSC |[Style MelGAN-csmsc](https://github.com/PaddlePaddle/Paddl
 HiFiGAN | CSMSC |[HiFiGAN-csmsc](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/voc5)|[hifigan_csmsc_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_csmsc_ckpt_0.1.1.zip)|[hifigan_csmsc_static_0.1.1.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_csmsc_static_0.1.1.zip)|50MB|
 HiFiGAN | LJSpeech |[HiFiGAN-ljspeech](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/ljspeech/voc5)|[hifigan_ljspeech_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_ljspeech_ckpt_0.2.0.zip)|||
 HiFiGAN | AISHELL-3 |[HiFiGAN-aishell3](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/voc5)|[hifigan_aishell3_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_aishell3_ckpt_0.2.0.zip)|||
-HiFiGAN | VCTK |[HiFiGAN-vctk](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/vctk/voc5)|[hifigan_aishell3_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_aishell3_ckpt_0.2.0.zip)|||
+HiFiGAN | VCTK |[HiFiGAN-vctk](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/vctk/voc5)|[hifigan_vctk_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/hifigan/hifigan_vctk_ckpt_0.2.0.zip)|||
 WaveRNN | CSMSC |[WaveRNN-csmsc](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/voc6)|[wavernn_csmsc_ckpt_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/wavernn/wavernn_csmsc_ckpt_0.2.0.zip)|[wavernn_csmsc_static_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/wavernn/wavernn_csmsc_static_0.2.0.zip)|18MB|
 
 

From 0f78d25f7619a7d99de404be935e92004f2cd413 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 24 Mar 2022 18:52:30 +0800
Subject: [PATCH 059/126] add vector cli batch and pipeline test demo, test=doc

---
 .../vector/exps/ecapa_tdnn/extract_emb.py     | 96 +------------------
 tests/unit/cli/test_cli.sh                    | 95 +++++++++---------
 2 files changed, 54 insertions(+), 137 deletions(-)

diff --git a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
index ec24be51..686de936 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/extract_emb.py
@@ -28,91 +28,6 @@ from paddlespeech.vector.training.seeding import seed_everything
 
 logger = Log(__name__).getlog()
 
-class VectorWrapper:
-    """ VectorWrapper extract the audio embedding,
-        and single audio will get only an embedding
-    """
-    def __init__(self,
-                 device,
-                 config_path,
-                 model_path,):
-        super(VectorWrapper, self).__init__()
-        # stage 0: config the 
-        self.device = device
-        self.config_path = config_path
-        self.model_path = model_path
-
-        # stage 1: set the run host device
-        paddle.device.set_device(device)
-
-        # stage 2: read the yaml config and set the seed factor
-        self.read_yaml_config(self.config_path)
-        seed_everything(self.config.seed)
-
-        # stage 3: init the speaker verification model
-        self.init_vector_model(self.config, self.model_path)
-        
-    def read_yaml_config(self, config_path):
-        """Read the yaml config from the config path
-
-        Args:
-            config_path (str): yaml config path
-        """
-        config = CfgNode(new_allowed=True)
-
-        if config_path:
-            config.merge_from_file(config_path)
-
-        config.freeze()
-        self.config = config
-
-    def init_vector_model(self, config, model_path):
-        """Init the vector model from yaml config
-
-        Args:
-            config (CfgNode): yaml config 
-            model_path (str): pretrained model path and the stored model is named as model.pdparams
-        """
-        # get the backbone network instance
-        ecapa_tdnn = EcapaTdnn(**config.model)
-        
-        # get the sid instance
-        model = SpeakerIdetification(backbone=ecapa_tdnn, num_class=config.num_speakers)
-
-        # read the model parameters to sid model
-        model_path = os.path.abspath(os.path.expanduser(model_path))
-        state_dict = paddle.load(os.path.join(model_path, "model.pdparams"))
-        model.set_state_dict(state_dict)
-
-        model.eval()
-        self.model = model
-
-    def extract_audio_embedding(self, audio_path):
-        """Extract the audio embedding
-
-        Args:
-            audio_path (str): audio path, which will be extracted the embedding
-
-        Returns:
-            embedding (numpy.array) : audio embedding
-        """
-        waveform, sr = load_audio(audio_path)
-        feat = melspectrogram(x=waveform,
-                sr=self.config.sr,
-                n_mels=self.config.n_mels,
-                window_size=self.config.window_size,
-                hop_length=self.config.hop_size)
-        # conver the audio feat to batch shape, which means batch_size is equal to one
-        feat = paddle.to_tensor(feat).unsqueeze(0)
-
-        # in inference period, the lengths is all one without padding
-        lengths = paddle.ones([1])
-        feat = feature_normalize(feat, mean_norm=True, std_norm=False)
-        
-        # model backbone network forward the feats and get the embedding
-        embedding = self.model.backbone(feat, lengths).squeeze().numpy()  # (1, emb_size, 1) -> (emb_size)
-
-        return embedding
 
 def extract_audio_embedding(args, config):
     # stage 0: set the training device, cpu or gpu
@@ -168,7 +83,7 @@ def extract_audio_embedding(args, config):
     # stage 5: do global norm with external mean and std
     rtf = elapsed_time / audio_length
     logger.info(f"{args.device} rft={rtf}")
-    paddle.save(embedding, "emb1")
+
     return embedding
 
 
@@ -177,7 +92,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(__doc__)
     parser.add_argument('--device',
                         choices=['cpu', 'gpu'],
-                        default="gpu",
+                        default="cpu",
                         help="Select which device to train model, defaults to gpu.")
     parser.add_argument("--config",
                         default=None,
@@ -202,10 +117,3 @@ if __name__ == "__main__":
     print(config)
 
     extract_audio_embedding(args, config)
-
-    # use the VectorWrapper to extract the audio embedding
-    vector_inst = VectorWrapper(device="gpu", 
-                    config_path=args.config, 
-                    model_path=args.load_checkpoint)
-    
-    embedding = vector_inst.extract_audio_embedding(args.audio_path)
diff --git a/tests/unit/cli/test_cli.sh b/tests/unit/cli/test_cli.sh
index fc3b8248..f2437eaf 100755
--- a/tests/unit/cli/test_cli.sh
+++ b/tests/unit/cli/test_cli.sh
@@ -1,49 +1,58 @@
 #!/bin/bash
 set -e
-# Audio classification
-wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/cat.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/dog.wav
-paddlespeech cls --input ./cat.wav --topk 10
-
-# Punctuation_restoration
-paddlespeech text --input 今天的天气真不错啊你下午有空吗我想约你一起去吃饭
-
-# Speech_recognition
-wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/zh.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/en.wav
-paddlespeech asr --input ./zh.wav
-paddlespeech asr --model transformer_librispeech --lang en --input ./en.wav
-
-# Text To Speech
-paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！"
-paddlespeech tts --am speedyspeech_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-paddlespeech tts --voc mb_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-paddlespeech tts --voc style_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-paddlespeech tts --voc hifigan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-paddlespeech tts --am fastspeech2_aishell3 --voc pwgan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
-paddlespeech tts --am fastspeech2_aishell3 --voc hifigan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
-paddlespeech tts --am fastspeech2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
-paddlespeech tts --am fastspeech2_ljspeech --voc hifigan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
-paddlespeech tts --am fastspeech2_vctk --voc pwgan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
-paddlespeech tts --am fastspeech2_vctk --voc hifigan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
-paddlespeech tts --am tacotron2_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-paddlespeech tts --am tacotron2_csmsc --voc wavernn_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-paddlespeech tts --am tacotron2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
-
-
-# Speech Translation (only support linux)
-paddlespeech st --input ./en.wav
-
-
-# batch process
-echo -e "1 欢迎光临。\n2 谢谢惠顾。" | paddlespeech tts
-
-# shell pipeline
-paddlespeech asr --input ./zh.wav | paddlespeech text --task punc
-
-# stats
-paddlespeech stats --task asr
-paddlespeech stats --task tts
-paddlespeech stats --task cls
+# # Audio classification
+# wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/cat.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/dog.wav
+# paddlespeech cls --input ./cat.wav --topk 10
+
+# # Punctuation_restoration
+# paddlespeech text --input 今天的天气真不错啊你下午有空吗我想约你一起去吃饭
+
+# # Speech_recognition
+# wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/zh.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/en.wav
+# paddlespeech asr --input ./zh.wav
+# paddlespeech asr --model transformer_librispeech --lang en --input ./en.wav
+
+# # Text To Speech
+# paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！"
+# paddlespeech tts --am speedyspeech_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+# paddlespeech tts --voc mb_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+# paddlespeech tts --voc style_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+# paddlespeech tts --voc hifigan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+# paddlespeech tts --am fastspeech2_aishell3 --voc pwgan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
+# paddlespeech tts --am fastspeech2_aishell3 --voc hifigan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
+# paddlespeech tts --am fastspeech2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
+# paddlespeech tts --am fastspeech2_ljspeech --voc hifigan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
+# paddlespeech tts --am fastspeech2_vctk --voc pwgan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
+# paddlespeech tts --am fastspeech2_vctk --voc hifigan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
+# paddlespeech tts --am tacotron2_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+# paddlespeech tts --am tacotron2_csmsc --voc wavernn_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+# paddlespeech tts --am tacotron2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
+
+
+# # Speech Translation (only support linux)
+# paddlespeech st --input ./en.wav
+
+
+# # batch process
+# echo -e "1 欢迎光临。\n2 谢谢惠顾。" | paddlespeech tts
+
+# # shell pipeline
+# paddlespeech asr --input ./zh.wav | paddlespeech text --task punc
+
+# # stats
+# paddlespeech stats --task asr
+# paddlespeech stats --task tts
+# paddlespeech stats --task cls
 
 # Speaker Verification 
 wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
 paddlespeech vector --task spk --input 85236145389.wav
+
+echo "demo 85236145389.wav" > vec.job
+paddlespeech vector --task spk --input vec.job
+
+echo "demo 85236145389.wav" | paddlespeech vector --task spk
+rm 85236145389.wav 
+rm vec.job
+
+

From 3054659901ebb198ad8556500e38bb7558b22653 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 24 Mar 2022 18:59:36 +0800
Subject: [PATCH 060/126] remove debug info, test=doc

---
 tests/unit/cli/test_cli.sh | 86 +++++++++++++++++++-------------------
 1 file changed, 43 insertions(+), 43 deletions(-)

diff --git a/tests/unit/cli/test_cli.sh b/tests/unit/cli/test_cli.sh
index f2437eaf..6fbb1570 100755
--- a/tests/unit/cli/test_cli.sh
+++ b/tests/unit/cli/test_cli.sh
@@ -1,48 +1,48 @@
 #!/bin/bash
 set -e
-# # Audio classification
-# wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/cat.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/dog.wav
-# paddlespeech cls --input ./cat.wav --topk 10
-
-# # Punctuation_restoration
-# paddlespeech text --input 今天的天气真不错啊你下午有空吗我想约你一起去吃饭
-
-# # Speech_recognition
-# wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/zh.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/en.wav
-# paddlespeech asr --input ./zh.wav
-# paddlespeech asr --model transformer_librispeech --lang en --input ./en.wav
-
-# # Text To Speech
-# paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！"
-# paddlespeech tts --am speedyspeech_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-# paddlespeech tts --voc mb_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-# paddlespeech tts --voc style_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-# paddlespeech tts --voc hifigan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-# paddlespeech tts --am fastspeech2_aishell3 --voc pwgan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
-# paddlespeech tts --am fastspeech2_aishell3 --voc hifigan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
-# paddlespeech tts --am fastspeech2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
-# paddlespeech tts --am fastspeech2_ljspeech --voc hifigan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
-# paddlespeech tts --am fastspeech2_vctk --voc pwgan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
-# paddlespeech tts --am fastspeech2_vctk --voc hifigan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
-# paddlespeech tts --am tacotron2_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-# paddlespeech tts --am tacotron2_csmsc --voc wavernn_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
-# paddlespeech tts --am tacotron2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
-
-
-# # Speech Translation (only support linux)
-# paddlespeech st --input ./en.wav
-
-
-# # batch process
-# echo -e "1 欢迎光临。\n2 谢谢惠顾。" | paddlespeech tts
-
-# # shell pipeline
-# paddlespeech asr --input ./zh.wav | paddlespeech text --task punc
-
-# # stats
-# paddlespeech stats --task asr
-# paddlespeech stats --task tts
-# paddlespeech stats --task cls
+# Audio classification
+wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/cat.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/dog.wav
+paddlespeech cls --input ./cat.wav --topk 10
+
+# Punctuation_restoration
+paddlespeech text --input 今天的天气真不错啊你下午有空吗我想约你一起去吃饭
+
+# Speech_recognition
+wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/zh.wav https://paddlespeech.bj.bcebos.com/PaddleAudio/en.wav
+paddlespeech asr --input ./zh.wav
+paddlespeech asr --model transformer_librispeech --lang en --input ./en.wav
+
+# Text To Speech
+paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！"
+paddlespeech tts --am speedyspeech_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+paddlespeech tts --voc mb_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+paddlespeech tts --voc style_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+paddlespeech tts --voc hifigan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+paddlespeech tts --am fastspeech2_aishell3 --voc pwgan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
+paddlespeech tts --am fastspeech2_aishell3 --voc hifigan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0
+paddlespeech tts --am fastspeech2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
+paddlespeech tts --am fastspeech2_ljspeech --voc hifigan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
+paddlespeech tts --am fastspeech2_vctk --voc pwgan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
+paddlespeech tts --am fastspeech2_vctk --voc hifigan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0
+paddlespeech tts --am tacotron2_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+paddlespeech tts --am tacotron2_csmsc --voc wavernn_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！"
+paddlespeech tts --am tacotron2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get."
+
+
+# Speech Translation (only support linux)
+paddlespeech st --input ./en.wav
+
+
+# batch process
+echo -e "1 欢迎光临。\n2 谢谢惠顾。" | paddlespeech tts
+
+# shell pipeline
+paddlespeech asr --input ./zh.wav | paddlespeech text --task punc
+
+# stats
+paddlespeech stats --task asr
+paddlespeech stats --task tts
+paddlespeech stats --task cls
 
 # Speaker Verification 
 wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav

From faf6b8daf855fd06d96f80b2d592579ba4b61ab5 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Thu, 24 Mar 2022 19:19:05 +0800
Subject: [PATCH 061/126] add the vec cli test audio name, test=doc

---
 tests/unit/cli/test_cli.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unit/cli/test_cli.sh b/tests/unit/cli/test_cli.sh
index 6fbb1570..96ab84d6 100755
--- a/tests/unit/cli/test_cli.sh
+++ b/tests/unit/cli/test_cli.sh
@@ -48,10 +48,10 @@ paddlespeech stats --task cls
 wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
 paddlespeech vector --task spk --input 85236145389.wav
 
-echo "demo 85236145389.wav" > vec.job
+echo -e "demo1 85236145389.wav \n demo2 85236145389.wav" > vec.job
 paddlespeech vector --task spk --input vec.job
 
-echo "demo 85236145389.wav" | paddlespeech vector --task spk
+echo -e "demo3 85236145389.wav \n demo4 85236145389.wav" | paddlespeech vector --task spk
 rm 85236145389.wav 
 rm vec.job
 

From 965b388c221b86323e4667f2f958bea604930448 Mon Sep 17 00:00:00 2001
From: lym0302 <lym0302@foxmail.com>
Date: Thu, 24 Mar 2022 19:27:32 +0800
Subject: [PATCH 062/126] update readme, test=doc

---
 README.md    | 31 +++++++++++++++++++++++++++++++
 README_cn.md | 28 ++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/README.md b/README.md
index ceef15af..f1570b4a 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@
 
   <h3>
   <a href="#quick-start"> Quick Start </a>
+  | <a href="#quick-start-server"> Quick Start Server </a>
   | <a href="#documents"> Documents </a>
   | <a href="#model-list"> Models List </a>
 </div>
@@ -242,6 +243,36 @@ For more command lines, please see: [demos](https://github.com/PaddlePaddle/Padd
 
 If you want to try more functions like training and tuning, please have a look at [Speech-to-Text Quick Start](./docs/source/asr/quick_start.md) and [Text-to-Speech Quick Start](./docs/source/tts/quick_start.md).
 
+
+<a name="quickstartserver"></a>
+## Quick Start Server
+
+Developers can have a try of our speech server with [PaddleSpeech Server Command Line](./paddlespeech/server/README.md).
+
+**Start server**     
+```shell
+paddlespeech_server start --config_file ./paddlespeech/server/conf/application.yaml
+```
+
+**Access Speech Recognition Services**     
+```shell
+paddlespeech_client asr --server_ip 127.0.0.1 --port 8090 --input input_16k.wav
+```
+
+**Access Text to Speech Services**     
+```shell
+paddlespeech_client tts --server_ip 127.0.0.1 --port 8090 --input "您好，欢迎使用百度飞桨语音合成服务。" --output output.wav
+```
+
+**Access Audio Classification Services**     
+```shell
+paddlespeech_client cls --server_ip 127.0.0.1 --port 8090 --input input.wav
+```
+
+
+For more information about server command lines, please see: [speech server demos](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/demos/speech_server)
+
+
 ## Model List
 
 PaddleSpeech supports a series of most popular models. They are summarized in [released models](./docs/source/released_model.md) and attached with available pretrained models.
diff --git a/README_cn.md b/README_cn.md
index 8ea91e98..70f6b2d9 100644
--- a/README_cn.md
+++ b/README_cn.md
@@ -6,6 +6,7 @@
 
   <h3>
   <a href="#quick-start"> 快速开始 </a>
+  | <a href="#quick-start-server"> 快速使用服务 </a>
   | <a href="#documents"> 教程文档 </a>
   | <a href="#model-list"> 模型列表 </a>
 </div>
@@ -236,6 +237,33 @@ paddlespeech asr --input ./zh.wav | paddlespeech text --task punc
 更多命令行命令请参考 [demos](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/demos)
 > Note: 如果需要训练或者微调，请查看[语音识别](./docs/source/asr/quick_start.md)， [语音合成](./docs/source/tts/quick_start.md)。
 
+
+## 快速使用服务
+安装完成后，开发者可以通过命令行快速使用服务。
+
+**启动服务**     
+```shell
+paddlespeech_server start --config_file ./paddlespeech/server/conf/application.yaml
+```
+
+**访问语音识别服务**     
+```shell
+paddlespeech_client asr --server_ip 127.0.0.1 --port 8090 --input input_16k.wav
+```
+
+**访问语音合成服务**     
+```shell
+paddlespeech_client tts --server_ip 127.0.0.1 --port 8090 --input "您好，欢迎使用百度飞桨语音合成服务。" --output output.wav
+```
+
+**访问音频分类服务**     
+```shell
+paddlespeech_client cls --server_ip 127.0.0.1 --port 8090 --input input.wav
+```
+
+更多服务相关的命令行使用信息，请参考 [demos](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/demos/speech_server)
+
+
 ## 模型列表
 PaddleSpeech 支持很多主流的模型，并提供了预训练模型，详情请见[模型列表](./docs/source/released_model.md)。
 

From 829f1e332eaa9d788df112d333749e83bf184f5e Mon Sep 17 00:00:00 2001
From: lym0302 <lym0302@foxmail.com>
Date: Thu, 24 Mar 2022 19:39:35 +0800
Subject: [PATCH 063/126] update readme, test=doc

---
 paddlespeech/server/README.md    | 8 ++++++--
 paddlespeech/server/README_cn.md | 7 ++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/paddlespeech/server/README.md b/paddlespeech/server/README.md
index 4ce9605d..819fe440 100644
--- a/paddlespeech/server/README.md
+++ b/paddlespeech/server/README.md
@@ -10,7 +10,7 @@
  paddlespeech_server help
  ```
  ### Start the server
- First set the service-related configuration parameters, similar to `./conf/application.yaml`,
+ First set the service-related configuration parameters, similar to `./conf/application.yaml`. Set `engine_list`, which represents the speech tasks included in the service to be started
  Then start the service:
  ```bash
  paddlespeech_server start --config_file ./conf/application.yaml
@@ -23,7 +23,7 @@
  ```
  ### Access speech recognition services 
  ```
- paddlespeech_client asr --server_ip 127.0.0.1 --port 8090 --input ./tests/16_audio.wav
+ paddlespeech_client asr --server_ip 127.0.0.1 --port 8090 --input input_16k.wav
  ```
  
  ### Access text to speech services
@@ -31,3 +31,7 @@
  paddlespeech_client tts --server_ip 127.0.0.1 --port 8090 --input "你好，欢迎使用百度飞桨深度学习框架！" --output output.wav
  ```
  
+ ### Access audio classification services
+ ```bash
+ paddlespeech_client cls --server_ip 127.0.0.1 --port 8090 --input input.wav
+ ```
diff --git a/paddlespeech/server/README_cn.md b/paddlespeech/server/README_cn.md
index 2dfd9474..c0a4a733 100644
--- a/paddlespeech/server/README_cn.md
+++ b/paddlespeech/server/README_cn.md
@@ -10,7 +10,7 @@
  paddlespeech_server help
  ```
  ### 启动服务
- 首先设置服务相关配置文件，类似于 `./conf/application.yaml`，同时设置服务配置中的语音任务模型相关配置，类似于 `./conf/tts/tts.yaml`。
+ 首先设置服务相关配置文件，类似于 `./conf/application.yaml`，设置 `engine_list`，该值表示即将启动的服务中包含的语音任务。
  然后启动服务：
  ```bash
  paddlespeech_server start --config_file ./conf/application.yaml
@@ -30,3 +30,8 @@
  ```bash
  paddlespeech_client tts --server_ip 127.0.0.1 --port 8090 --input "你好，欢迎使用百度飞桨深度学习框架！" --output output.wav
  ```
+
+ ### 访问音频分类服务
+ ```bash
+ paddlespeech_client cls --server_ip 127.0.0.1 --port 8090 --input input.wav
+ ```

From 754ecabac493e34f4fe42ec24c1d73af44c16f47 Mon Sep 17 00:00:00 2001
From: lym0302 <lym0302@foxmail.com>
Date: Thu, 24 Mar 2022 20:01:31 +0800
Subject: [PATCH 064/126] update readme, test=doc

---
 demos/speech_server/README.md    | 4 ++--
 demos/speech_server/README_cn.md | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/demos/speech_server/README.md b/demos/speech_server/README.md
index 10489e71..66bf11ca 100644
--- a/demos/speech_server/README.md
+++ b/demos/speech_server/README.md
@@ -15,8 +15,8 @@ You can choose one way from meduim and hard to install paddlespeech.
 
 ### 2. Prepare config File
 The configuration file can be found in `conf/application.yaml` .
-Among them, `engine_list` indicates the speech engine that will be included in the service to be started, in the format of <speech task>_<engine type>.
-At present, the speech tasks integrated by the service include: asr (speech recognition) and tts (speech synthesis).
+Among them, `engine_list` indicates the speech engine that will be included in the service to be started, in the format of speech task_engine type.
+At present, the speech tasks integrated by the service include: asr (speech recognition), tts (text to sppech) and cls (audio classification).
 Currently the engine type supports two forms: python and inference (Paddle Inference)
 
 
diff --git a/demos/speech_server/README_cn.md b/demos/speech_server/README_cn.md
index 2bd8af6c..687b51f1 100644
--- a/demos/speech_server/README_cn.md
+++ b/demos/speech_server/README_cn.md
@@ -17,7 +17,7 @@
 ### 2. 准备配置文件
 配置文件可参见 `conf/application.yaml` 。
 其中，`engine_list`表示即将启动的服务将会包含的语音引擎，格式为 <语音任务>_<引擎类型>。
-目前服务集成的语音任务有： asr(语音识别)、tts(语音合成)。
+目前服务集成的语音任务有： asr(语音识别)、tts(语音合成)以及cls(音频分类)。
 目前引擎类型支持两种形式：python 及 inference (Paddle Inference)
 
 

From ec8c870f85e06ea5cc3625125d9df94795ec99ae Mon Sep 17 00:00:00 2001
From: lym0302 <lym0302@foxmail.com>
Date: Thu, 24 Mar 2022 20:03:38 +0800
Subject: [PATCH 065/126] update readme, test=doc

---
 demos/speech_server/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/demos/speech_server/README.md b/demos/speech_server/README.md
index 66bf11ca..0323d398 100644
--- a/demos/speech_server/README.md
+++ b/demos/speech_server/README.md
@@ -15,7 +15,7 @@ You can choose one way from meduim and hard to install paddlespeech.
 
 ### 2. Prepare config File
 The configuration file can be found in `conf/application.yaml` .
-Among them, `engine_list` indicates the speech engine that will be included in the service to be started, in the format of speech task_engine type.
+Among them, `engine_list` indicates the speech engine that will be included in the service to be started, in the format of `<speech task>_<engine type>`.
 At present, the speech tasks integrated by the service include: asr (speech recognition), tts (text to sppech) and cls (audio classification).
 Currently the engine type supports two forms: python and inference (Paddle Inference)
 

From ad2caf2ccba6e18fee2a360d6449b3816ed8e71a Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Fri, 25 Mar 2022 18:31:48 +0800
Subject: [PATCH 066/126] add speaker verification demo and doc, test=doc

---
 README.md                               |  29 +++++
 README_cn.md                            |  29 +++++
 demos/speaker_verification/README.md    | 158 ++++++++++++++++++++++++
 demos/speaker_verification/README_cn.md | 156 +++++++++++++++++++++++
 demos/speaker_verification/run.sh       |   6 +
 docs/source/released_model.md           |   8 +-
 examples/voxceleb/sv0/RESULT.md         |   8 ++
 paddlespeech/cli/README.md              |   6 +
 paddlespeech/cli/README_cn.md           |   6 +
 9 files changed, 405 insertions(+), 1 deletion(-)
 create mode 100644 demos/speaker_verification/README.md
 create mode 100644 demos/speaker_verification/README_cn.md
 create mode 100644 demos/speaker_verification/run.sh
 create mode 100644 examples/voxceleb/sv0/RESULT.md

diff --git a/README.md b/README.md
index ceef15af..cb2b1227 100644
--- a/README.md
+++ b/README.md
@@ -203,6 +203,11 @@ Developers can have a try of our models with [PaddleSpeech Command Line](./paddl
 paddlespeech cls --input input.wav
 ```
 
+**Speaker Verification**
+```
+paddlespeech vector --task spk --input input_16k.wav
+```
+
 **Automatic Speech Recognition**
 ```shell
 paddlespeech asr --lang zh --input input_16k.wav
@@ -458,6 +463,29 @@ PaddleSpeech supports a series of most popular models. They are summarized in [r
   </tbody>
 </table>
 
+**Speaker Verification**
+
+<table style="width:100%">
+  <thead>
+    <tr>
+      <th> Task </th>
+      <th> Dataset </th>
+      <th> Model Type </th>
+      <th> Link </th>
+    </tr>
+  </thead>
+  <tbody>
+  <tr>
+      <td>Speaker Verification</td>
+      <td>VoxCeleb12</td>
+      <td>ECAPA-TDNN</td>
+      <td>
+      <a href = "./examples/voxceleb/sv0">ecapa-tdnn-voxceleb12</a>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
 **Punctuation Restoration**
 
 <table style="width:100%">
@@ -499,6 +527,7 @@ Normally, [Speech SoTA](https://paperswithcode.com/area/speech), [Audio SoTA](ht
     - [Chinese Rule Based Text Frontend](./docs/source/tts/zh_text_frontend.md)
     - [Test Audio Samples](https://paddlespeech.readthedocs.io/en/latest/tts/demo.html)
   - [Audio Classification](./demos/audio_tagging/README.md)
+  - [Speaker Verification](./demos/speaker_verification/README.md)
   - [Speech Translation](./demos/speech_translation/README.md)
 - [Released Models](./docs/source/released_model.md)
 - [Community](#Community)
diff --git a/README_cn.md b/README_cn.md
index 8ea91e98..4d88ab8b 100644
--- a/README_cn.md
+++ b/README_cn.md
@@ -202,6 +202,10 @@ from https://github.com/18F/open-source-guide/blob/18f-pages/pages/making-readme
 ```shell
 paddlespeech cls --input input.wav
 ```
+**声纹识别**
+```shell
+paddlespeech vector --task spk --input input_16k.wav
+```
 **语音识别**
 ```shell
 paddlespeech asr --lang zh --input input_16k.wav
@@ -453,6 +457,30 @@ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声
   </tbody>
 </table>
 
+
+**声纹识别**
+
+<table style="width:100%">
+  <thead>
+    <tr>
+      <th> Task </th>
+      <th> Dataset </th>
+      <th> Model Type </th>
+      <th> Link </th>
+    </tr>
+  </thead>
+  <tbody>
+  <tr>
+      <td>Speaker Verification</td>
+      <td>VoxCeleb12</td>
+      <td>ECAPA-TDNN</td>
+      <td>
+      <a href = "./examples/voxceleb/sv0">ecapa-tdnn-voxceleb12</a>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
 **标点恢复**
 
 <table style="width:100%">
@@ -499,6 +527,7 @@ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声
     - [中文文本前端](./docs/source/tts/zh_text_frontend.md)
     - [测试语音样本](https://paddlespeech.readthedocs.io/en/latest/tts/demo.html)
   - [声音分类](./demos/audio_tagging/README_cn.md)
+  - [声纹识别](./demos/speaker_verification/README_cn.md)
   - [语音翻译](./demos/speech_translation/README_cn.md)
 - [模型列表](#模型列表)
   - [语音识别](#语音识别模型)
diff --git a/demos/speaker_verification/README.md b/demos/speaker_verification/README.md
new file mode 100644
index 00000000..b1dfbc7c
--- /dev/null
+++ b/demos/speaker_verification/README.md
@@ -0,0 +1,158 @@
+([简体中文](./README_cn.md)|English)
+# Speech Verification)
+
+## Introduction
+
+Speaker Verification, refers to the problem of getting a speaker embedding from an audio. 
+
+This demo is an implementation to extract speaker embedding from a specific audio file. It can be done by a single command or a few lines in python using `PaddleSpeech`. 
+
+## Usage
+### 1. Installation
+see [installation](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/docs/source/install.md).
+
+You can choose one way from easy, meduim and hard to install paddlespeech.
+
+### 2. Prepare Input File
+The input of this demo should be a WAV file(`.wav`), and the sample rate must be the same as the model.
+
+Here are sample files for this demo that can be downloaded:
+```bash
+wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
+```
+
+### 3. Usage
+- Command Line(Recommended)
+  ```bash
+  paddlespeech vector --task spk --input 85236145389.wav
+
+  echo -e "demo1 85236145389.wav" > vec.job
+  paddlespeech vector --task spk --input vec.job
+
+  echo -e "demo2 85236145389.wav \n demo3 85236145389.wav" | paddlespeech vector --task spk
+  ```
+  
+  Usage:
+  ```bash
+  paddlespeech asr --help
+  ```
+  Arguments:
+  - `input`(required): Audio file to recognize.
+  - `model`: Model type of asr task. Default: `conformer_wenetspeech`.
+  - `sample_rate`: Sample rate of the model. Default: `16000`.
+  - `config`: Config of asr task. Use pretrained model when it is None. Default: `None`.
+  - `ckpt_path`: Model checkpoint. Use pretrained model when it is None. Default: `None`.
+  - `device`: Choose device to execute model inference. Default: default device of paddlepaddle in current environment.
+
+  Output:
+
+```bash
+  demo [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+  -3.04878      1.611095    10.127234   -10.534177   -15.821609
+   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+ -11.343508     2.3385992   -8.719341    14.213509    15.404744
+  -0.39327756   6.338786     2.688887     8.7104025   17.469526
+  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+   8.013747    13.891729    -9.926753     5.655307    -5.9422326
+ -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+  -6.40137     23.63524      2.9711294  -22.708025     9.93719
+  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+  15.999649     3.3004563   12.747926    15.429879     4.7849145
+   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+  -9.224193    14.568347   -10.568833     4.982321    -4.342062
+   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+ -11.54324      7.681869     0.44475392   9.708182    -8.932846
+   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+   2.9079323    6.049952     9.275183   -18.078873     6.2983274
+  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+  18.495346   -14.293832     7.89578      2.2714825   22.976387
+  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+ -11.924197     2.171869     2.0423572   -6.173772    10.778437
+  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+ -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+  10.833007    -6.717991     4.504732    13.4244375    1.1306485
+   7.3435574    1.400918    14.704036    -9.501399     7.2315617
+  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+  -3.2701402  -11.508579  ]
+  ```
+
+- Python API
+  ```python
+  import paddle
+  from paddlespeech.cli import VectorExecutor
+
+  vector_executor = VectorExecutor()
+  audio_emb = vector_executor(
+      model='ecapatdnn_voxceleb12',
+      sample_rate=16000,
+      config=None, 
+      ckpt_path=None,
+      audio_file='./85236145389.wav',
+      force_yes=False,
+      device=paddle.get_device())
+  print('Audio embedding Result: \n{}'.format(audio_emb))
+  ```
+
+  Output:
+  ```bash
+  # Vector Result:
+  [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+  -3.04878      1.611095    10.127234   -10.534177   -15.821609
+   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+ -11.343508     2.3385992   -8.719341    14.213509    15.404744
+  -0.39327756   6.338786     2.688887     8.7104025   17.469526
+  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+   8.013747    13.891729    -9.926753     5.655307    -5.9422326
+ -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+  -6.40137     23.63524      2.9711294  -22.708025     9.93719
+  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+  15.999649     3.3004563   12.747926    15.429879     4.7849145
+   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+  -9.224193    14.568347   -10.568833     4.982321    -4.342062
+   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+ -11.54324      7.681869     0.44475392   9.708182    -8.932846
+   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+   2.9079323    6.049952     9.275183   -18.078873     6.2983274
+  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+  18.495346   -14.293832     7.89578      2.2714825   22.976387
+  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+ -11.924197     2.171869     2.0423572   -6.173772    10.778437
+  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+ -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+  10.833007    -6.717991     4.504732    13.4244375    1.1306485
+   7.3435574    1.400918    14.704036    -9.501399     7.2315617
+  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+  -3.2701402  -11.508579  ]
+  ```
+
+### 4.Pretrained Models
+
+Here is a list of pretrained models released by PaddleSpeech that can be used by command and python API:
+
+| Model | Sample Rate
+| :--- | :---: |
+| ecapatdnn_voxceleb12 | 16k
diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md
new file mode 100644
index 00000000..dd7a39fb
--- /dev/null
+++ b/demos/speaker_verification/README_cn.md
@@ -0,0 +1,156 @@
+(简体中文|[English](./README.md))
+
+# 声纹识别
+## 介绍
+声纹识别是一项用计算机程序自动提取说话人特征的技术。
+
+这个 demo 是一个从给定音频文件提取说话人特征，它可以通过使用 `PaddleSpeech` 的单个命令或 python 中的几行代码来实现。
+
+## 使用方法
+### 1. 安装
+请看[安装文档](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/docs/source/install_cn.md)。
+
+你可以从 easy，medium，hard 三中方式中选择一种方式安装。
+
+### 2. 准备输入
+这个 demo 的输入应该是一个 WAV 文件（`.wav`），并且采样率必须与模型的采样率相同。
+
+可以下载此 demo 的示例音频：
+```bash
+# 该音频的内容是数字串 85236145389
+wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
+```
+### 3. 使用方法
+- 命令行 (推荐使用)
+  ```bash
+  paddlespeech vector --task spk --input 85236145389.wav
+
+  echo -e "demo1 85236145389.wav" > vec.job
+  paddlespeech vector --task spk --input vec.job
+
+  echo -e "demo2 85236145389.wav \n demo3 85236145389.wav" | paddlespeech vector --task spk
+  ```
+  
+  使用方法：
+  ```bash
+  paddlespeech asr --help
+  ```
+  参数：
+  - `input`(必须输入)：用于识别的音频文件。
+  - `model`：声纹任务的模型，默认值：`ecapatdnn_voxceleb12`。
+  - `sample_rate`：音频采样率，默认值：`16000`。
+  - `config`：声纹任务的参数文件，若不设置则使用预训练模型中的默认配置，默认值：`None`。
+  - `ckpt_path`：模型参数文件，若不设置则下载预训练模型使用，默认值：`None`。
+  - `device`：执行预测的设备，默认值：当前系统下 paddlepaddle 的默认 device。
+
+  输出：
+  ```bash
+  demo [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+  -3.04878      1.611095    10.127234   -10.534177   -15.821609
+   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+ -11.343508     2.3385992   -8.719341    14.213509    15.404744
+  -0.39327756   6.338786     2.688887     8.7104025   17.469526
+  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+   8.013747    13.891729    -9.926753     5.655307    -5.9422326
+ -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+  -6.40137     23.63524      2.9711294  -22.708025     9.93719
+  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+  15.999649     3.3004563   12.747926    15.429879     4.7849145
+   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+  -9.224193    14.568347   -10.568833     4.982321    -4.342062
+   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+ -11.54324      7.681869     0.44475392   9.708182    -8.932846
+   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+   2.9079323    6.049952     9.275183   -18.078873     6.2983274
+  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+  18.495346   -14.293832     7.89578      2.2714825   22.976387
+  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+ -11.924197     2.171869     2.0423572   -6.173772    10.778437
+  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+ -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+  10.833007    -6.717991     4.504732    13.4244375    1.1306485
+   7.3435574    1.400918    14.704036    -9.501399     7.2315617
+  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+  -3.2701402  -11.508579  ]
+  ```
+
+- Python API
+  ```python
+  import paddle
+  from paddlespeech.cli import VectorExecutor
+
+  vector_executor = VectorExecutor()
+  audio_emb = vector_executor(
+      model='ecapatdnn_voxceleb12',
+      sample_rate=16000,
+      config=None,  # Set `config` and `ckpt_path` to None to use pretrained model.
+      ckpt_path=None,
+      audio_file='./zh.wav',
+      force_yes=False,
+      device=paddle.get_device())
+  print('Audio embedding Result: \n{}'.format(audio_emb))
+  ```
+
+  输出：
+  ```bash
+  # Vector Result:
+  [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+  -3.04878      1.611095    10.127234   -10.534177   -15.821609
+   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+ -11.343508     2.3385992   -8.719341    14.213509    15.404744
+  -0.39327756   6.338786     2.688887     8.7104025   17.469526
+  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+   8.013747    13.891729    -9.926753     5.655307    -5.9422326
+ -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+  -6.40137     23.63524      2.9711294  -22.708025     9.93719
+  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+  15.999649     3.3004563   12.747926    15.429879     4.7849145
+   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+  -9.224193    14.568347   -10.568833     4.982321    -4.342062
+   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+ -11.54324      7.681869     0.44475392   9.708182    -8.932846
+   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+   2.9079323    6.049952     9.275183   -18.078873     6.2983274
+  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+  18.495346   -14.293832     7.89578      2.2714825   22.976387
+  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+ -11.924197     2.171869     2.0423572   -6.173772    10.778437
+  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+ -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+  10.833007    -6.717991     4.504732    13.4244375    1.1306485
+   7.3435574    1.400918    14.704036    -9.501399     7.2315617
+  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+  -3.2701402  -11.508579  ]
+  ```
+
+### 4.预训练模型
+以下是 PaddleSpeech 提供的可以被命令行和 python API 使用的预训练模型列表：
+
+| 模型 | 采样率
+| :--- | :---: |
+| ecapatdnn_voxceleb12 | 16k
+
diff --git a/demos/speaker_verification/run.sh b/demos/speaker_verification/run.sh
new file mode 100644
index 00000000..856886d3
--- /dev/null
+++ b/demos/speaker_verification/run.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
+
+# asr
+paddlespeech vector --task spk --input ./85236145389.wav
\ No newline at end of file
diff --git a/docs/source/released_model.md b/docs/source/released_model.md
index c5c65c82..354ecf30 100644
--- a/docs/source/released_model.md
+++ b/docs/source/released_model.md
@@ -75,10 +75,16 @@ Model Type | Dataset| Example Link | Pretrained Models | Static Models
 PANN | Audioset| [audioset_tagging_cnn](https://github.com/qiuqiangkong/audioset_tagging_cnn) | [panns_cnn6.pdparams](https://bj.bcebos.com/paddleaudio/models/panns_cnn6.pdparams), [panns_cnn10.pdparams](https://bj.bcebos.com/paddleaudio/models/panns_cnn10.pdparams), [panns_cnn14.pdparams](https://bj.bcebos.com/paddleaudio/models/panns_cnn14.pdparams) | [panns_cnn6_static.tar.gz](https://paddlespeech.bj.bcebos.com/cls/inference_model/panns_cnn6_static.tar.gz)(18M), [panns_cnn10_static.tar.gz](https://paddlespeech.bj.bcebos.com/cls/inference_model/panns_cnn10_static.tar.gz)(19M), [panns_cnn14_static.tar.gz](https://paddlespeech.bj.bcebos.com/cls/inference_model/panns_cnn14_static.tar.gz)(289M) 
 PANN | ESC-50 |[pann-esc50](../../examples/esc50/cls0)|[esc50_cnn6.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn6.tar.gz), [esc50_cnn10.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn10.tar.gz), [esc50_cnn14.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn14.tar.gz)
 
+## Speaker Verification Models
+
+Model Type | Dataset| Example Link | Pretrained Models | Static Models 
+:-------------:| :------------:| :-----: | :-----: | :-----:
+PANN | VoxCeleb| [voxceleb_ecapatdnn](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/voxceleb/sv0) | [ecapatdnn.tar.gz](https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_1.tar.gz) | -
+
 ## Punctuation Restoration Models
 Model Type | Dataset| Example Link | Pretrained Models
 :-------------:| :------------:| :-----: | :-----:
-Ernie Linear | IWLST2012_zh |[iwslt2012_punc0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/iwslt2012/punc0)|[ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/text/ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip)
+Ernie Linear | IWLST2012_zh |[iwslt2012_punc0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/iwslt2012/punc0)|[ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_1.tar.gz)
 
 ## Speech Recognition Model  from paddle 1.8
 
diff --git a/examples/voxceleb/sv0/RESULT.md b/examples/voxceleb/sv0/RESULT.md
new file mode 100644
index 00000000..3df2af51
--- /dev/null
+++ b/examples/voxceleb/sv0/RESULT.md
@@ -0,0 +1,8 @@
+# VoxCeleb
+
+## ECAPA-TDNN 
+
+| Model | Number of Params | Release | Config | Test set |  Cosine | Cosine + S-Norm | 
+| --- | --- | --- | --- | --- | --- | --- | 
+| ECAPA-TDNN | 85MM | 0.1.1 | conf/model.yaml | test | 1.15 |  1.06 | 
+
diff --git a/paddlespeech/cli/README.md b/paddlespeech/cli/README.md
index 5ac7a3bc..19c82204 100644
--- a/paddlespeech/cli/README.md
+++ b/paddlespeech/cli/README.md
@@ -13,6 +13,12 @@
  paddlespeech cls --input input.wav
  ```
 
+ ## Speaker Verification
+
+ ```bash
+ paddlespeech vector --task spk --input input_16k.wav
+ ```
+
  ## Automatic Speech Recognition
  ```
  paddlespeech asr --lang zh --input input_16k.wav
diff --git a/paddlespeech/cli/README_cn.md b/paddlespeech/cli/README_cn.md
index 75ab9e41..4b15d6c7 100644
--- a/paddlespeech/cli/README_cn.md
+++ b/paddlespeech/cli/README_cn.md
@@ -12,6 +12,12 @@
  ## 声音分类
  ```bash
  paddlespeech cls --input input.wav
+ ```
+
+  ## 声纹识别
+
+ ```bash
+ paddlespeech vector --task spk --input input_16k.wav
  ```
 
  ## 语音识别

From bf0addc4fa758ffcb0535a8335afe8a5f4905f3e Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Fri, 25 Mar 2022 19:28:39 +0800
Subject: [PATCH 067/126] remove incorrect doc, test=doc

---
 demos/speaker_verification/README.md    | 144 ++++++++++++------------
 demos/speaker_verification/README_cn.md | 144 ++++++++++++------------
 docs/source/released_model.md           |   2 +-
 3 files changed, 145 insertions(+), 145 deletions(-)

diff --git a/demos/speaker_verification/README.md b/demos/speaker_verification/README.md
index b1dfbc7c..6e8f9693 100644
--- a/demos/speaker_verification/README.md
+++ b/demos/speaker_verification/README.md
@@ -50,42 +50,42 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   demo [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
   -3.04878      1.611095    10.127234   -10.534177   -15.821609
    1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
- -11.343508     2.3385992   -8.719341    14.213509    15.404744
-  -0.39327756   6.338786     2.688887     8.7104025   17.469526
-  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-   8.013747    13.891729    -9.926753     5.655307    -5.9422326
- -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-  -6.40137     23.63524      2.9711294  -22.708025     9.93719
-  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-  15.999649     3.3004563   12.747926    15.429879     4.7849145
-   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-  -9.224193    14.568347   -10.568833     4.982321    -4.342062
-   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
- -11.54324      7.681869     0.44475392   9.708182    -8.932846
-   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-   2.9079323    6.049952     9.275183   -18.078873     6.2983274
-  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-  18.495346   -14.293832     7.89578      2.2714825   22.976387
-  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
- -11.924197     2.171869     2.0423572   -6.173772    10.778437
-  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
- -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-  10.833007    -6.717991     4.504732    13.4244375    1.1306485
-   7.3435574    1.400918    14.704036    -9.501399     7.2315617
-  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-  -3.2701402  -11.508579  ]
+  -11.343508     2.3385992   -8.719341    14.213509    15.404744
+    -0.39327756   6.338786     2.688887     8.7104025   17.469526
+    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+    8.013747    13.891729    -9.926753     5.655307    -5.9422326
+  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+    -6.40137     23.63524      2.9711294  -22.708025     9.93719
+    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+    15.999649     3.3004563   12.747926    15.429879     4.7849145
+    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+    -9.224193    14.568347   -10.568833     4.982321    -4.342062
+    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+  -11.54324      7.681869     0.44475392   9.708182    -8.932846
+    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+    2.9079323    6.049952     9.275183   -18.078873     6.2983274
+    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+    18.495346   -14.293832     7.89578      2.2714825   22.976387
+    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+  -11.924197     2.171869     2.0423572   -6.173772    10.778437
+    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+    10.833007    -6.717991     4.504732    13.4244375    1.1306485
+    7.3435574    1.400918    14.704036    -9.501399     7.2315617
+    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+    -3.2701402  -11.508579  ]
   ```
 
 - Python API
@@ -111,42 +111,42 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
   -3.04878      1.611095    10.127234   -10.534177   -15.821609
    1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
- -11.343508     2.3385992   -8.719341    14.213509    15.404744
-  -0.39327756   6.338786     2.688887     8.7104025   17.469526
-  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-   8.013747    13.891729    -9.926753     5.655307    -5.9422326
- -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-  -6.40137     23.63524      2.9711294  -22.708025     9.93719
-  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-  15.999649     3.3004563   12.747926    15.429879     4.7849145
-   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-  -9.224193    14.568347   -10.568833     4.982321    -4.342062
-   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
- -11.54324      7.681869     0.44475392   9.708182    -8.932846
-   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-   2.9079323    6.049952     9.275183   -18.078873     6.2983274
-  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-  18.495346   -14.293832     7.89578      2.2714825   22.976387
-  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
- -11.924197     2.171869     2.0423572   -6.173772    10.778437
-  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
- -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-  10.833007    -6.717991     4.504732    13.4244375    1.1306485
-   7.3435574    1.400918    14.704036    -9.501399     7.2315617
-  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-  -3.2701402  -11.508579  ]
+  -11.343508     2.3385992   -8.719341    14.213509    15.404744
+    -0.39327756   6.338786     2.688887     8.7104025   17.469526
+    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+    8.013747    13.891729    -9.926753     5.655307    -5.9422326
+  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+    -6.40137     23.63524      2.9711294  -22.708025     9.93719
+    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+    15.999649     3.3004563   12.747926    15.429879     4.7849145
+    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+    -9.224193    14.568347   -10.568833     4.982321    -4.342062
+    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+  -11.54324      7.681869     0.44475392   9.708182    -8.932846
+    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+    2.9079323    6.049952     9.275183   -18.078873     6.2983274
+    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+    18.495346   -14.293832     7.89578      2.2714825   22.976387
+    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+  -11.924197     2.171869     2.0423572   -6.173772    10.778437
+    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+    10.833007    -6.717991     4.504732    13.4244375    1.1306485
+    7.3435574    1.400918    14.704036    -9.501399     7.2315617
+    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+    -3.2701402  -11.508579  ]
   ```
 
 ### 4.Pretrained Models
diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md
index dd7a39fb..242c07e6 100644
--- a/demos/speaker_verification/README_cn.md
+++ b/demos/speaker_verification/README_cn.md
@@ -48,42 +48,42 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   demo [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
   -3.04878      1.611095    10.127234   -10.534177   -15.821609
    1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
- -11.343508     2.3385992   -8.719341    14.213509    15.404744
-  -0.39327756   6.338786     2.688887     8.7104025   17.469526
-  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-   8.013747    13.891729    -9.926753     5.655307    -5.9422326
- -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-  -6.40137     23.63524      2.9711294  -22.708025     9.93719
-  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-  15.999649     3.3004563   12.747926    15.429879     4.7849145
-   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-  -9.224193    14.568347   -10.568833     4.982321    -4.342062
-   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
- -11.54324      7.681869     0.44475392   9.708182    -8.932846
-   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-   2.9079323    6.049952     9.275183   -18.078873     6.2983274
-  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-  18.495346   -14.293832     7.89578      2.2714825   22.976387
-  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
- -11.924197     2.171869     2.0423572   -6.173772    10.778437
-  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
- -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-  10.833007    -6.717991     4.504732    13.4244375    1.1306485
-   7.3435574    1.400918    14.704036    -9.501399     7.2315617
-  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-  -3.2701402  -11.508579  ]
+  -11.343508     2.3385992   -8.719341    14.213509    15.404744
+    -0.39327756   6.338786     2.688887     8.7104025   17.469526
+    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+    8.013747    13.891729    -9.926753     5.655307    -5.9422326
+  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+    -6.40137     23.63524      2.9711294  -22.708025     9.93719
+    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+    15.999649     3.3004563   12.747926    15.429879     4.7849145
+    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+    -9.224193    14.568347   -10.568833     4.982321    -4.342062
+    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+  -11.54324      7.681869     0.44475392   9.708182    -8.932846
+    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+    2.9079323    6.049952     9.275183   -18.078873     6.2983274
+    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+    18.495346   -14.293832     7.89578      2.2714825   22.976387
+    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+  -11.924197     2.171869     2.0423572   -6.173772    10.778437
+    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+    10.833007    -6.717991     4.504732    13.4244375    1.1306485
+    7.3435574    1.400918    14.704036    -9.501399     7.2315617
+    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+    -3.2701402  -11.508579  ]
   ```
 
 - Python API
@@ -109,42 +109,42 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
   -3.04878      1.611095    10.127234   -10.534177   -15.821609
    1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
- -11.343508     2.3385992   -8.719341    14.213509    15.404744
-  -0.39327756   6.338786     2.688887     8.7104025   17.469526
-  -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-   8.013747    13.891729    -9.926753     5.655307    -5.9422326
- -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-   3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-   1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-  -6.40137     23.63524      2.9711294  -22.708025     9.93719
-  20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-  15.999649     3.3004563   12.747926    15.429879     4.7849145
-   5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-  15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-  -9.224193    14.568347   -10.568833     4.982321    -4.342062
-   0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-  -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
- -11.54324      7.681869     0.44475392   9.708182    -8.932846
-   0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-   2.9079323    6.049952     9.275183   -18.078873     6.2983274
-  -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-   4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-  18.495346   -14.293832     7.89578      2.2714825   22.976387
-  -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
- -11.924197     2.171869     2.0423572   -6.173772    10.778437
-  25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-  -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-  -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-  -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-  -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
- -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-  -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-   8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-  10.833007    -6.717991     4.504732    13.4244375    1.1306485
-   7.3435574    1.400918    14.704036    -9.501399     7.2315617
-  -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-   6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-  -3.2701402  -11.508579  ]
+  -11.343508     2.3385992   -8.719341    14.213509    15.404744
+    -0.39327756   6.338786     2.688887     8.7104025   17.469526
+    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+    8.013747    13.891729    -9.926753     5.655307    -5.9422326
+  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+    -6.40137     23.63524      2.9711294  -22.708025     9.93719
+    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+    15.999649     3.3004563   12.747926    15.429879     4.7849145
+    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+    -9.224193    14.568347   -10.568833     4.982321    -4.342062
+    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+  -11.54324      7.681869     0.44475392   9.708182    -8.932846
+    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+    2.9079323    6.049952     9.275183   -18.078873     6.2983274
+    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+    18.495346   -14.293832     7.89578      2.2714825   22.976387
+    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+  -11.924197     2.171869     2.0423572   -6.173772    10.778437
+    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+    10.833007    -6.717991     4.504732    13.4244375    1.1306485
+    7.3435574    1.400918    14.704036    -9.501399     7.2315617
+    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+    -3.2701402  -11.508579  ]
   ```
 
 ### 4.预训练模型
diff --git a/docs/source/released_model.md b/docs/source/released_model.md
index 354ecf30..809ace8d 100644
--- a/docs/source/released_model.md
+++ b/docs/source/released_model.md
@@ -84,7 +84,7 @@ PANN | VoxCeleb| [voxceleb_ecapatdnn](https://github.com/PaddlePaddle/PaddleSpee
 ## Punctuation Restoration Models
 Model Type | Dataset| Example Link | Pretrained Models
 :-------------:| :------------:| :-----: | :-----:
-Ernie Linear | IWLST2012_zh |[iwslt2012_punc0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/iwslt2012/punc0)|[ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_1.tar.gz)
+Ernie Linear | IWLST2012_zh |[iwslt2012_punc0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/iwslt2012/punc0)|[ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip]((https://paddlespeech.bj.bcebos.com/text/ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip)
 
 ## Speech Recognition Model  from paddle 1.8
 

From 30dc4585ceac4c05519119627bb48f076859972c Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Fri, 25 Mar 2022 19:43:34 +0800
Subject: [PATCH 068/126] repari the cls mode url, test=doc

---
 docs/source/released_model.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/released_model.md b/docs/source/released_model.md
index 809ace8d..807501a2 100644
--- a/docs/source/released_model.md
+++ b/docs/source/released_model.md
@@ -84,7 +84,7 @@ PANN | VoxCeleb| [voxceleb_ecapatdnn](https://github.com/PaddlePaddle/PaddleSpee
 ## Punctuation Restoration Models
 Model Type | Dataset| Example Link | Pretrained Models
 :-------------:| :------------:| :-----: | :-----:
-Ernie Linear | IWLST2012_zh |[iwslt2012_punc0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/iwslt2012/punc0)|[ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip]((https://paddlespeech.bj.bcebos.com/text/ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip)
+Ernie Linear | IWLST2012_zh |[iwslt2012_punc0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/iwslt2012/punc0)|[ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/text/ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip)
 
 ## Speech Recognition Model  from paddle 1.8
 

From d5142e5e1591f2ef0fab6580c58aba0b6f1ec6a3 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sat, 26 Mar 2022 01:00:19 +0800
Subject: [PATCH 069/126] add vector cli annotation, test=doc

---
 demos/speaker_verification/README_cn.md |   2 +-
 paddlespeech/cli/vector/infer.py        | 103 ++++++++++++++++++++++--
 2 files changed, 97 insertions(+), 8 deletions(-)

diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md
index 242c07e6..fde636db 100644
--- a/demos/speaker_verification/README_cn.md
+++ b/demos/speaker_verification/README_cn.md
@@ -97,7 +97,7 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
       sample_rate=16000,
       config=None,  # Set `config` and `ckpt_path` to None to use pretrained model.
       ckpt_path=None,
-      audio_file='./zh.wav',
+      audio_file='./85236145389.wav',
       force_yes=False,
       device=paddle.get_device())
   print('Audio embedding Result: \n{}'.format(audio_emb))
diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index 91974761..53324f93 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -42,13 +42,15 @@ pretrained_models = {
     # "paddlespeech vector --task spk --model ecapatdnn_voxceleb12-16k --sr 16000 --input ./input.wav"
     "ecapatdnn_voxceleb12-16k": {
         'url':
-        'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz',
+        'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_1.tar.gz',
         'md5':
-        '85ff08ce0ef406b8c6d7b5ffc5b2b48f',
+        'a1c0dba7d4de997187786ff517d5b4ec',
         'cfg_path':
-        'conf/model.yaml',
+        'conf/model.yaml',  # the yaml config path
         'ckpt_path':
-        'model/model',
+        'model/model',  # the format is ${dir}/{model_name}, 
+        # so the first 'model' is dir, the second 'model' is the name
+        # this means we have a model stored as model/model.pdparams
     },
 }
 
@@ -173,22 +175,54 @@ class VectorExecutor(BaseExecutor):
                  sample_rate: int=16000,
                  config: os.PathLike=None,
                  ckpt_path: os.PathLike=None,
-                 force_yes: bool=False,
                  device=paddle.get_device()):
+        """Extract the audio embedding
+
+        Args:
+            audio_file (os.PathLike): audio path, 
+                                      whose format must be wav and sample rate must be matched the model
+            model (str, optional): mode type, which is been loaded from the pretrained model list. 
+                                   Defaults to 'ecapatdnn-voxceleb12'.
+            sample_rate (int, optional): model sample rate. Defaults to 16000.
+            config (os.PathLike, optional): yaml config. Defaults to None.
+            ckpt_path (os.PathLike, optional): pretrained model path. Defaults to None.
+            device (_type_, optional): paddle running host device. Defaults to paddle.get_device().
+
+        Returns:
+            dict: return the audio embedding and the embedding shape
+        """
+        # stage 0: check the audio format
         audio_file = os.path.abspath(audio_file)
         if not self._check(audio_file, sample_rate):
             sys.exit(-1)
 
+        # stage 1: set the paddle runtime host device
         logger.info(f"device type: {device}")
         paddle.device.set_device(device)
+
+        # stage 2: read the specific pretrained model
         self._init_from_path(model, sample_rate, config, ckpt_path)
+
+        # stage 3: preprocess the audio and get the audio feat
         self.preprocess(model, audio_file)
+
+        # stage 4: infer the model and get the audio embedding
         self.infer(model)
+
+        # stage 5: process the result and set them to output dict
         res = self.postprocess()
 
         return res
 
     def _get_pretrained_path(self, tag: str) -> os.PathLike:
+        """get the neural network path from the pretrained model list
+
+        Args:
+            tag (str): model tag in the pretrained model list
+
+        Returns:
+            os.PathLike: the downloaded pretrained model path in the disk
+        """
         support_models = list(pretrained_models.keys())
         assert tag in pretrained_models, \
             'The model "{}" you want to use has not been supported,'\
@@ -210,15 +244,33 @@ class VectorExecutor(BaseExecutor):
                         sample_rate: int=16000,
                         cfg_path: Optional[os.PathLike]=None,
                         ckpt_path: Optional[os.PathLike]=None):
+        """Init the neural network from the model path
+
+        Args:
+            model_type (str, optional): model tag in the pretrained model list. 
+                                        Defaults to 'ecapatdnn_voxceleb12'.
+            sample_rate (int, optional): model sample rate. 
+                                         Defaults to 16000.
+            cfg_path (Optional[os.PathLike], optional): yaml config file path. 
+                                                        Defaults to None.
+            ckpt_path (Optional[os.PathLike], optional): the pretrained model path, which is stored in the disk. 
+                                                         Defaults to None.
+        """
+        # stage 0: avoid to init the mode again
         if hasattr(self, "model"):
             logger.info("Model has been initialized")
             return
 
         # stage 1: get the model and config path
+        #          if we want init the network from the model stored in the disk,
+        #          we must pass the config path and the ckpt model path
         if cfg_path is None or ckpt_path is None:
+            # get the mode from pretrained list
             sample_rate_str = "16k" if sample_rate == 16000 else "8k"
             tag = model_type + "-" + sample_rate_str
             logger.info(f"load the pretrained model: {tag}")
+            # get the model from the pretrained list
+            # we download the pretrained model and store it in the res_path
             res_path = self._get_pretrained_path(tag)
             self.res_path = res_path
 
@@ -227,6 +279,7 @@ class VectorExecutor(BaseExecutor):
             self.ckpt_path = os.path.join(
                 res_path, pretrained_models[tag]['ckpt_path'] + '.pdparams')
         else:
+            # get the model from disk
             self.cfg_path = os.path.abspath(cfg_path)
             self.ckpt_path = os.path.abspath(ckpt_path + ".pdparams")
             self.res_path = os.path.dirname(
@@ -241,7 +294,6 @@ class VectorExecutor(BaseExecutor):
         self.config.merge_from_file(self.cfg_path)
 
         # stage 3: get the model name to instance the model network with dynamic_import
-        # Noet: we use the '-' to get the model name instead of '_'
         logger.info("start to dynamic import the model class")
         model_name = model_type[:model_type.rindex('_')]
         logger.info(f"model name {model_name}")
@@ -262,31 +314,54 @@ class VectorExecutor(BaseExecutor):
 
     @paddle.no_grad()
     def infer(self, model_type: str):
+        """Infer the model to get the embedding
 
+        Args:
+            model_type (str): speaker verification model type
+        """
+        # stage 0: get the feat and length from _inputs
         feats = self._inputs["feats"]
         lengths = self._inputs["lengths"]
         logger.info("start to do backbone network model forward")
         logger.info(
             f"feats shape:{feats.shape}, lengths shape: {lengths.shape}")
+
+        # stage 1: get the audio embedding
         # embedding from (1, emb_size, 1) -> (emb_size)
         embedding = self.model.backbone(feats, lengths).squeeze().numpy()
         logger.info(f"embedding size: {embedding.shape}")
 
+        # stage 2: put the embedding and dim info to _outputs property
         self._outputs["embedding"] = embedding
 
     def postprocess(self) -> Union[str, os.PathLike]:
+        """Return the audio embedding info
+
+        Returns:
+            Union[str, os.PathLike]: audio embedding info
+        """
+        embedding = self._outputs["embedding"]
+        dim = embedding.shape[0]
+        # return {"dim": dim, "embedding": embedding}
         return self._outputs["embedding"]
 
     def preprocess(self, model_type: str, input_file: Union[str, os.PathLike]):
+        """Extract the audio feat
+
+        Args:
+            model_type (str): speaker verification model type
+            input_file (Union[str, os.PathLike]): audio file path
+        """
         audio_file = input_file
         if isinstance(audio_file, (str, os.PathLike)):
             logger.info(f"Preprocess audio file: {audio_file}")
 
-        # stage 1: load the audio
+        # stage 1: load the audio sample points
         waveform, sr = load_audio(audio_file)
         logger.info(f"load the audio sample points, shape is: {waveform.shape}")
 
         # stage 2: get the audio feat
+        # Note: Now we only support fbank feature
         try:
             feat = melspectrogram(
                 x=waveform,
@@ -302,8 +377,13 @@ class VectorExecutor(BaseExecutor):
         feat = paddle.to_tensor(feat).unsqueeze(0)
         # in inference period, the lengths is all one without padding
         lengths = paddle.ones([1])
+
+        # stage 3: we do feature normalize,
+        #          Now we assume that the feat must do normalize
         feat = feature_normalize(feat, mean_norm=True, std_norm=False)
 
+        # stage 4: store the feat and length in the _inputs,
+        #          which will be used in other function
         logger.info(f"feats shape: {feat.shape}")
         self._inputs["feats"] = feat
         self._inputs["lengths"] = lengths
@@ -311,6 +391,15 @@ class VectorExecutor(BaseExecutor):
         logger.info("audio extract the feat success")
 
     def _check(self, audio_file: str, sample_rate: int):
+        """Check if the model sample match the audio sample rate 
+
+        Args:
+            audio_file (str): audio file path, which will be extracted the embedding
+            sample_rate (int): the desired model sample rate 
+
+        Returns:
+            bool: return if the audio sample rate matches the  model sample rate
+        """
         self.sample_rate = sample_rate
         if self.sample_rate != 16000 and self.sample_rate != 8000:
             logger.error(

From 1fdb36f757528a4cb4a01002aebedff2fbda3666 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sat, 26 Mar 2022 01:03:53 +0800
Subject: [PATCH 070/126] add mode emb dim info, test=doc

---
 examples/voxceleb/sv0/RESULT.md | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/examples/voxceleb/sv0/RESULT.md b/examples/voxceleb/sv0/RESULT.md
index 3df2af51..10cd1fb6 100644
--- a/examples/voxceleb/sv0/RESULT.md
+++ b/examples/voxceleb/sv0/RESULT.md
@@ -2,7 +2,6 @@
 
 ## ECAPA-TDNN 
 
-| Model | Number of Params | Release | Config | Test set |  Cosine | Cosine + S-Norm | 
-| --- | --- | --- | --- | --- | --- | --- | 
-| ECAPA-TDNN | 85MM | 0.1.1 | conf/model.yaml | test | 1.15 |  1.06 | 
-
+| Model | Number of Params | Release | Config | dim | Test set |  Cosine | Cosine + S-Norm | 
+| --- | --- | --- | --- | --- | --- | --- | ---- |
+| ECAPA-TDNN | 85M | 0.1.1 | conf/model.yaml |192 | test | 1.15 |  1.06 | 

From ef1bc5e8155b34c278f1abff1e6a61ae14174ac2 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sat, 26 Mar 2022 16:55:01 +0800
Subject: [PATCH 071/126] vector cli output dim info, test=doc

---
 demos/speaker_verification/README.md    | 176 ++++++++++++-----------
 demos/speaker_verification/README_cn.md | 177 +++++++++++++-----------
 examples/voxceleb/sv0/RESULT.md         |   2 +-
 paddlespeech/cli/vector/infer.py        |   5 +-
 4 files changed, 199 insertions(+), 161 deletions(-)

diff --git a/demos/speaker_verification/README.md b/demos/speaker_verification/README.md
index 6e8f9693..c289e8f4 100644
--- a/demos/speaker_verification/README.md
+++ b/demos/speaker_verification/README.md
@@ -47,45 +47,55 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   Output:
 
 ```bash
-  demo [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
-  -3.04878      1.611095    10.127234   -10.534177   -15.821609
-   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
-  -11.343508     2.3385992   -8.719341    14.213509    15.404744
-    -0.39327756   6.338786     2.688887     8.7104025   17.469526
-    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-    8.013747    13.891729    -9.926753     5.655307    -5.9422326
-  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-    -6.40137     23.63524      2.9711294  -22.708025     9.93719
-    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-    15.999649     3.3004563   12.747926    15.429879     4.7849145
-    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-    -9.224193    14.568347   -10.568833     4.982321    -4.342062
-    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
-  -11.54324      7.681869     0.44475392   9.708182    -8.932846
-    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-    2.9079323    6.049952     9.275183   -18.078873     6.2983274
-    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-    18.495346   -14.293832     7.89578      2.2714825   22.976387
-    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
-  -11.924197     2.171869     2.0423572   -6.173772    10.778437
-    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
-  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-    10.833007    -6.717991     4.504732    13.4244375    1.1306485
-    7.3435574    1.400918    14.704036    -9.501399     7.2315617
-    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-    -3.2701402  -11.508579  ]
+  demo  {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
+         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
+       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
+         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
+         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
+        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
+        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
+        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
+        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
+         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
+         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
+       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
+         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
+         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
+       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
+        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
+         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
+         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
+       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
+        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
+         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
+        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
+        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
+         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
+        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
+         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
+         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
+         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
+        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
+       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
+        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
+         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
+        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
+        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
+        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
+        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
+         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
+        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
+        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
+       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
+        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
+       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
+         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
+         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
+         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
+        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
+         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
+        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
+      dtype=float32)}
   ```
 
 - Python API
@@ -108,45 +118,55 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   Output:
   ```bash
   # Vector Result:
-  [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
-  -3.04878      1.611095    10.127234   -10.534177   -15.821609
-   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
-  -11.343508     2.3385992   -8.719341    14.213509    15.404744
-    -0.39327756   6.338786     2.688887     8.7104025   17.469526
-    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-    8.013747    13.891729    -9.926753     5.655307    -5.9422326
-  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-    -6.40137     23.63524      2.9711294  -22.708025     9.93719
-    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-    15.999649     3.3004563   12.747926    15.429879     4.7849145
-    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-    -9.224193    14.568347   -10.568833     4.982321    -4.342062
-    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
-  -11.54324      7.681869     0.44475392   9.708182    -8.932846
-    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-    2.9079323    6.049952     9.275183   -18.078873     6.2983274
-    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-    18.495346   -14.293832     7.89578      2.2714825   22.976387
-    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
-  -11.924197     2.171869     2.0423572   -6.173772    10.778437
-    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
-  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-    10.833007    -6.717991     4.504732    13.4244375    1.1306485
-    7.3435574    1.400918    14.704036    -9.501399     7.2315617
-    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-    -3.2701402  -11.508579  ]
+   {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
+         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
+       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
+         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
+         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
+        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
+        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
+        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
+        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
+         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
+         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
+       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
+         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
+         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
+       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
+        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
+         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
+         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
+       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
+        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
+         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
+        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
+        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
+         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
+        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
+         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
+         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
+         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
+        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
+       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
+        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
+         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
+        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
+        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
+        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
+        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
+         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
+        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
+        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
+       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
+        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
+       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
+         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
+         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
+         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
+        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
+         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
+        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
+      dtype=float32)}
   ```
 
 ### 4.Pretrained Models
diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md
index fde636db..b4e5ca65 100644
--- a/demos/speaker_verification/README_cn.md
+++ b/demos/speaker_verification/README_cn.md
@@ -45,45 +45,55 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
 
   输出：
   ```bash
-  demo [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
-  -3.04878      1.611095    10.127234   -10.534177   -15.821609
-   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
-  -11.343508     2.3385992   -8.719341    14.213509    15.404744
-    -0.39327756   6.338786     2.688887     8.7104025   17.469526
-    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-    8.013747    13.891729    -9.926753     5.655307    -5.9422326
-  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-    -6.40137     23.63524      2.9711294  -22.708025     9.93719
-    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-    15.999649     3.3004563   12.747926    15.429879     4.7849145
-    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-    -9.224193    14.568347   -10.568833     4.982321    -4.342062
-    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
-  -11.54324      7.681869     0.44475392   9.708182    -8.932846
-    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-    2.9079323    6.049952     9.275183   -18.078873     6.2983274
-    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-    18.495346   -14.293832     7.89578      2.2714825   22.976387
-    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
-  -11.924197     2.171869     2.0423572   -6.173772    10.778437
-    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
-  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-    10.833007    -6.717991     4.504732    13.4244375    1.1306485
-    7.3435574    1.400918    14.704036    -9.501399     7.2315617
-    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-    -3.2701402  -11.508579  ]
+  demo  {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
+         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
+       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
+         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
+         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
+        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
+        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
+        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
+        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
+         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
+         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
+       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
+         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
+         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
+       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
+        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
+         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
+         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
+       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
+        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
+         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
+        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
+        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
+         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
+        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
+         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
+         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
+         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
+        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
+       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
+        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
+         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
+        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
+        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
+        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
+        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
+         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
+        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
+        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
+       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
+        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
+       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
+         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
+         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
+         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
+        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
+         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
+        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
+      dtype=float32)}
   ```
 
 - Python API
@@ -106,45 +116,55 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   输出：
   ```bash
   # Vector Result:
-  [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
-  -3.04878      1.611095    10.127234   -10.534177   -15.821609
-   1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
-  -11.343508     2.3385992   -8.719341    14.213509    15.404744
-    -0.39327756   6.338786     2.688887     8.7104025   17.469526
-    -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
-    8.013747    13.891729    -9.926753     5.655307    -5.9422326
-  -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
-    3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
-    1.7594414   -0.6485091    4.485623     2.0207152    7.264915
-    -6.40137     23.63524      2.9711294  -22.708025     9.93719
-    20.354511   -10.324688    -0.700492    -8.783211    -5.27593
-    15.999649     3.3004563   12.747926    15.429879     4.7849145
-    5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
-    15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
-    -9.224193    14.568347   -10.568833     4.982321    -4.342062
-    0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
-    -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
-  -11.54324      7.681869     0.44475392   9.708182    -8.932846
-    0.4123232   -4.361452     1.3948607    9.511665     0.11667654
-    2.9079323    6.049952     9.275183   -18.078873     6.2983274
-    -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
-    4.010979    11.000591    -2.8873312    7.1352735  -16.79663
-    18.495346   -14.293832     7.89578      2.2714825   22.976387
-    -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
-  -11.924197     2.171869     2.0423572   -6.173772    10.778437
-    25.77281     -4.9495463   14.57806      0.3044315    2.6132357
-    -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
-    -4.9401326   23.465864     5.1685796   -9.018578     9.037825
-    -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
-    -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
-  -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
-    -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
-    8.731719   -20.778936   -11.495662     5.8033476   -4.752041
-    10.833007    -6.717991     4.504732    13.4244375    1.1306485
-    7.3435574    1.400918    14.704036    -9.501399     7.2315617
-    -6.417456     1.3333273   11.872697    -0.30664724   8.8845
-    6.5569253    4.7948146    0.03662816  -8.704245     6.224871
-    -3.2701402  -11.508579  ]
+   {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
+         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
+       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
+         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
+         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
+        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
+        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
+        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
+        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
+         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
+         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
+       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
+         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
+         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
+       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
+        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
+         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
+         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
+       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
+        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
+         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
+        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
+        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
+         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
+        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
+         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
+         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
+         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
+        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
+       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
+        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
+         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
+        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
+        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
+        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
+        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
+         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
+        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
+        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
+       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
+        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
+       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
+         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
+         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
+         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
+        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
+         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
+        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
+      dtype=float32)}
   ```
 
 ### 4.预训练模型
@@ -153,4 +173,3 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
 | 模型 | 采样率
 | :--- | :---: |
 | ecapatdnn_voxceleb12 | 16k
-
diff --git a/examples/voxceleb/sv0/RESULT.md b/examples/voxceleb/sv0/RESULT.md
index 10cd1fb6..c37bcece 100644
--- a/examples/voxceleb/sv0/RESULT.md
+++ b/examples/voxceleb/sv0/RESULT.md
@@ -4,4 +4,4 @@
 
 | Model | Number of Params | Release | Config | dim | Test set |  Cosine | Cosine + S-Norm | 
 | --- | --- | --- | --- | --- | --- | --- | ---- |
-| ECAPA-TDNN | 85M | 0.1.1 | conf/model.yaml |192 | test | 1.15 |  1.06 | 
+| ECAPA-TDNN | 85M | 0.1.1 | conf/ecapa_tdnn.yaml |192 | test | 1.15 |  1.06 | 
diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index 53324f93..378a3d83 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -171,7 +171,7 @@ class VectorExecutor(BaseExecutor):
     @stats_wrapper
     def __call__(self,
                  audio_file: os.PathLike,
-                 model: str='ecapatdnn-voxceleb12',
+                 model: str='ecapatdnn_voxceleb12',
                  sample_rate: int=16000,
                  config: os.PathLike=None,
                  ckpt_path: os.PathLike=None,
@@ -342,8 +342,7 @@ class VectorExecutor(BaseExecutor):
         """
         embedding = self._outputs["embedding"]
         dim = embedding.shape[0]
-        # return {"dim": dim, "embedding": embedding}
-        return self._outputs["embedding"]
+        return {"dim": dim, "embedding": embedding}
 
     def preprocess(self, model_type: str, input_file: Union[str, os.PathLike]):
         """Extract the audio feat

From 2c9dc0c89b2febe5396e44a20f6292200c697a0c Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Sat, 26 Mar 2022 18:39:12 +0800
Subject: [PATCH 072/126] add some vector cli comments, test=doc

---
 paddlespeech/cli/vector/infer.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index 378a3d83..79d3b5db 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -68,12 +68,13 @@ class VectorExecutor(BaseExecutor):
 
         self.parser = argparse.ArgumentParser(
             prog="paddlespeech.vector", add_help=True)
+
         self.parser.add_argument(
             "--model",
             type=str,
             default="ecapatdnn_voxceleb12",
             choices=["ecapatdnn_voxceleb12"],
-            help="Choose model type of asr task.")
+            help="Choose model type of vector task.")
         self.parser.add_argument(
             "--task",
             type=str,
@@ -81,7 +82,7 @@ class VectorExecutor(BaseExecutor):
             choices=["spk"],
             help="task type in vector domain")
         self.parser.add_argument(
-            "--input", type=str, default=None, help="Audio file to recognize.")
+            "--input", type=str, default=None, help="Audio file to extract embedding.")
         self.parser.add_argument(
             "--sample_rate",
             type=int,
@@ -186,7 +187,7 @@ class VectorExecutor(BaseExecutor):
             sample_rate (int, optional): model sample rate. Defaults to 16000.
             config (os.PathLike, optional): yaml config. Defaults to None.
             ckpt_path (os.PathLike, optional): pretrained model path. Defaults to None.
-            device (_type_, optional): paddle running host device. Defaults to paddle.get_device().
+            device (optional): paddle running host device. Defaults to paddle.get_device().
 
         Returns:
             dict: return the audio embedding and the embedding shape
@@ -216,6 +217,7 @@ class VectorExecutor(BaseExecutor):
 
     def _get_pretrained_path(self, tag: str) -> os.PathLike:
         """get the neural network path from the pretrained model list
+           we stored all the pretained mode in the variable `pretrained_models`
 
         Args:
             tag (str): model tag in the pretrained model list
@@ -332,6 +334,7 @@ class VectorExecutor(BaseExecutor):
         logger.info(f"embedding size: {embedding.shape}")
 
         # stage 2: put the embedding and dim info to _outputs property
+        #          the embedding type is numpy.array
         self._outputs["embedding"] = embedding
 
     def postprocess(self) -> Union[str, os.PathLike]:
@@ -356,6 +359,7 @@ class VectorExecutor(BaseExecutor):
             logger.info(f"Preprocess audio file: {audio_file}")
 
         # stage 1: load the audio sample points
+        #    Note: this process must match the training process
         waveform, sr = load_audio(audio_file)
         logger.info(f"load the audio sample points, shape is: {waveform.shape}")
 
@@ -397,7 +401,7 @@ class VectorExecutor(BaseExecutor):
             sample_rate (int): the desired model sample rate 
 
         Returns:
-            bool: return if the audio sample rate matches the  model sample rate
+            bool: return if the audio sample rate matches the model sample rate
         """
         self.sample_rate = sample_rate
         if self.sample_rate != 16000 and self.sample_rate != 8000:

From 45550faa12359352b9ff43e325efec62a1fed46e Mon Sep 17 00:00:00 2001
From: qingen <qingenz123@126.com>
Date: Sun, 27 Mar 2022 00:08:32 +0800
Subject: [PATCH 073/126] [vec][search] update to paddlespeech model test=doc
 #1608

---
 demos/audio_searching/README.md        | 118 ++++++++++++++++++-------
 demos/audio_searching/README_cn.md     | 115 +++++++++++++++++-------
 demos/audio_searching/requirements.txt |   3 +-
 demos/audio_searching/src/config.py    |   2 +-
 demos/audio_searching/src/encode.py    |  16 ++--
 demos/audio_searching/src/test_main.py |  16 ++--
 paddlespeech/cli/vector/infer.py       |   2 +-
 7 files changed, 188 insertions(+), 84 deletions(-)

diff --git a/demos/audio_searching/README.md b/demos/audio_searching/README.md
index 2bce9313..c0df12ec 100644
--- a/demos/audio_searching/README.md
+++ b/demos/audio_searching/README.md
@@ -3,20 +3,25 @@
 # Audio Searching
 
 ## Introduction
-As the Internet continues to evolve, unstructured data such as emails, social media photos, live videos, and customer service voice calls have become increasingly common.  If we want to process the data on a computer, we need to use embedding technology to transform the data into vector and store, index, and query it
+As the Internet continues to evolve, unstructured data such as emails, social media photos, live videos, and customer service voice calls have become increasingly common. If we want to process the data on a computer, we need to use embedding technology to transform the data into vector and store, index, and query it.
 
-However, when there is a large amount of data, such as hundreds of millions of audio tracks, it is more difficult to do a similarity search.  The exhaustive method is feasible, but very time consuming.  For this scenario, this demo will introduce how to build an audio similarity retrieval system using the open source vector database Milvus
+However, when there is a large amount of data, such as hundreds of millions of audio tracks, it is more difficult to do a similarity search. The exhaustive method is feasible, but very time consuming.  For this scenario, this demo will introduce how to build an audio similarity retrieval system using the open source vector database Milvus.
 
-Audio retrieval (speech, music, speaker, etc.) enables querying and finding similar sounds (or the same speaker) in a large amount of audio data.  The audio similarity retrieval system can be used to identify similar sound effects, minimize intellectual property infringement, quickly retrieve the voice print library, and help enterprises control fraud and identity theft.  Audio retrieval also plays an important role in the classification and statistical analysis of audio data
+Audio retrieval (speech, music, speaker, etc.) enables querying and finding similar sounds (or the same speaker) in a large amount of audio data.  The audio similarity retrieval system can be used to identify similar sound effects, minimize intellectual property infringement, quickly retrieve the voice print library, and help enterprises control fraud and identity theft. Audio retrieval also plays an important role in the classification and statistical analysis of audio data.
 
-In this demo, you will learn how to build an audio retrieval system to retrieve similar sound snippets.  The uploaded audio clips are converted into vector data using paddlespeech-based pre-training models (audio classification model, speaker recognition model, etc.) and stored in Milvus.  Milvus automatically generates a unique ID for each vector, then stores the ID and the corresponding audio information (audio ID, audio speaker ID, etc.) in MySQL to complete the library construction.  During retrieval, users upload test audio to obtain vector, and then conduct vector similarity search in Milvus. The retrieval result returned by Milvus is vector ID, and the corresponding audio information can be queried in MySQL by ID
+In this demo, you will learn how to build an audio retrieval system to retrieve similar sound snippets. The uploaded audio clips are converted into vector data using paddlespeech-based pre-training models (audio classification model, speaker recognition model, etc.) and stored in Milvus.  Milvus automatically generates a unique ID for each vector, then stores the ID and the corresponding audio information (audio ID, audio speaker ID, etc.) in MySQL to complete the library construction.  During retrieval, users upload test audio to obtain vector, and then conduct vector similarity search in Milvus.The retrieval result returned by Milvus is vector ID, and the corresponding audio information can be queried in MySQL by ID.
 
 ![Workflow of an audio searching system](./img/audio_searching.png)
 
-Note：this demo uses the [CN-Celeb](http://openslr.org/82/) dataset of at least 650,000 audio entries and 3000 speakers to build the audio vector library, which is then retrieved using a preset distance calculation. The dataset can also use other,  Adjust as needed, e.g. Librispeech, VoxCeleb, UrbanSound, GloVe, MNIST, etc
+Note：this demo uses the [CN-Celeb](http://openslr.org/82/) dataset of at least 650,000 audio entries and 3000 speakers to build the audio vector library, which is then retrieved using a preset distance calculation. The dataset can also use other,  Adjust as needed, e.g. Librispeech, VoxCeleb, UrbanSound, GloVe, MNIST, etc.
 
 ## Usage
-### 1. Prepare MySQL and Milvus services by docker-compose
+### 1. Prepare PaddleSpeech
+Audio vector extraction requires PaddleSpeech training model, so please make sure that PaddleSpeech has been installed before running. Specific installation steps: See [installation](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/docs/source/install.md).  
+
+You can choose one way from easy, meduim and hard to install paddlespeech.
+
+### 2. Prepare MySQL and Milvus services by docker-compose
 The audio similarity search system requires Milvus, MySQL services. We can start these containers with one click through [docker-compose.yaml](./docker-compose.yaml), so please make sure you have [installed Docker Engine](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) before running. then
 
 ```bash
@@ -45,7 +50,7 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
 15c84a506754  qingen1/paddlespeech-audio-search-client:2.3  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
 ```
 
-### 2. Start API Server
+### 3. Start API Server
 Then to start the system server, and it provides HTTP backend services.
 
 - Install the Python packages
@@ -75,73 +80,120 @@ Then to start the system server, and it provides HTTP backend services.
   Then start the server with Fastapi.
 
   ```bash
-  export PYTHONPATH=$PYTHONPATH:./src
+  export PYTHONPATH=$PYTHONPATH:./src:../../paddleaudio
   python src/main.py
   ```
 
   Then you will see the Application is started:
 
   ```bash
-  INFO:     Started server process [3949]
-  2022-03-07 17:39:14,864 ｜ INFO ｜ server.py ｜ serve ｜ 75 ｜ Started server process [3949]
+  INFO:     Started server process [13352]
+  2022-03-26 22:45:30,838 ｜ INFO ｜ server.py ｜ serve ｜ 75 ｜ Started server process [13352]
   INFO:     Waiting for application startup.
-  2022-03-07 17:39:14,865 ｜ INFO ｜ on.py ｜ startup ｜ 45 ｜ Waiting for application startup.
+  2022-03-26 22:45:30,839 ｜ INFO ｜ on.py ｜ startup ｜ 45 ｜ Waiting for application startup.
   INFO:     Application startup complete.
-  2022-03-07 17:39:14,866 ｜ INFO ｜ on.py ｜ startup ｜ 59 ｜ Application startup complete.
+  2022-03-26 22:45:30,839 ｜ INFO ｜ on.py ｜ startup ｜ 59 ｜ Application startup complete.
   INFO:     Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit)
-  2022-03-07 17:39:14,867 ｜ INFO ｜ server.py ｜ _log_started_message ｜ 206 ｜ Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit)
+  2022-03-26 22:45:30,840 ｜ INFO ｜ server.py ｜ _log_started_message ｜ 206 ｜ Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit)
   ```
 
-### 3. Usage
+### 4. Usage
 - Prepare data
   ```bash
   wget -c https://www.openslr.org/resources/82/cn-celeb_v2.tar.gz && tar -xvf cn-celeb_v2.tar.gz 
   ```
   Note: If you want to build a quick demo, you can use ./src/test_main.py:download_audio_data function, it downloads 20 audio files , Subsequent results show this collection as an example
  
- - scripts test (recommend!)
+- Scripts test (Recommended)
 
-    The internal process is downloading data, loading the Paddlespeech model, extracting embedding, storing library, retrieving and deleting library  
+    The internal process is downloading data, loading the paddlespeech model, extracting embedding, storing library, retrieving and deleting library  
     ```bash
     python ./src/test_main.py
     ```
 
     Output：
     ```bash
-    Checkpoint path: %your model path%
+    Downloading https://paddlespeech.bj.bcebos.com/vector/audio/example_audio.tar.gz ...
+    ...
+    Unpacking ./example_audio.tar.gz ...
+    [2022-03-26 22:50:54,987] [    INFO] - checking the aduio file format......
+    [2022-03-26 22:50:54,987] [    INFO] - The sample rate is 16000
+    [2022-03-26 22:50:54,987] [    INFO] - The audio file format is right
+    [2022-03-26 22:50:54,988] [    INFO] - device type: cpu
+    [2022-03-26 22:50:54,988] [    INFO] - load the pretrained model: ecapatdnn_voxceleb12-16k
+    [2022-03-26 22:50:54,990] [    INFO] - Downloading sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz from https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz
+    ...
+    [2022-03-26 22:51:17,285] [    INFO] - start to dynamic import the model class
+    [2022-03-26 22:51:17,285] [    INFO] - model name ecapatdnn
+    [2022-03-26 22:51:23,864] [    INFO] - start to set the model parameters to model
+    [2022-03-26 22:54:08,115] [    INFO] - create the model instance success
+    [2022-03-26 22:54:08,116] [    INFO] - Preprocess audio file: /home/zhaoqingen/PaddleSpeech/demos/audio_
+    searching/example_audio/knife_hit_iron3.wav
+    [2022-03-26 22:54:08,116] [    INFO] - load the audio sample points, shape is: (11012,)
+    [2022-03-26 22:54:08,150] [    INFO] - extract the audio feat, shape is: (80, 69)
+    [2022-03-26 22:54:08,152] [    INFO] - feats shape: [1, 80, 69]
+    [2022-03-26 22:54:08,154] [    INFO] - audio extract the feat success
+    [2022-03-26 22:54:08,155] [    INFO] - start to do backbone network model forward
+    [2022-03-26 22:54:08,155] [    INFO] - feats shape:[1, 80, 69], lengths shape: [1]
+    [2022-03-26 22:54:08,433] [    INFO] - embedding size: (192,)
     Extracting feature from audio No. 1 , 20 audios in total
+    [2022-03-26 22:54:08,435] [    INFO] - checking the aduio file format......
+    [2022-03-26 22:54:08,435] [    INFO] - The sample rate is 16000
+    [2022-03-26 22:54:08,436] [    INFO] - The audio file format is right
+    [2022-03-26 22:54:08,436] [    INFO] - device type: cpu
+    [2022-03-26 22:54:08,436] [    INFO] - Model has been initialized
+    [2022-03-26 22:54:08,436] [    INFO] - Preprocess audio file: /home/zhaoqingen/PaddleSpeech/demos/audio_searching/example_audio/sword_wielding.wav
+    [2022-03-26 22:54:08,436] [    INFO] - load the audio sample points, shape is: (6391,)
+    [2022-03-26 22:54:08,452] [    INFO] - extract the audio feat, shape is: (80, 40)
+    [2022-03-26 22:54:08,454] [    INFO] - feats shape: [1, 80, 40]
+    [2022-03-26 22:54:08,454] [    INFO] - audio extract the feat success
+    [2022-03-26 22:54:08,454] [    INFO] - start to do backbone network model forward
+    [2022-03-26 22:54:08,455] [    INFO] - feats shape:[1, 80, 40], lengths shape: [1]
+    [2022-03-26 22:54:08,633] [    INFO] - embedding size: (192,)
     Extracting feature from audio No. 2 , 20 audios in total
     ...
-    2022-03-09 17:22:13,870 ｜ INFO ｜ main.py ｜ load_audios ｜ 85 ｜ Successfully loaded data, total count: 20
-    2022-03-09 17:22:13,898 ｜ INFO ｜ main.py ｜ count_audio ｜ 147 ｜ Successfully count the number of data!
-    2022-03-09 17:22:13,918 ｜ INFO ｜ main.py ｜ audio_path ｜ 57 ｜ Successfully load audio: ./example_audio/test.wav
+    2022-03-26 22:54:15,892 ｜ INFO ｜ main.py ｜ load_audios ｜ 85 ｜ Successfully loaded data, total count: 20
+    2022-03-26 22:54:15,908 ｜ INFO ｜ main.py ｜ count_audio ｜ 148 ｜ Successfully count the number of data!
+    [2022-03-26 22:54:15,916] [    INFO] - checking the aduio file format......
+    [2022-03-26 22:54:15,916] [    INFO] - The sample rate is 16000
+    [2022-03-26 22:54:15,916] [    INFO] - The audio file format is right
+    [2022-03-26 22:54:15,916] [    INFO] - device type: cpu
+    [2022-03-26 22:54:15,916] [    INFO] - Model has been initialized
+    [2022-03-26 22:54:15,916] [    INFO] - Preprocess audio file: /home/zhaoqingen/PaddleSpeech/demos/audio_searching/example_audio/test.wav
+    [2022-03-26 22:54:15,917] [    INFO] - load the audio sample points, shape is: (8456,)
+    [2022-03-26 22:54:15,923] [    INFO] - extract the audio feat, shape is: (80, 53)
+    [2022-03-26 22:54:15,924] [    INFO] - feats shape: [1, 80, 53]
+    [2022-03-26 22:54:15,924] [    INFO] - audio extract the feat success
+    [2022-03-26 22:54:15,924] [    INFO] - start to do backbone network model forward
+    [2022-03-26 22:54:15,924] [    INFO] - feats shape:[1, 80, 53], lengths shape: [1]
+    [2022-03-26 22:54:16,051] [    INFO] - embedding size: (192,)
     ...
-    2022-03-09 17:22:32,580 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 131 ｜ search result http://testserver/data?audio_path=./example_audio/test.wav, distance 0.0
-    2022-03-09 17:22:32,580 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 131 ｜ search result http://testserver/data?audio_path=./example_audio/knife_chopping.wav, distance 0.021805256605148315
-    2022-03-09 17:22:32,580 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 131 ｜ search result http://testserver/data?audio_path=./example_audio/knife_cut_into_flesh.wav, distance 0.052762262523174286
+    2022-03-26 22:54:16,086 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 132 ｜ search result http://testserver/data?audio_path=./example_audio/test.wav, score 100.0
+    2022-03-26 22:54:16,087 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 132 ｜ search result http://testserver/data?audio_path=./example_audio/knife_chopping.wav, score 29.182177782058716
+    2022-03-26 22:54:16,087 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 132 ｜ search result http://testserver/data?audio_path=./example_audio/knife_cut_into_body.wav, score 22.73637056350708
     ...
-    2022-03-09 17:22:32,582 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 135 ｜ Successfully searched similar audio!
-    2022-03-09 17:22:33,658 ｜ INFO ｜ main.py ｜ drop_tables ｜ 159 ｜ Successfully drop tables in Milvus and MySQL!
+    2022-03-26 22:54:16,088 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 136 ｜ Successfully searched similar audio!
+    2022-03-26 22:54:17,164 ｜ INFO ｜ main.py ｜ drop_tables ｜ 160 ｜ Successfully drop tables in Milvus and MySQL!
     ```
-- GUI test (optional)
+- GUI test (Optional)
   
-    Navigate to 127.0.0.1:8068 in your browser to access the front-end interface
+    Navigate to 127.0.0.1:8068 in your browser to access the front-end interface.
 
-    Note: If the browser and the service are not on the same machine, then the IP needs to be changed to the IP of the machine where the service is located, and the corresponding API_URL in docker-compose.yaml needs to be changed and the service can be restarted
+    Note: If the browser and the service are not on the same machine, then the IP needs to be changed to the IP of the machine where the service is located, and the corresponding API_URL in docker-compose.yaml needs to be changed, and the docker-compose.yaml file needs to be re-executed for the change to take effect.
 
     - Insert data
 
-      Download the data and decompress it to a path named /home/speech/data. Then enter /home/speech/data in the address bar of the upload page to upload the data  
+      Download the data on the server and decompress it to a file, for example, /home/speech/data/. Then enter /home/speech/data/ in the address bar of the upload page to upload the data.
     
       ![](./img/insert.png)
 
     - Search for similar audio
 
-      Select the magnifying glass icon on the left side of the interface. Then, press the "Default Target Audio File" button and upload a .wav sound file you'd like to search. Results will be displayed
+      Select the magnifying glass icon on the left side of the interface. Then, press the "Default Target Audio File" button and upload a .wav sound file from the client you'd like to search. Results will be displayed.
 
       ![](./img/search.png)
 
-### 4.Result
+### 5.Result
 
  machine configuration：
 - OS: CentOS release 7.6 
@@ -157,9 +209,9 @@ recall and elapsed time statistics are shown in the following figure：
   ![](./img/result.png)
 
 
-The retrieval framework based on Milvus takes about 2.9 milliseconds to retrieve on the premise of 90% recall rate, and it takes about 500 milliseconds for feature extraction (testing audio takes about 5 seconds), that is, a single audio test takes about 503 milliseconds in total, which can meet most application scenarios
+The retrieval framework based on Milvus takes about 2.9 milliseconds to retrieve on the premise of 90% recall rate, and it takes about 500 milliseconds for feature extraction (testing audio takes about 5 seconds), that is, a single audio test takes about 503 milliseconds in total, which can meet most application scenarios.
 
-### 5.Pretrained Models
+### 6.Pretrained Models
 
 Here is a list of pretrained models released by PaddleSpeech :
 
diff --git a/demos/audio_searching/README_cn.md b/demos/audio_searching/README_cn.md
index a4cb7312..c851bd0f 100644
--- a/demos/audio_searching/README_cn.md
+++ b/demos/audio_searching/README_cn.md
@@ -4,21 +4,26 @@
 # 音频相似性检索
 ## 介绍
 
-随着互联网不断发展，电子邮件、社交媒体照片、直播视频、客服语音等非结构化数据已经变得越来越普遍。如果想要使用计算机来处理这些数据，需要使用 embedding 技术将这些数据转化为向量 vector，然后进行存储、建索引、并查询
+随着互联网不断发展，电子邮件、社交媒体照片、直播视频、客服语音等非结构化数据已经变得越来越普遍。如果想要使用计算机来处理这些数据，需要使用 embedding 技术将这些数据转化为向量 vector，然后进行存储、建索引、并查询。
 
-但是，当数据量很大，比如上亿条音频要做相似度搜索，就比较困难了。穷举法固然可行，但非常耗时。针对这种场景，该 demo 将介绍如何使用开源向量数据库 Milvus 搭建音频相似度检索系统
+但是，当数据量很大，比如上亿条音频要做相似度搜索，就比较困难了。穷举法固然可行，但非常耗时。针对这种场景，该 demo 将介绍如何使用开源向量数据库 Milvus 搭建音频相似度检索系统。
 
-音频检索（如演讲、音乐、说话人等检索）实现了在海量音频数据中查询并找出相似声音（或相同说话人）片段。音频相似性检索系统可用于识别相似的音效、最大限度减少知识产权侵权等，还可以快速的检索声纹库、帮助企业控制欺诈和身份盗用等。在音频数据的分类和统计分析中，音频检索也发挥着重要作用
+音频检索（如演讲、音乐、说话人等检索）实现了在海量音频数据中查询并找出相似声音（或相同说话人）片段。音频相似性检索系统可用于识别相似的音效、最大限度减少知识产权侵权等，还可以快速的检索声纹库、帮助企业控制欺诈和身份盗用等。在音频数据的分类和统计分析中，音频检索也发挥着重要作用。
 
-在本 demo 中，你将学会如何构建一个音频检索系统，用来检索相似的声音片段。使用基于 PaddleSpeech 预训练模型（音频分类模型，说话人识别模型等）将上传的音频片段转换为向量数据，并存储在 Milvus 中。Milvus 自动为每个向量生成唯一的 ID，然后将 ID 和 相应的音频信息（音频id，音频的说话人id等等）存储在 MySQL，这样就完成建库的工作。用户在检索时，上传测试音频，得到向量，然后在 Milvus 中进行向量相似度搜索，Milvus 返回的检索结果为向量 ID，通过 ID 在 MySQL 内部查询相应的音频信息即可
+在本 demo 中，你将学会如何构建一个音频检索系统，用来检索相似的声音片段。使用基于 PaddleSpeech 预训练模型（音频分类模型，说话人识别模型等）将上传的音频片段转换为向量数据，并存储在 Milvus 中。Milvus 自动为每个向量生成唯一的 ID，然后将 ID 和 相应的音频信息（音频id，音频的说话人id等等）存储在 MySQL，这样就完成建库的工作。用户在检索时，上传测试音频，得到向量，然后在 Milvus 中进行向量相似度搜索，Milvus 返回的检索结果为向量 ID，通过 ID 在 MySQL 内部查询相应的音频信息即可。
 
 ![音频检索流程图](./img/audio_searching.png)
 
-注：该 demo 使用 [CN-Celeb](http://openslr.org/82/) 数据集，包括至少 650000 条音频，3000 个说话人，来建立音频向量库（音频特征，或音频说话人特征），然后通过预设的距离计算方式进行音频（或说话人）检索，这里面数据集也可以使用其他的，根据需要调整，如Librispeech，VoxCeleb，UrbanSound，GloVe，MNIST等
+注：该 demo 使用 [CN-Celeb](http://openslr.org/82/) 数据集，包括至少 650000 条音频，3000 个说话人，来建立音频向量库（音频特征，或音频说话人特征），然后通过预设的距离计算方式进行音频（或说话人）检索，这里面数据集也可以使用其他的，根据需要调整，如Librispeech，VoxCeleb，UrbanSound，GloVe，MNIST等。
 
 ## 使用方法
-### 1. MySQL 和 Milvus 安装
-音频相似度搜索系统需要用到 Milvus, MySQL 服务。 我们可以通过 [docker-compose.yaml](./docker-compose.yaml) 一键启动这些容器，所以请确保在运行之前已经安装了 [Docker Engine](https://docs.docker.com/engine/install/) 和 [Docker Compose](https://docs.docker.com/compose/install/)。 即
+### 1. PaddleSpeech 安装
+音频向量的提取需要用到基于 PaddleSpeech 训练的模型，所以请确保在运行之前已经安装了 PaddleSpeech，具体安装步骤，详见[安装文档](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/docs/source/install_cn.md)。
+
+你可以从 easy，medium，hard 三中方式中选择一种方式安装。
+
+### 2. MySQL 和 Milvus 安装
+音频相似性的检索需要用到 Milvus, MySQL 服务。 我们可以通过 [docker-compose.yaml](./docker-compose.yaml) 一键启动这些容器，所以请确保在运行之前已经安装了 [Docker Engine](https://docs.docker.com/engine/install/) 和 [Docker Compose](https://docs.docker.com/compose/install/)。 即
 
 ```bash
 docker-compose -f docker-compose.yaml up -d
@@ -47,8 +52,8 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
 
 ```
 
-### 2. 配置并启动 API 服务
-启动系统服务程序，它会提供基于 Http 后端服务
+### 3. 配置并启动 API 服务
+启动系统服务程序，它会提供基于 HTTP 后端服务。
 
 - 安装服务依赖的 python 基础包
 
@@ -77,24 +82,24 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
   启动用 Fastapi 构建的服务
 
   ```bash
-  export PYTHONPATH=$PYTHONPATH:./src
+  export PYTHONPATH=$PYTHONPATH:./src:../../paddleaudio
   python src/main.py
   ```
 
   然后你会看到应用程序启动:
 
   ```bash
-  INFO:     Started server process [3949]
-  2022-03-07 17:39:14,864 ｜ INFO ｜ server.py ｜ serve ｜ 75 ｜ Started server process [3949]
+  INFO:     Started server process [13352]
+  2022-03-26 22:45:30,838 ｜ INFO ｜ server.py ｜ serve ｜ 75 ｜ Started server process [13352]
   INFO:     Waiting for application startup.
-  2022-03-07 17:39:14,865 ｜ INFO ｜ on.py ｜ startup ｜ 45 ｜ Waiting for application startup.
+  2022-03-26 22:45:30,839 ｜ INFO ｜ on.py ｜ startup ｜ 45 ｜ Waiting for application startup.
   INFO:     Application startup complete.
-  2022-03-07 17:39:14,866 ｜ INFO ｜ on.py ｜ startup ｜ 59 ｜ Application startup complete.
+  2022-03-26 22:45:30,839 ｜ INFO ｜ on.py ｜ startup ｜ 59 ｜ Application startup complete.
   INFO:     Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit)
-  2022-03-07 17:39:14,867 ｜ INFO ｜ server.py ｜ _log_started_message ｜ 206 ｜ Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit)
+  2022-03-26 22:45:30,840 ｜ INFO ｜ server.py ｜ _log_started_message ｜ 206 ｜ Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit)
   ```
 
-### 3. 测试方法
+### 4. 测试方法
 - 准备数据
   ```bash
   wget -c https://www.openslr.org/resources/82/cn-celeb_v2.tar.gz && tar -xvf cn-celeb_v2.tar.gz 
@@ -110,40 +115,88 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
 
     输出：
     ```bash
-    Checkpoint path: %your model path%
+    Downloading https://paddlespeech.bj.bcebos.com/vector/audio/example_audio.tar.gz ...
+    ...
+    Unpacking ./example_audio.tar.gz ...
+    [2022-03-26 22:50:54,987] [    INFO] - checking the aduio file format......
+    [2022-03-26 22:50:54,987] [    INFO] - The sample rate is 16000
+    [2022-03-26 22:50:54,987] [    INFO] - The audio file format is right
+    [2022-03-26 22:50:54,988] [    INFO] - device type: cpu
+    [2022-03-26 22:50:54,988] [    INFO] - load the pretrained model: ecapatdnn_voxceleb12-16k
+    [2022-03-26 22:50:54,990] [    INFO] - Downloading sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz from https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_0.tar.gz
+    ...
+    [2022-03-26 22:51:17,285] [    INFO] - start to dynamic import the model class
+    [2022-03-26 22:51:17,285] [    INFO] - model name ecapatdnn
+    [2022-03-26 22:51:23,864] [    INFO] - start to set the model parameters to model
+    [2022-03-26 22:54:08,115] [    INFO] - create the model instance success
+    [2022-03-26 22:54:08,116] [    INFO] - Preprocess audio file: /home/zhaoqingen/PaddleSpeech/demos/audio_
+    searching/example_audio/knife_hit_iron3.wav
+    [2022-03-26 22:54:08,116] [    INFO] - load the audio sample points, shape is: (11012,)
+    [2022-03-26 22:54:08,150] [    INFO] - extract the audio feat, shape is: (80, 69)
+    [2022-03-26 22:54:08,152] [    INFO] - feats shape: [1, 80, 69]
+    [2022-03-26 22:54:08,154] [    INFO] - audio extract the feat success
+    [2022-03-26 22:54:08,155] [    INFO] - start to do backbone network model forward
+    [2022-03-26 22:54:08,155] [    INFO] - feats shape:[1, 80, 69], lengths shape: [1]
+    [2022-03-26 22:54:08,433] [    INFO] - embedding size: (192,)
     Extracting feature from audio No. 1 , 20 audios in total
+    [2022-03-26 22:54:08,435] [    INFO] - checking the aduio file format......
+    [2022-03-26 22:54:08,435] [    INFO] - The sample rate is 16000
+    [2022-03-26 22:54:08,436] [    INFO] - The audio file format is right
+    [2022-03-26 22:54:08,436] [    INFO] - device type: cpu
+    [2022-03-26 22:54:08,436] [    INFO] - Model has been initialized
+    [2022-03-26 22:54:08,436] [    INFO] - Preprocess audio file: /home/zhaoqingen/PaddleSpeech/demos/audio_searching/example_audio/sword_wielding.wav
+    [2022-03-26 22:54:08,436] [    INFO] - load the audio sample points, shape is: (6391,)
+    [2022-03-26 22:54:08,452] [    INFO] - extract the audio feat, shape is: (80, 40)
+    [2022-03-26 22:54:08,454] [    INFO] - feats shape: [1, 80, 40]
+    [2022-03-26 22:54:08,454] [    INFO] - audio extract the feat success
+    [2022-03-26 22:54:08,454] [    INFO] - start to do backbone network model forward
+    [2022-03-26 22:54:08,455] [    INFO] - feats shape:[1, 80, 40], lengths shape: [1]
+    [2022-03-26 22:54:08,633] [    INFO] - embedding size: (192,)
     Extracting feature from audio No. 2 , 20 audios in total
     ...
-    2022-03-09 17:22:13,870 ｜ INFO ｜ main.py ｜ load_audios ｜ 85 ｜ Successfully loaded data, total count: 20
-    2022-03-09 17:22:13,898 ｜ INFO ｜ main.py ｜ count_audio ｜ 147 ｜ Successfully count the number of data!
-    2022-03-09 17:22:13,918 ｜ INFO ｜ main.py ｜ audio_path ｜ 57 ｜ Successfully load audio: ./example_audio/test.wav
+    2022-03-26 22:54:15,892 ｜ INFO ｜ main.py ｜ load_audios ｜ 85 ｜ Successfully loaded data, total count: 20
+    2022-03-26 22:54:15,908 ｜ INFO ｜ main.py ｜ count_audio ｜ 148 ｜ Successfully count the number of data!
+    [2022-03-26 22:54:15,916] [    INFO] - checking the aduio file format......
+    [2022-03-26 22:54:15,916] [    INFO] - The sample rate is 16000
+    [2022-03-26 22:54:15,916] [    INFO] - The audio file format is right
+    [2022-03-26 22:54:15,916] [    INFO] - device type: cpu
+    [2022-03-26 22:54:15,916] [    INFO] - Model has been initialized
+    [2022-03-26 22:54:15,916] [    INFO] - Preprocess audio file: /home/zhaoqingen/PaddleSpeech/demos/audio_searching/example_audio/test.wav
+    [2022-03-26 22:54:15,917] [    INFO] - load the audio sample points, shape is: (8456,)
+    [2022-03-26 22:54:15,923] [    INFO] - extract the audio feat, shape is: (80, 53)
+    [2022-03-26 22:54:15,924] [    INFO] - feats shape: [1, 80, 53]
+    [2022-03-26 22:54:15,924] [    INFO] - audio extract the feat success
+    [2022-03-26 22:54:15,924] [    INFO] - start to do backbone network model forward
+    [2022-03-26 22:54:15,924] [    INFO] - feats shape:[1, 80, 53], lengths shape: [1]
+    [2022-03-26 22:54:16,051] [    INFO] - embedding size: (192,)
     ...
-    2022-03-09 17:22:32,580 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 131 ｜ search result http://testserver/data?audio_path=./example_audio/test.wav, distance 0.0
-    2022-03-09 17:22:32,580 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 131 ｜ search result http://testserver/data?audio_path=./example_audio/knife_chopping.wav, distance 0.021805256605148315
-    2022-03-09 17:22:32,580 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 131 ｜ search result http://testserver/data?audio_path=./example_audio/knife_cut_into_flesh.wav, distance 0.052762262523174286
+    2022-03-26 22:54:16,086 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 132 ｜ search result http://testserver/data?audio_path=./example_audio/test.wav, score 100.0
+    2022-03-26 22:54:16,087 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 132 ｜ search result http://testserver/data?audio_path=./example_audio/knife_chopping.wav, score 29.182177782058716
+    2022-03-26 22:54:16,087 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 132 ｜ search result http://testserver/data?audio_path=./example_audio/knife_cut_into_body.wav, score 22.73637056350708
     ...
-    2022-03-09 17:22:32,582 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 135 ｜ Successfully searched similar audio!
-    2022-03-09 17:22:33,658 ｜ INFO ｜ main.py ｜ drop_tables ｜ 159 ｜ Successfully drop tables in Milvus and MySQL!
+    2022-03-26 22:54:16,088 ｜ INFO ｜ main.py ｜ search_local_audio ｜ 136 ｜ Successfully searched similar audio!
+    2022-03-26 22:54:17,164 ｜ INFO ｜ main.py ｜ drop_tables ｜ 160 ｜ Successfully drop tables in Milvus and MySQL!
     ```
+
   - 前端测试（可选）
   
     在浏览器中输入 127.0.0.1:8068 访问前端页面
     
-    注：如果浏览器和服务不在同一台机器上，那么 IP 需要修改成服务所在的机器 IP，并且 docker-compose.yaml 中相应的 API_URL 也要修改，并重新起服务即可
+    注：如果浏览器和服务不在同一台机器上，那么 IP 需要修改成服务所在的机器 IP，并且 docker-compose.yaml 中相应的 API_URL 也要修改，然后重新执行 docker-compose.yaml 文件，使修改生效。
 
     - 上传音频
     
-      下载数据并解压到一文件夹，假设为 /home/speech/data，那么在上传页面地址栏输入 /home/speech/data 进行数据上传
+      在服务端下载数据并解压到一文件夹，假设为 /home/speech/data/，那么在上传页面地址栏输入 /home/speech/data/ 进行数据上传
     
       ![](./img/insert.png)
 
     - 检索相似音频
 
-      选择左上角放大镜，点击 “Default Target Audio File” 按钮，上传测试音频，接着你将看到检索结果
+      选择左上角放大镜，点击 “Default Target Audio File” 按钮，从客户端上传测试音频，接着你将看到检索结果
 
       ![](./img/search.png)
 
-### 4. 结果
+### 5. 结果
 
 机器配置：
 - 操作系统: CentOS release 7.6 
@@ -158,9 +211,9 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
 
   ![](./img/result.png)
 
-基于 Milvus 的检索框架在召回率 90% 的前提下，检索耗时约 2.9 毫秒，加上特征提取(Embedding)耗时约 500毫秒(测试音频时长约 5秒)，即单条音频测试总共耗时约 503 毫秒，可以满足大多数应用场景
+基于 Milvus 的检索框架在召回率 90% 的前提下，检索耗时约 2.9 毫秒，加上特征提取(Embedding)耗时约 500 毫秒(测试音频时长约 5 秒)，即单条音频测试总共耗时约 503 毫秒，可以满足大多数应用场景。
 
-### 5. 预训练模型
+### 6. 预训练模型
 
 以下是 PaddleSpeech 提供的预训练模型列表：
 
diff --git a/demos/audio_searching/requirements.txt b/demos/audio_searching/requirements.txt
index 95c6140d..057c6ab9 100644
--- a/demos/audio_searching/requirements.txt
+++ b/demos/audio_searching/requirements.txt
@@ -1,7 +1,8 @@
 diskcache==5.2.1
+dtaidistance==2.3.1
 fastapi
 librosa==0.8.0
-numpy
+numpy==1.21.0
 pydantic
 pymilvus==2.0.1
 pymysql
diff --git a/demos/audio_searching/src/config.py b/demos/audio_searching/src/config.py
index 70ac494c..3d6d3d43 100644
--- a/demos/audio_searching/src/config.py
+++ b/demos/audio_searching/src/config.py
@@ -16,7 +16,7 @@ import os
 ############### Milvus Configuration ###############
 MILVUS_HOST = os.getenv("MILVUS_HOST", "127.0.0.1")
 MILVUS_PORT = int(os.getenv("MILVUS_PORT", "19530"))
-VECTOR_DIMENSION = int(os.getenv("VECTOR_DIMENSION", "2048"))
+VECTOR_DIMENSION = int(os.getenv("VECTOR_DIMENSION", "192"))
 INDEX_FILE_SIZE = int(os.getenv("INDEX_FILE_SIZE", "1024"))
 METRIC_TYPE = os.getenv("METRIC_TYPE", "L2")
 DEFAULT_TABLE = os.getenv("DEFAULT_TABLE", "audio_table")
diff --git a/demos/audio_searching/src/encode.py b/demos/audio_searching/src/encode.py
index eba5c48c..83b9e3df 100644
--- a/demos/audio_searching/src/encode.py
+++ b/demos/audio_searching/src/encode.py
@@ -15,7 +15,12 @@ import os
 
 import librosa
 import numpy as np
+from config import DEFAULT_TABLE
+
 from logs import LOGGER
+from paddlespeech.cli import VectorExecutor
+
+vector_executor = VectorExecutor()
 
 
 def get_audio_embedding(path):
@@ -23,16 +28,9 @@ def get_audio_embedding(path):
     Use vpr_inference to generate embedding of audio
     """
     try:
-        RESAMPLE_RATE = 16000
-        audio, _ = librosa.load(path, sr=RESAMPLE_RATE, mono=True)
-
-        # TODO add infer/python interface to get embedding, now fake it by rand
-        # vpr = ECAPATDNN(checkpoint_path=None, device='cuda')
-        # embedding = vpr.inference(audio)
-        np.random.seed(hash(os.path.basename(path)) % 1000000)
-        embedding = np.random.rand(1, 2048)
+        embedding = vector_executor(audio_file=path)
         embedding = embedding / np.linalg.norm(embedding)
-        embedding = embedding.tolist()[0]
+        embedding = embedding.tolist()
         return embedding
     except Exception as e:
         LOGGER.error(f"Error with embedding:{e}")
diff --git a/demos/audio_searching/src/test_main.py b/demos/audio_searching/src/test_main.py
index 331208ff..32030bae 100644
--- a/demos/audio_searching/src/test_main.py
+++ b/demos/audio_searching/src/test_main.py
@@ -11,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import zipfile
-
-import gdown
 from fastapi.testclient import TestClient
 from main import app
 
+from utils.utility import download
+from utils.utility import unpack
+
 client = TestClient(app)
 
 
@@ -24,11 +24,11 @@ def download_audio_data():
     """
     download audio data
     """
-    url = 'https://drive.google.com/uc?id=1bKu21JWBfcZBuEuzFEvPoAX6PmRrgnUp'
-    gdown.download(url)
-
-    with zipfile.ZipFile('example_audio.zip', 'r') as zip_ref:
-        zip_ref.extractall('./example_audio')
+    url = "https://paddlespeech.bj.bcebos.com/vector/audio/example_audio.tar.gz"
+    md5sum = "52ac69316c1aa1fdef84da7dd2c67b39"
+    target_dir = "./"
+    filepath = download(url, md5sum, target_dir)
+    unpack(filepath, target_dir, True)
 
 
 def test_drop():
diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index 91974761..56eccd13 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -169,7 +169,7 @@ class VectorExecutor(BaseExecutor):
     @stats_wrapper
     def __call__(self,
                  audio_file: os.PathLike,
-                 model: str='ecapatdnn-voxceleb12',
+                 model: str='ecapatdnn_voxceleb12',
                  sample_rate: int=16000,
                  config: os.PathLike=None,
                  ckpt_path: os.PathLike=None,

From 5ae57206f34218b416658ac162b7bad61d061390 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 28 Mar 2022 09:01:58 +0800
Subject: [PATCH 074/126] add paddlespeech vector modules __init__.py

---
 demos/speaker_verification/README.md     |  6 +++---
 demos/speaker_verification/README_cn.md  |  2 +-
 paddlespeech/vector/cluster/__init__.py  | 13 +++++++++++++
 paddlespeech/vector/io/__init__.py       | 13 +++++++++++++
 paddlespeech/vector/modules/__init__.py  | 13 +++++++++++++
 paddlespeech/vector/training/__init__.py | 13 +++++++++++++
 paddlespeech/vector/utils/__init__.py    | 13 +++++++++++++
 7 files changed, 69 insertions(+), 4 deletions(-)
 create mode 100644 paddlespeech/vector/cluster/__init__.py
 create mode 100644 paddlespeech/vector/io/__init__.py
 create mode 100644 paddlespeech/vector/modules/__init__.py
 create mode 100644 paddlespeech/vector/training/__init__.py
 create mode 100644 paddlespeech/vector/utils/__init__.py

diff --git a/demos/speaker_verification/README.md b/demos/speaker_verification/README.md
index c289e8f4..c4d10ccf 100644
--- a/demos/speaker_verification/README.md
+++ b/demos/speaker_verification/README.md
@@ -34,13 +34,13 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   
   Usage:
   ```bash
-  paddlespeech asr --help
+  paddlespeech vector --help
   ```
   Arguments:
   - `input`(required): Audio file to recognize.
-  - `model`: Model type of asr task. Default: `conformer_wenetspeech`.
+  - `model`: Model type of vector task. Default: `ecapatdnn_voxceleb12`.
   - `sample_rate`: Sample rate of the model. Default: `16000`.
-  - `config`: Config of asr task. Use pretrained model when it is None. Default: `None`.
+  - `config`: Config of vector task. Use pretrained model when it is None. Default: `None`.
   - `ckpt_path`: Model checkpoint. Use pretrained model when it is None. Default: `None`.
   - `device`: Choose device to execute model inference. Default: default device of paddlepaddle in current environment.
 
diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md
index b4e5ca65..e2799b75 100644
--- a/demos/speaker_verification/README_cn.md
+++ b/demos/speaker_verification/README_cn.md
@@ -33,7 +33,7 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   
   使用方法：
   ```bash
-  paddlespeech asr --help
+  paddlespeech vector --help
   ```
   参数：
   - `input`(必须输入)：用于识别的音频文件。
diff --git a/paddlespeech/vector/cluster/__init__.py b/paddlespeech/vector/cluster/__init__.py
new file mode 100644
index 00000000..97043fd7
--- /dev/null
+++ b/paddlespeech/vector/cluster/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/paddlespeech/vector/io/__init__.py b/paddlespeech/vector/io/__init__.py
new file mode 100644
index 00000000..97043fd7
--- /dev/null
+++ b/paddlespeech/vector/io/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/paddlespeech/vector/modules/__init__.py b/paddlespeech/vector/modules/__init__.py
new file mode 100644
index 00000000..97043fd7
--- /dev/null
+++ b/paddlespeech/vector/modules/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/paddlespeech/vector/training/__init__.py b/paddlespeech/vector/training/__init__.py
new file mode 100644
index 00000000..97043fd7
--- /dev/null
+++ b/paddlespeech/vector/training/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/paddlespeech/vector/utils/__init__.py b/paddlespeech/vector/utils/__init__.py
new file mode 100644
index 00000000..97043fd7
--- /dev/null
+++ b/paddlespeech/vector/utils/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

From 7c291dfa9e0b5aad1f5130d12e6c46c2fefb871d Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Mon, 28 Mar 2022 10:15:42 +0800
Subject: [PATCH 075/126] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 083a0e6f..a8520970 100644
--- a/README.md
+++ b/README.md
@@ -179,6 +179,7 @@ Via the easy-to-use, efficient, flexible and scalable implementation, our vision
 <!---
 2021.12.14: We would like to have an online courses to introduce basics and research of speech, as well as code practice with `paddlespeech`. Please pay attention to our [Calendar](https://www.paddlepaddle.org.cn/live).
 --->
+- 👏🏻  2022.03.28: PaddleSpeech CLI is available for Speaker Verfication.
 - 🤗  2021.12.14: Our PaddleSpeech [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) and [TTS](https://huggingface.co/spaces/KPatrick/PaddleSpeechTTS) Demos on Hugging Face Spaces are available!
 - 👏🏻  2021.12.10: PaddleSpeech CLI is available for Audio Classification, Automatic Speech Recognition, Speech Translation (English to Chinese) and Text-to-Speech.
 

From d879ba2fb090cac3f278973dacddbace4020d641 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Mon, 28 Mar 2022 10:20:38 +0800
Subject: [PATCH 076/126] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index a8520970..1144d3ab 100644
--- a/README.md
+++ b/README.md
@@ -179,6 +179,7 @@ Via the easy-to-use, efficient, flexible and scalable implementation, our vision
 <!---
 2021.12.14: We would like to have an online courses to introduce basics and research of speech, as well as code practice with `paddlespeech`. Please pay attention to our [Calendar](https://www.paddlepaddle.org.cn/live).
 --->
+- 👏🏻  2022.03.28: PaddleSpeech Server is available for Audio Classification, Automatic Speech Recognition and Text-to-Speech.
 - 👏🏻  2022.03.28: PaddleSpeech CLI is available for Speaker Verfication.
 - 🤗  2021.12.14: Our PaddleSpeech [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) and [TTS](https://huggingface.co/spaces/KPatrick/PaddleSpeechTTS) Demos on Hugging Face Spaces are available!
 - 👏🏻  2021.12.10: PaddleSpeech CLI is available for Audio Classification, Automatic Speech Recognition, Speech Translation (English to Chinese) and Text-to-Speech.

From 2177a19dd9d79831e7a0b1bac1201588dc44d821 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Mon, 28 Mar 2022 10:20:44 +0800
Subject: [PATCH 077/126] Update README_cn.md

---
 README_cn.md | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/README_cn.md b/README_cn.md
index f5f5a9e3..ab4ce6e6 100644
--- a/README_cn.md
+++ b/README_cn.md
@@ -180,7 +180,9 @@ from https://github.com/18F/open-source-guide/blob/18f-pages/pages/making-readme
 <!---
 2021.12.14: We would like to have an online courses to introduce basics and research of speech, as well as code practice with `paddlespeech`. Please pay attention to our [Calendar](https://www.paddlepaddle.org.cn/live).
 --->
-- 🤗 2021.12.14: 我们在 Hugging Face Spaces 上的 [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) 以及 [TTS](https://huggingface.co/spaces/akhaliq/paddlespeech) Demos 上线啦!
+- 👏🏻 2022.03.28: PaddleSpeech Server 上线! 覆盖了声音分类、语音识别、以及语音合成。
+- 👏🏻 2022.03.28: PaddleSpeech CLI 上线声纹验证。
+- 🤗  2021.12.14: Our PaddleSpeech [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) and [TTS](https://huggingface.co/spaces/KPatrick/PaddleSpeechTTS) Demos on Hugging Face Spaces are available!
 - 👏🏻 2021.12.10: PaddleSpeech CLI 上线！覆盖了声音分类、语音识别、语音翻译（英译中）以及语音合成。
 
 ### 技术交流群
@@ -578,6 +580,15 @@ author={PaddlePaddle Authors},
 howpublished = {\url{https://github.com/PaddlePaddle/PaddleSpeech}},
 year={2021}
 }
+
+@inproceedings{zheng2021fused,
+  title={Fused acoustic and text encoding for multimodal bilingual pretraining and speech translation},
+  author={Zheng, Renjie and Chen, Junkun and Ma, Mingbo and Huang, Liang},
+  booktitle={International Conference on Machine Learning},
+  pages={12736--12746},
+  year={2021},
+  organization={PMLR}
+}
 ```
 
 <a name="欢迎贡献"></a>
@@ -625,7 +636,6 @@ year={2021}
 ## 致谢
 
 - 非常感谢 [yeyupiaoling](https://github.com/yeyupiaoling)/[PPASR](https://github.com/yeyupiaoling/PPASR)/[PaddlePaddle-DeepSpeech](https://github.com/yeyupiaoling/PaddlePaddle-DeepSpeech)/[VoiceprintRecognition-PaddlePaddle](https://github.com/yeyupiaoling/VoiceprintRecognition-PaddlePaddle)/[AudioClassification-PaddlePaddle](https://github.com/yeyupiaoling/AudioClassification-PaddlePaddle) 多年来的关注和建议，以及在诸多问题上的帮助。
-- 非常感谢 [AK391](https://github.com/AK391) 在 Huggingface Spaces 上使用 Gradio 对我们的语音合成功能进行网页版演示。
 - 非常感谢 [mymagicpower](https://github.com/mymagicpower) 采用PaddleSpeech 对 ASR 的[短语音](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_sdk)及[长语音](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_long_audio_sdk)进行 Java 实现。
 - 非常感谢 [JiehangXie](https://github.com/JiehangXie)/[PaddleBoBo](https://github.com/JiehangXie/PaddleBoBo) 采用 PaddleSpeech 语音合成功能实现 Virtual Uploader(VUP)/Virtual YouTuber(VTuber) 虚拟主播。
 - 非常感谢 [745165806](https://github.com/745165806)/[PaddleSpeechTask](https://github.com/745165806/PaddleSpeechTask) 贡献标点重建相关模型。

From 2f4aa3cd155228fa170a7adf8e6a3a24216bcd53 Mon Sep 17 00:00:00 2001
From: qingen <qingenz123@126.com>
Date: Mon, 28 Mar 2022 10:38:36 +0800
Subject: [PATCH 078/126] [vec][search] update search picture, test=doc #1608

---
 demos/audio_searching/img/search.png | Bin 86119 -> 83102 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/demos/audio_searching/img/search.png b/demos/audio_searching/img/search.png
index 26bcd9bddc202b0c129bfae432d5391b9a641d81..cccc7fb92c91369c3804f00133ac310b4b8abedc 100644
GIT binary patch
literal 83102
zcmeFZcT|&E*EfteGh%^Y2MhrbyC5J+3lJ4YR1j2D5D*L?QCg4^Y7!kF5fu<n0cko2
z%FqHxFHs^OQX-&`&|>H%5K2f0Y2Sr__uS9Sv)1>=yVmorcRg9UG$-deXP<rc+2yzQ
zxxz1+8B1;0xj{@!OzPa()0f4>)<MO@epy(z7HE<GW+MXrS>t`#_>@>~i~Jby<rl=s
z3n#_I@}oBL;lBdk*L$A5?ky&^WAD=6HRmqxnHCchrJp-}@`}G5Lx6O%9ZCKkG(L6H
zlQUmw)lyN^S*Cr$adVW+{rMv=G>W_L-M3rQ6%iu4RW@RmyX5cd<X)>fZ7B`=<>9mS
zJyXAxywuovDyvoDa6*SwA2|08#kw{BnMs7I^zi}}y#mPRg6&7&oxb$&mc?@?1DiMF
zN&T4?=Fz_uJbV*5K36N~8T*=QfmJBy_j9>P7esR+K2{Q-_{T@UnyO&VH)UXjWKqCs
z<LDdjk(Q-KHd%Q6-;Go%HHFH83Cu<K)y6O8ifEr#2(54Dh1k_LwD(wBdP*ilO}Klt
zsmhBxJM>?hhCV;L8n8I_Caq+@OZ_9_x|Pl+@;>5r=$%^&IhxON{jco~vmv15wF>{)
zf1AV(2Zs<)0)OB?+XwO69UR0#zfm3kbDTj5@I!|XspLITdH+G=WqD7lsIHlQwSTRU
z^5t;;tFWR!|D)3uu(egz|7a)b%bLS^l?5I}1GV<QtWf&WQW6l?<<Bz*$}9X-kdL|l
zuwv@D$@MheK##z_RLzYiT~U+E)ZaAKV1T#dI#a&!r(6|8&Vw4sJ#tgLacNR+JrSa?
z;KjhKjI{RL5FVo7?`H4u{^c+eBdhR0-$OYV8LXsIv?-F$V^lN4lx5lyT~$-MWTp=5
zmmsS>?H7gzcr|mzZ)(0~CJh+Z6k<yb2hY)Z<gj)_jExp!92@2&I=cQNnwj2Hu4)#B
z<|=Jpoz1Kk52==k930s=u8Iq<cZixKMv_1x;n_*-P~ryO2J&PTlw=gNpHvgU+6ygu
z`0Cz=;tn{TO?;I}j4}W%4|DJ2rAauxeqW74`N?GD%&7E06tfT*#@>Io=`d}SF-K1f
z8)0{eV`19`9$h;Z;dm%%>wfDU`}IYL{Q5P^gQy=kWWbmvrIlS6=gUbNrmU;j9mOnZ
ziOYY>oh>1yEZ)*Wh2g88R8%Qe6ZV_#Xc;BwK;=^Ms@NsOnTrc_n*P=lNa}_{#m*^N
zV1kbZ^zI_&U;?&B289-&%CaSvXL&!FRYL-qzjGTkPmOoejr}kfn?J(pEFgivV+{=~
zgV)JuE&WYFwgnBMWaJr-yg%lDZt}&w&mgGMIpi#Ou_0Yz142_Le}9i$Kp#44YCU-m
z!s>1J)@n|DSUsPb+an-{-iS8Y*~6Xha;)1=`ikEOr;{?(eOc5+xixyy&>orU62Gx{
zvc7_%dg&Z3SdAf<xX$8D2C1h05uEPk#TX%_7&9MXo<|C+Z*>y~{PMF0O}8}!1q2nd
z?RM0#G8;Sx6``rm`cq&6IAzY<xMnajQymT0P_c^#S=2{;g-sZ^M~R#%&v=nwT0OuF
zY^T5QYb%=ms>b^d+66DZBJ3%qm_i^W(_bmV!28C@=U^lnx?)oBPI!Uw`G&)5f=3Wh
zW-6;UC0kTz3{KSyP~3`t4z)4%>Uc2X&4lmPx51M0nY-A*E28x9kRHPmLGj+*`_=3P
z+ED*W<z$@`Wh?cb-3()6*vb2uL^2JFSYX30lq8+}gBn3qpGUoM#Hz_R(5~uFyq8hf
z5EMD0_@%F(s!R`uH5(vwVJQbH>%-K`w80y!8UQ-0V&Shm<sLgfDx`CbZPez!%vKp&
z^QXDfRnLnKfcELPmmk?^T*E1-&t>%<Q?$_^E~JM$&Wr#jrPC|pQ3o+RlaLo+5ASu!
zn?F6tR4y0FxG>WeIP5p(q2W}P@YF`&?D#BVJH7WUnIzzIFZZf9g&vi9hk77IFl02}
z&2$4_x!wEJJR3owR}&`)t&p5<I9{Trx}-wi#m&fI91rgwVCiy?n1&(J5Y|?+C+=#B
z>Kv^2rTbufg3fmK&{{QiRdQd-JK2EXoUj?@klh>gcmm-fuh{FjglzeOhn^JFXJ^OT
z64`wUjeU)WaF7`0ZUM}(IdX{HaxrKsj1}oIP9hjJmcK5_HEJQk+RE1!IYi|sYQi4J
zxCG<5%p;f}hCz`+-gNStA`s+w_PkXVFW0gDwxD;Em3$0ivYV*!Yu5EQ4CHY(`s;QK
zai=dv`Y;x+J5P)rGBl5)`OP46+zrb4;L-lV+~K(*A?WL&>b%8?B2mQt@w|3C7Pov}
zN3&x{AP(fVobNg^R=J9~F-(f-mzrsh+9m4gkjnW~cLOf&ekUYH=LA94xt<e>6edC^
z;ynF_uiW)Jf@RbU-|(==o-h1r9lwFh2Uuv3I3>wJyIT4SdpjoQYtvwEhJLz!hF#SG
zy^Mh|&%uUI2@2I0OSgTZB1k2L^oMG&&HKDlVIo8JH$&dnS2yU1_GXGUunr@7f?vDD
zezg8()Gf|-iLK+`s>vn$Yvvl#51D<nD`A%+g$170p|w=?89$h0w+oG%p$6MjEm$ao
zk57)YM>Mh>k5u#A7U-#c7Q|ob3d5Kh#Dtzj?ffQ!QCnCzVZ??L1|{;F32*#|A1un9
z*-4%(%>^`l@EkHSl-mFGw4&DeZmpZ`7l!-K1O|&62pF{y^idqMjkC?*%#=ZP<-s0*
zv&JA(QG$vGDsRl{c!bpSkk;f}S5)glwHNK=VpXnCaNMWDteI1Aktg!i^Ke5K(0~8r
z?ZDI!axPlHT0WCNlT7IT*&BQFWOx<VXOO!VODKO<9KT&@H-j_?u%Bcq2?Kj`k1xw`
zmBj}*9$z!ZHJGN3khBvyy;O>iixwn>JHYpj)0*9x<3OPUn(z6UNBdI_NNx-+oJSYQ
zLwijdT|6FcH#C_P7}```sg*$$fv)$&OPUWd?dy_7Qwi$Nu)U`m)K7TFzZ$N_9+Hga
zU!Rz%7F?dXDHHisV#_S2<PP_Ux6wiOkoAv>nB4mvBgIm2r>hfpggNdbZRZx=<j)jB
z)WEHa+R?Nom28{tpaa<#ykDl3=1$UvlCkZJoUc(4ZINIq_d+iSJTDc&e0Pha&0bh^
zw}AgSIlq%khM|To&?&e8MsbnC;cvBP0*n~4H-dAwONc-BDKZOZ1@*-Vh+N)L2(0rE
zkwoFy&uFU7q2FAyD?A;?EP8aDF+4vWHRf52-_clU=a;&1`N-S-*+(Fy8Yo&jL#&TL
zzF-bM8pD?74GD`t(06XE{c$H3?$QH}Cr9;CDqTBzwjFl+l8P>#G~E_{qki_ynzs-R
zbTVp;kup6=gl@^uv%L(T`C2{fIt6{!7tflx;xAN8T~8i_G-Bs+5-|-_RrVglEuG`r
zon>~!W?kR*8LJpZhIHTNn-%20nE1Fghf{J;zR-OjUp@Xw1#tq7|HR8idTBMLbT^mh
zvM`-$#cOPOPDgqCIr5H{irLQ{*e7cGs!*7>Kv{&D<cJ>UR34P?9-Z=<tGK`qI82L^
z39yw35zC_Tr<{{tlzda{T<8u5y;UhxQmjChai}Q!JRIaXH^_C=t16C{ojGJ^-}^B2
zAR?jDX2y3}oJPw&OF@IZvxi^_(ha$b{q?%F7ue4=VXwis=G}?5^HMqK1yZ#g_e1#C
zH$3+-L^;moUN1h#j_CLhVFq42Ts@Kt_&q_#P#Jhewm2sYLC(<jy;?lRIYIHRoO~HQ
z%dB`5AyhOEeU9YENf*H~YR}g*2y2h$(2y9-!s(<>``pp7h1VB*`mgUXtJON$D2F=G
z_ngnXrOls6{E!2#oM#^9+&C6>iK27F@JJA2HuZQxYcD)D;6nZ9>s)R)+wo&+9M-m=
zxL4%86KcnEj~GA@jpuoeM&gZ9VyCs(aXe60N1g-;6yfOO&%BU7aN1*{mS7}Pf7^OE
z|8Vt+!((8X56AbXOO<ENZ~50@5a_G+yXT*^WKwma`!43OD?d@U>nT>`U+Prp-o~rZ
zyy5oHZzjcZU}F*IvQDGXfM&5Y^qpk%y9CJ~mwMi?nhoB3N1x%A6aiqT@ihMfZqO$p
zA1<f2_6(L@pw#H`vP0?%!RYRTTNiPVacuV_HtsOeOh!0t9ofDhA3=OoH!~~poAEBf
z*he@dcB%R<c$6DldqeK*UF$n2*hY*pEtkXW;RadP2If|pQQgak%@jw6sK)M>aJ&N*
zdD}v3`r_egt;vaw1;o4QvhmcSi}cjiXTmdGh67#>d2z_?uFK1Qzj)P^LeK9IbrHL}
zw3`dMw5Jpfx%DByzB%NL@Y#lrk-Z2riiP}x+X@wg^^~4%LAy=+&LT|?C58R*N+(o(
zI~)CKA^vw5dYAsD3WKYB6>7Sn^%2Lrjz{$``P)?>OC-%y6tKH{5{SfG-s&Q>PkyU?
z<>T54@@ci1e5ZRj$V2Ad;@q&te3oO{J9Jvf0;jJ!vnQik<3^7@SxRTylzo>zS^A{P
zbr0{VSJNqa?&F==i&noLhQT)Lo+-ahD-NgegTje4vuk#5SQ@HWYy`7pvQX+hfmDDs
z*EJFKY^NdF2E&hnGlmRZUDA(x@E$i_AD%Dg*(?VqFXIj!XM71f7*2VA@=t%c`%(LA
zq}zE9Ns@(*G}}Nsua1zKeKy`kg`OGrJ=xUhiyoujTa3ZrmnK8nzizB|2h^#=o*k}v
zoD+fe_<`KgG5c7z&#Hi@piq_PyRd6!ND&uagJ`O*=UKOW+*HGY6<5b)sL{4gWr}Dy
ziHkXjNz$Q+z6;E^aU#PFB(c^DHT~E1U1-fP!>{asDej`t7$?6?1DbOZhJz?<AeZnH
zws+3o@0#1-2t`v4Ogf9B(wqCtIEeaOS}HbTk!CkJ1JS}+V$36$`l6C6*p{PAC9F*r
z+fg-K+VR~Z#k5K95v2Vv@9}l6_cmd%&7t7>Mz>G(eKsSVk-<|6($HhCQ*(w4d;*eg
z8M->xGK(>fhYT%Rj^>rj?p$Wr7uRocsD+x8j>27l=?KzGp}#~I2K48%sug9(pnO?p
zTFN?Oz&&Q23be=_b{++cjQuG`xSnYwOffR)g{r(}W+EK_f<@dyF7ZSimFK=0A${zw
z9q5SUjwYLsCtp_|j6*qUs08b5&oKKIEX%zIXpS3H&O+G!8Yg<lu_Gokrj$ADpc+k)
zRggI%*3k)!$w8pjX*-h!VUw8#Y--oQL)s@7JVYx@LYVKnu=~zk)Sv8R7&rx{w>=}a
z)5D_=)J8CxtP)wwRA~!aQWO&(2$Sal@yn8;L?RBJJyzXL$?$d@sAOwD<HMq*W-?1+
zp@GL<exV!Wj#*_+;kWeHkFvVZa;VXF4GKIvJ&#Y?9R(9ibySj%dk~cI1cOjzzG@s$
z(F}ev#zs1E>t8<)sCs6<9If25%W48VbgYF=VNJNfBtrGm!ITg18A>jz8o6Vw(yS^j
zv&4qkgaq|*{4Anh;j(K9o=R9`90_z>nlJFGZ+@6c-Subn!Ojblq^*;Pw-1Wm(K%Id
zdaywHDPi?DY3Lrr3IBe|2Bz}~JY<~QEdBeo#N^)iMk}AYJ&d^$kI*+X#G8>f;K`hc
z7gmcc4vF7-Jq(TU8?CEp^9rtI7kHJQH1s6x_XNU#g`~FflsUfgmmer9v%6-9_!kP2
z&^>^M0(C{Ve6G)&XdoC}z%%!A6|DP0;4AC9e>ob61;L5u$})tQFirX30d7b0AUKvD
z&Zvg<Zrumw1(@_&t+Nm<OnM^V7hjgrIq%7lp=13{rzXo{gA@<By%^E{wf#av_NNSr
z0WDyksc)A&QGI&~x5dDnJT^*Zhc!^H%NN8H8dM5`{re|onA0s-j_D*9Qm5uZPBy4k
z3ohg?NYtbZ)OK1+whx$p#p^t`ZX_7>w@goBlOzjk3te8TRtrr1-wI1lf>Cd|H6<j?
zH(j0Z86c{~+uJq!5wx<p?zw~1REqe{-t69YNTD<BoSW>+1@hfAHnVcFNEzJLtB{p*
z72P@G_|eejj}a1RY{ZT|KPz(k;eRV!e1O3x_hAQ-d+yxH(F>-94Q#hB6cqX{fDb&x
zbaV{l@U>&^V43cDNXoMBee>m&;U<v4lbmX}O=0njgH37Pz+Mf9srAxdQ{()1^=w5q
z?L>3$vr46;gLP-)b9<MdhPM3i>Ogxx1pe%{zD6%{)f~;u)zAaYEGtZ(Kc9_i{v0dv
zC>A98ynn6!xNy3yq&sCLWIoYzQ5t$ecS^DX2ttzbW%ps^gXEXJ0Wl36<fR%rES%af
z%^0A$8Hm@$5g}{{vh4l&b%o@SBO6y>o%iP@p(pGa8ymI@6IGAw6WPBG^Bintwe46L
z$ETJc&iV~bQR=9<;y+B2^I2)PQYC@D$N<rUq0lW}=Qw1Cp`GNTgfObQFaNCn@C*_c
z^aojdx$n)rILOLV_s(lDR{kq?<>893iHW_u{4<sp0|1*9+(GQ^Hqwe&7dw$Ii&}l~
z5J1w(|1Tc=zi{ZMDIn-C^XcY>+*Ls8tQ_j?iTtn?s_(&XQC|J4cE{@4aL?fQ4clyV
zR@NkY4F@@N(T%?H^`_PTKKwt(u;YLX#vQq_0a-PF&3`OC?gfU|P3u7Bo%+jm`gVh$
zdqO0M{2?{cKom?t0nKWSjNyqIv0OEAgN03h+1ip`0_jaE##+(tfs|JEo&dX9{>j6|
zo-2E;p6!tZ$I}E;esnd_tXf)#WbgSF))RTelZUFS2U{|$+nGXNm>FoLhZUV&yKn8C
z?@E6i;1L|}q^_j(j$p^E;@B;27~1id>7Oa07Fg}(o#?TLW-%t#d+OCHC^aj`?LFa7
zK7pi$nRzt&@>Je`{b`UuvO!jD7cQ-do+nz&OB3Iatk0%6&YM{1a|dHvdC)SSvCE5}
zhyE(b&vv5x2O_k`<DNf|+LOIU&S#;QiLr#4dEI&VO;Z0U8ioJs^dAXEN0X+%TA2SP
zvAH~2(dUzO^LwMCH!nH=h1bWYNuY*{H7;tF(K6uW9$wr>1%KtwePEGz>;ist`lR;r
zr9P8;Rbx1@wq}mXkN(2O`wA!>4By}9`%7@Iq@Gg_>bp|3?X|S-fxMEx5Vdm@z)uyi
z`g*qsG2fQD-TMAxZ?v8MgY9ck6naic-$RgWwv%cI=(;P!zZ76cV5sn~>=1`CrXSh!
zck9^_lIE8Nn~&KcQ2ER5Ee!(`yb*1Ks93eY_#{|poLyjM?aV-ZyK<e$l$zm49Fc#E
zwa^#<Y;Kz}yP8t-;lv9$=^$(Yl#xs2&6@HAbM5f{p$2vDbkCq)n~0j5>&#CZw(r-3
z^E2xZeduYQ!4*LJWhXG3lGa7Sj)pHEy>f27-93!u(S#+e2UTjtp100xq3i8rZ9*E<
zx9TcnX>8pm!Q}-bu$1yU7L+T6z9OFq^xRhXNL2(+tC!~#-$W~c&(dhK`}z`)hDhOP
z3qe<4o0uBk1X0s~u#qOPPRD^T-m;wE!>BX9q`~n-;RMkvio$c)Xqho4umOWBvWhhy
z8%Ne3lJAC0d0<lg@V)iBqYk!vkwEPdH6hI$;aQzyu@y+tV|;QmhfvAqrD7I8r1PqM
z->N(Pv>RGo??F&~;le=ctKYwMy@lgX6CbMDCP+fF-7qft3{itJy#7NCPNa8%KRETA
z$(l+vJ*{Z@Vh9H@OpxAct}kEQQ*zu*ZPH8H6^vSN^4P7<jZlR*7EjW@ToL-OU$seu
zR)Bx6ERELTAX+VLn?4d!Kfv!8RiQ~($Z&m4WHhU^yNwy>WGtH)rC+Sl2!cxf-a{}V
zuid(8>)xLMQ{Z?@d-QR=iv(-%amMsuta*kp1DziRK8=GM7099D6?;=K#HdgsHK({j
z+4WGn_$b5bZ8*qo&WbIm(jxrOPY*%cvpdd6LHYeM#IrwGs{KF`>a?iAP~Qp)R_?th
z>UgC-*Dyxln74@p5#mK6NRZijN$BaR6<;FI{U#m~6SL-b%FPchZeIzcA2PQjU{Zg0
z;30tJm*%BX6cnnFCX`^y$r)vPNt>x49XIUwMID2!*p~fBg$)(hf(mD(Y#z*cpQ$hy
zA&|9kLFsF^vy&1WC2FRS?ou{WRcY*$sdb62dHLQV|M=<{Sl^WAXmiI%(ZbE$@(aMu
zOrR4<kRmdwzRISi18HzQ$ZT-lt}jtxF4_}Y{tCnC^gCvc7^}+rkQkI$-(%RN6H6L<
zGi6R>=D<bJ=#gW|AbcB#z!_q5dcRKk9lc>gtXBZpRL|C`7u?wsJagEFqzu#Dv@e!i
z%pj0VWwD0@M?fh@dOBT(KfWSxfK23BA}1R5XU<%)am|#Ipu?9Mc^m|7dW^4v5EGB!
zA-PSV;i*hel$(iYirHl=GBr2!3-rWYU6$;ZyVnL%lyoadj!_0XAV*~PM8Smqi;+AF
zH~QLE03Ot&fANd?5aQ#gGn|Vj-Ov2wZQqw*xQ|Ln1u`@49sza}b&F%3qqL&FRPEVk
zUA&+p(7p`9+)&TNV3J{}ft%bNEQWNJGMhG|o*Kk&8(})7zY1-65hA4wt+6Vmrh-*U
zMVUi5iE{n+({s=%R&o=*;f+GBw4zg#Yjk!m9gy*I*$wfDl0n4$8rr8%&k1wWM~%RZ
z)&m5isR|E~NBM%S1-^!z_gDpu`+bo%Xgj!)2R-ri1Q3-}&3D+et_e9AQ(2GMVN-hY
z-7LYV{9tgWfF8Xc6gsoD9q#O$G3+CY9gimHcJq6SVWu*viC`3k_cc=eGLOil6dM?9
z{lF&7jZo_XCw6^=bfa(rHB;Q!pl?Qu<`y^iq=Pq|?hNP_=fec7konHr@{WEwPpdk+
zG}st~Ng(a4zo)3}$w^TQLwz#}n|Ze~`gt>}CtbDr9n|!l{{B2|66g&_4ZUC4K0rsG
zI~VR*E=)EJ6*t%Kk(JCu=TRN688Fl>gHkyeoc*LdShkanC&Op4$~@G_d<bu@)!FAb
zASsh&Q9~d>0fj^5RlB-U1h$JDD^Z30qN%byP0>td1&|awEyX7oUG@rpKzZ_E*XYR5
z;R1(^+7HU*P&)6J7YB}RiRPIz8;&#*_A*rM4!tbfjP)z!nJCjgpl3AYnmQHy`~-Is
zYliFV(&8lIbt}E^5dO$;pIZzuqDT9P>tP=EocK*>y79sdkc`g+Ws0TQ3qdKmRw0K+
zq_z8u2dnoQ$f7>Evpcni99?yauy-to&b}nj9K7F{`sDL4!SNj_0qo*hXaM0;e3)M1
z3dw^wh_L^np4(On+1FNcfbi?Ea(oEGeO)k_Xn5QpvUrB0vjtU2c)n)EF_krZCW|_|
zxMZ=3wx`{`zf=Khj6T`6yzQh7eLdjfuc^d?3?182wUW)mF>^L*LG62N#1KoPRVlP7
zSeEyw1%ZlR80#I=D8}cT#qIdyjzKD_dR*jqQIt4#jtFGD;>Paw&Yj#x`Oh0&RIZB%
zMsv{X^vm*GlYnd&3Oz)ePG<%9VTtpzyMEJ@P`s0J-Ep3@rVNfB+D1N70HiyIUw%G2
z$lMbfB=m#kC@mktCzc6te74j3?+yD?AA=O#p4tOik=~YWG6o&TF6q{Zov6~Iiq4Tt
zcaH-l?oUUw`k@|_5iKE8*l?P!kwAjJsCVtHG=#AGI!mkOV>bV`6^;np5mJG*4$e^k
zl!5$)sMqx_*yPh5vVRQRs}ICOo>#|Spy_QhusPHIwe8}27a%*d23O~*=7)QD1@nHu
z_;9v&Vgv-GQ-v@YU`G#$65<zHkBX}<(|aNjKyID(lGZLcY)d|-*2k>8YHMuO(TA!%
zyT$`4aEzWQKNv|72#jkwc*UX3e(B&^-DEuEOifE#S@)b50yoOHZ+)ZZ&KSXMJR`hq
zLb)7-O%B$ma5Cz?c$x`7$44wSm%TG~5r#p=5<psEc&tJNpQML}c#(d!VSiLpY<mi%
z@|p>xwgly3HSaT#O=AR+C|Y4;Lc$uq<=Rs)$_f>qe)9qe#MO)l)>zd3$}*-Voaykt
zwrzP0vPXdwtZc^80l21sieJCg+)c{wt77|jWKc<{sy)}iyS*BaxubEHM*W?Ce_$5X
z?C#>1g9J`1sSnm*HxfzEEk2j>35|3|?0IW50sw#FlE1pjqhzrvRaIHd!k)W-r26TC
zbs0dK*>yGMdkJ2Sl-9PMCxN!YdXxHKTZ)4bNLi=`#M#*kans5ptuhpjSEa!=)d9W?
zHN2m?E$|p)dbrOM8<;fJtEeju{JeLs6tt&AS^j}w@?C<h@aT#dok;!>mH|o*uoQeN
zO0*rDKE;f}bNdD1^C0Mwi(-~4`t*S^B>%iA%ELU$vZoTj-rBmE!m-m+>~Et0J3pEi
zm_U0JQ9}J<jg&%U&|F~&k(tiXoK3rBXz>P)?@2ef#y9hd+j1S~Jcg#yT0-dWeT2Fi
zfW+0CJjixZ#zPF(575k!qP8jZ=!%s%+DefG+J2zeGvqcE7-LsWBq@w{RE=p%yQSil
zIGb1v-J$sQYy?azMj@{>Ai29#7+Xd6nbPeA0CUo5iXfeZ4jwomBPtRB{y;Tr#vMFS
z#(U1Sd)|?^kUW?|tnu?&%;Y830|H@$nN$>VXG0Eg?>~Hov+i;Sc+uv!6FRU|2?Try
zE5Yj8OC=;wx3PCz?(G-r@zq7g?h!KJ_%01laM}$2mUi5m=UTa2Ynvz1rFa#jNK!uM
zh4)@B8xJFN4gw#Kg&T7cMVaXw13>>g!P4g7jEV?H_nlOQx?$^7OKsR@R9~WDM#e(2
zDgPL>@#9(id{zQj8d|U>;>A@d=pGV;I)S}oEE`o-Up`p)8oM8`hLm82Mk5vQulVU_
z^-@~*Y!$3v#bOSU&?_&O+?rc>$SmSUX$eYXI#qqRo&JSW5*i&HJ)cZfy6fggNvdkO
zA&sx9w~_eOIdW1;npTu4w2m0~O)GB>EK;_qi5$u3*3N{*OX4vF$9p#Ss*{t~D{-?S
z+EdVTYhi68$iO}FpRtyW4ep<!_`gla$vb0&o`IEJ=p`dEg8#*NAE_^|vzj+l9qR`Q
zl_)eZeoZ?k6mtDQVG#RF18muGTfo0WJ?`3xa0Q%!NjMfhX_#OxG8>PQyH2_e*#g#Q
zOg97rnq;^RUBsyz;^Aw2?w%O<Y#JZz;YSIy983i-Z_=*M1fyS<bcr=EpmYBlY6h}D
zlEAH2g_|_XOeBsbl3E1jTG~%L^%)m6xnBj%Qk;*U6pz~>8WqzMxea19-qaa~#Lm=E
z?$)3<EzVYDdpCLIyG)2li62FsY~P|3G&Njt4a4+RlwT)}Am5;S$>+=>iv4xu7j|K3
zOFm}-Z_5}*^h`~wIxAX&Bt%1BaO$gL+s$F}Q@0I?!@12lG-I#FBom`GSrF9OZT4wp
zC@L=-yk2@L!M2tYQ_>0mr`{3pn7SbI$DwMNy#iZ~dkflb)fN8&fKkimAsvW*H8K6+
zIXU3r<H}gpP5;?Z(?;BEAP%zSO=`DXao@K^y@s_S0j*bYU#zC;u_zfDC69MgpMg;)
zHl%3I)=qy1oK>g#!Xrr_e06Mc*si&oI_i)hw=Dk4)Ac52eAK<Cqldk_t4U?y(@ukP
zdk~j&)UQ?$A2FSVM`P(9+-~m~iFB_ZS<O<P+mMa`(`wdBhbDJ>iSO0*<DVORES+_^
zhG5idDl2LT8M|rTdUl~d1WLN^i$1cvS+c-0OYqO3$<M*@h8hZh;*ySS&+2ZnPv)v?
z8pzez%WfRlpFuJe&aS~17RtIw@nR!{&qua5kwX(Q!@GEpjgIwZ_=JS3l_O;di-_d-
zKOt%U+}{hn_O*{W7LjLbPU}>?AjfFJy!A|-X!U=pwYR3TDom<5i=589PU^)XVgsWh
z3oFuu3(qs|E)(_MYx&UR8*YVFK_MuLLfzV1>sEk+d%3bd;4MeUa2+X9Ej@120uXF$
z4O%U$69-URK>Xw(mAVRbeb<NEBrVPG0(-@C<Z(|w3fBv!mbNm|3E1*TX(4fb9TIX=
z3$cI`F2L7N8e`#s!#?r;a20ufmmom41JB@JG)G0?xfG3M3dLk^FY|5m`)G@L3`o-l
zWlUN87}{vL^81yi%P5;(qUh#P`9K#xif}2aL+jEg3MoCX9cJW@sjIl^@ntgpPvUjC
zrZIcYt^9dSKUC~zyKh-$Z~eP{kCbJ8)xgnZyzg-A&;EPvls=q<!~r-xz((7If483_
zU3FuYnl3N=^IzI`<~jt`-#W^;cpr$Z_pQ#m=UkiT_HO8r*k!|Z^xy3<xGYNFT_8kW
z(u~jk(tgagmL1vX^aHQ_`w}t$q?7+B%~HHd#-{Wihdbd$|3|vk<cIz90zdBf`!q|O
z+VsB@tjfUu+bp`{2@RNaT9_tDw~O}vTN0}dt{j;qLAc{-0%TGFlvQ0z>v`YqfA^YD
z`%5n=(~?}=9MUY*(k@4^rXwlox+zn(uK*}PBxBfxp5tFLSqeWNy>qX^*FYsHbwgg&
zg0&9kPGEOnIc9HCYM^rEZ2xT?|LL2}xBX`<x58(8%Lx95(M?BHF)X`yeJzE$adJ)o
zT))yQ_)rqj%K-iHXwFCxQWaO`x1aPeI-gZ0L83%k)_^TWjI`By$twqQ=>S-8Z%A7r
z$f4mfZHmvoew4sLW=Ws~aS2dpSXdZq^H0UlCg$s|1dyUysq<ozcOg57LLaV4@O&3g
z`9L`q&URV+UYv`gZXK8z`4)joW2%HEj-@@ynJCI@!8IsHk2sx)!f9?ywks1Yon;>z
z5Y#P!e~dqt_(*k1pM8#%e|3qc&pINq==~B6$1Jthd;=VYV2aXAh$4TNyg}eG^wXcZ
z<%&nAObb3!#CDK%@8&Fd>dlYf<DLUnX~dKb-j;>*dx=Gz`X$FqJsGwSCXXD!s7F^v
z&8TxfoGMK*5Y&BB=dfE@0tJx8T=c6D-zv6)j-N`E=#!I9iD#AW#k+Ikw(*O;<;H>j
zhio5Lvd(M+)K$B?CSy7;61zp%&dkMoHzsigI^&Qy(R4TY53-`u5+mhQQ?SajMx6lN
zBXE3EUsVu^T8L;jHu=fJy~kkBx<|kpjrY?w79#eyi>j*O_=yqPZFvOCDZV1QFCIxZ
zmCYlz&^<FI2S+#3P4%)pee_U^VH0rFQHY)nrkYK(w&`^3oPydCnYZJGI#sH)BC2{q
zibj9L+>kAKHz0Z$pO;d%Zf9}w*DQ{1nnAwVkyZ~ry-eYSi@wej_z*ze=;-azyUOg1
zK|xwG0~g@!mZ|Ft$3#=c;F+C2=`Hzf&H~P=?X3AKAwzONJ^y`J>?7dB#K1%A`skxy
zYgxq(CWwnVd^33eOI=SwQ_$JIAc00Ky!cjPAUdyVx`n<tncSii?Q{1K<XwD{JNO;t
z!IJ3x$s!neY2mf=Fe#X#EarPA@^lpR_7Vbn)I|>E58s`{9d(FrF&l_E%J?Dx(ycS~
ziqiYdlFp_EK(#lc+{Zb)YR**RR^_41{W8NzU?7JAa=~L%>3s;jC~=j_WRO%Prur7)
z<_xKGXNjVr1@tn02N~Zv&h$QJdgDAYt0bk<PIAN^sbAtQrffMUh%ocTV9_c+IJlHc
z0mUIds$JJW%{VVajpS+@%vVg%LtJO-ufdm4_gjb3Y7=7V5!!D^+^||!*Re9!j@JOZ
z>p@YP1F{?VjWS1G8<#>f8M{`G_p#}DpqOsaE#8y^dNbVKz5BSxXu|T1k<ej#Z_*8=
z&`0n?=!jS%+05jU>y5lBAMWE@9Q{OEYlC`R&{)Khwaz#X&=r^aAi0%3;;lW-sZEJp
zXxEk@Wqz8Ax&MGNAn_Yu>5j?Dp&k@H2Q*A11X|OmModx|VrOFRggdQ!Ra-pWgdY=+
zF$~`4W-}#xM-}h(ilS~qL0o(KzPUO<x~~$k-S=zGp2l|<b~f2%>0XKE41}|)t&<~D
z-}Rt>3dXHYAj4x$4h#2T60+`xgjgtHKZHNB80&KcstZ(ku@ixlpn8>#*h2g4swJC#
z2#A@JGL>l0z$G=rWb4O<TKnW49w@#w+dk?jaq=h80G8536Hd}_el)bULDyv4QN)c0
zl48ejGS>L*J4+`2gxtvtR~!-fl^{?LV7a>Lo2v|hk_XFjn6Cr^Ka65N`KPizrY?uN
z4Rh82$T_6Vcw$@q2gL=rz)*W1Z*N~Mhwj<+0B~Z~Dw9C@9YCE$z*N~tO_6J7TiUfM
zGjXx8Nv@{y*TedT&^e>-fmvULqX#++bz1$Wiu3j%?tYS1_6fvtok^PZ?&#iyi$)x#
z`6_2JpGrdCDHg`V@jIS@Q3q1KDalzsF!0#hFmxnF^VA^tYaw*Wa)tnf4OTdaL#_5A
zIPSrN<gs0lU7yx&0nEZJiC?d2m_A8~colHx;o?jzCmLkngt&9o-fwKnj@klFLQvkh
z{Kl)<{wk|t^*TnA@deZ5fP+|H@u1Lq9479aI{FSMQr8v^ly!UvSLCw`M+ZIwNa<0j
zGr*i>kAKTv*mY{IZ8PMS_%Ga7zt?O!S$I_vx~*l~CB^c$Fg0{Ce=-dIxP_^U&3M2r
zG&mOd4Ey-!S-S`1jRo<wz%t){1y1pBSg@l^r{7q3|6Yx?^Otu3YNq|B6OgI5#zRmH
z%4RzZcZhYM#HLXS`mu$v>9D~WvxzSxpmNz7w;8wz67ec@k>?7wx{aNz8@UZ}x(#`H
z8!h#7Y4@{bz}!v*$*J6r^6mYHk7b_6t5omMlSvWX74j(m37|5lt=uk1DM`1--@}gt
zLKaEhal5kQzg`89J9C=b4a#7XWPW{4okZvJmTl2V$9J>i3u;$)z!gw7z;zc=*ha2>
ze|HYR6)k;2xe4?g$Uw!w?>Grm1`l-%Ycx5gcDGfhukE6d9$}L%)X0sZ1PwYZ;5Iip
zz2>aax<|^vK-cxdsra4l3`jK_NJ^2yN{qpI$92K8o&U1pPZ!lqrdbpz``(@j_RkR-
zO5-(ErLD=XYPh*P?G)%Ds@~Er5Q``Ktm>dwim0<K!%x)yN^llHFn^|=6uxzT8=|kJ
z$iCy19o=h7Rwnx5A0$_wL+HsHFf$x@He|!t6QfCX$C@y+dNNM!q0uIjsMNE4t|iQ$
z#was8@F>I6pvg>&4Kg-w4(mw%o}C3>{8r~^m(Y^h`Kkjj?xjM7NZeRdd-sN3zMz4H
zPLcFhRU=e)dZb+KAN6UZ?t5M?H{9o@&-tF197&@sdlfe+@NYj3RyhF1K_F7v9zADM
zeR-W@G20`$AH}`C-P0%Ubaw654dR>g3lDy@Pit2{g$laBqD+#<bZ%}oz~n4m-(4K$
zUR1vgmikld-hTrEhE2JtSV!)FO49m)dEO(0^I7Ziq>uE>N746xcY3LWx%S|-Zxzb>
zOTTCQi>Sv(Z4;$p#}btHAs*&cYAksjSH|G@H<%J393T897o&hSr7oGA51$A|+0Sqg
z_EaypCB-1oFfjhJl*~4XEo~;ogQ`;7j(uJ$_2Li2`3F^J0I)3m6|NsRJVm<^Nme^U
zZ=dZ$xK00YiQP46q_C<3&WT%gl0e#|TAI9+TD6YCKG@?Var*G7c)d0XOkVKK2F@OD
z`~H4DY;3x2;}|+GRXV@7cJT7%w0Vxkjy|1=O}PkjVi{2W7Pz+xsc(AYq2!+IiK1AO
z>#sw$sa0;ykjd4?t9oGcz<rBX18LEjgiQfs5eI&l2LMA1$o5SW^z`S94wM(&Dwt1=
z$$zAs7IoNWRmpz4L!nE+7EbB}vTiRP_$j@2-j4%m-#WMWCiXgeovmGFEf*(->)Yu?
zER}VajQcitceY8C_!=}N?~H=wc^s=mv0dsL&jNJiHHVe7!NcFScGj@!_g>KZx)dQ~
zUM7LU`)}<BDyu#!bVj6-i&E-D-Avm@hPiI58seH8h5r7&I2^9T)>QjRht$~{sZ_W|
z5_X_gJ<wA{9*uZ4@a1GF;024Co#J(lK3DKNR=JR+UDHtL4WGHT7=O!}e_Jm}UwGbM
z18nOL*EcmySwE8Jb9a6Cd?h=lR^5;7<Ue59fN-m1Q=-GwxruYH662;2KXe43E+vQp
z6~Q5JRau>h`8EIx7^=%zOi*@+QvJ!WrE&!-8MDJIS?^h&@=rp~xh}c+831a&$FQ_R
zZ_4XCNBfVci9K>IrNL?QvZ9%u`M=t>XHw9|5fSMnXMajb+v1}oP678MM5sT}7En3=
z(cvb^206&X5KzPP$CrTZ*t{BM81kn{EbPDMzKs0AW#HNHRC4cGVq1EQ5-v4`r^3$~
zm<-w2@Wklorp<@tQVtkmp8Qj!mhl;IBpk%0uaE3Nh5-igddX#~BT>*&G&=EA74U6-
z5*f1s2V4a*iZhDj8|K8WZ_jMYck8AZ&5DM>AMXRT6MqlYuW*5Uw^#T>kCyLGe3m$R
z?_+ZP&baai2FNdK75)hgXm{6OJUNF`zySXAxVR)&Q!A-_F+jG0Qfm5}*5H$K9*>Tn
z6_*A?=Wl?-Sn*DjRn|#G|0RRIsxAQfS5b+}A^Ow*P24^xIsV@0zrhC7AN=!l0RHju
zrIh|ZF%>`vAU-?&mprnxDCV1gVvG|7E(f?Ez@9xY%%%MkBLFS8-AP>fmXzL}|K{f-
zPw$RBto-NEU;XR9d1%9b!<|FB{%N-X@aNwDPOu7V{ZF!J1(SSF0AMsATK^9)&|%{W
zR47JT!qz|jXD#1@e^(hD{sCS6M`<6hrWHt1JsSYr0EBGi>yJu5VDo=YJU&J+x@iBG
z(x#=+fV9tFAAk9m{>ymZe^B;+_~EOb^W#_VU9QC>F4OSVaG9cz1I4WiK%^-CbT>P3
z8*g%@EI)81)$*|ai~}ZZMjE>yej;<0c&?U~MM3Ud9^%{0AK;?LCMPhimX2dYjn57>
z`fl8~Q4D;DCuqP#JtRr5Raox(r)syvLn*avAi^@-Zc|z?BmpT{gLfk=C3k)2=H}@A
zwU5No&tgIfXlZU@yTQTgz`gUO&qenYjP$9L?w(mAP(xjRHRnheHmKig!BR9SO}7^=
zx-7^TulXF!U%Qw1TLbd)Z1+?CnIXJF0Zl_(dU=#@j>|ZxN^PO86T-`XHaTpXXcjEr
z>-mf9FAc~mvo_C?$CS8Ijj2OQ?8N1Lcz<&Rg~d@`l2|>p)1suxExlI1$ewOMUYqT{
z^?iA_UzRVk%Ke?wR&_AyleC_jt`I-!y!-tw6HLXdxHOdza({WmdoJofaNUwtz*@<u
z*-3T_c|)W7cAwb3hX`*#UcdWYi49)9KmPuv#&Xx_f|K~cE6g}Gd(aY_!f8#$+R|(O
z+{0Bh!L-NFwBibvSNImRyr1Aq|7-Xt!^H!9HBkisp?{0xKi^Bd(=xT(?V{>3JGIUi
zjKzIw8Gn*AE6Ah;Z|gnw%iBK&wN<(9jV)781XHz`t;@@kmHhbtJ?=)$ZmwGD$nbED
zkw|dahqUSC!x-AT!4g$&g!jRf%U-K%yadE$gHe0Ras^FIrVDK!hemNqC$<BmG$3te
z!EyW!xs|eD+2Fs-z>jNbP(0`Lf-^Tl#$&k?3;*SzTX6*s1pAhSR9$w3^Px_KYGBma
z+*??BO_=9YPuZxm(urq#iS7+Z+gY19DX6Uf^6F)6SEf33mqJgAGTgu<iZDKDn_hm7
zp%o8mx2#krpIFHvbgyeTWA8P&=thBicpOPG2+}IQwC>`q<plsP3=zGW&t4n149rJ7
zQAAl;S*bI`dWx6UU$Q#n`|p~|`?=WxsEBKCZ;d#0?~n<md$pqM3Vvk(Po6Y1c#DwF
zZ}uDxT>NeSjVgrzTakg_&cKkGy-+0(Mq(6Bk4Mze>Jw-seSYMiPSS@mFuZfDqg$e(
zvyXTwD5{f`R=<%W5YE-%%kOIHKPOi0S*7z&;NI0zKJg3%2)4m=(?#9xMgFyT-olxM
zEI-2teWakc1*4FguJ+7l(z*E?xV4^bV~&SpAaRWmq?|u&=-<XZ<*%#AeQkf~PU37b
z4&qXR2+>PriJ0vC#*#J}n@lpPy?I1cFozC!HC^JqQV@R6cSZ0wYXwv2b3Wg9MNfq|
zH{YmMsNR7VTHn~HhVIZ|7nvhNVQNCl5wIIjCvsK#6rb(v592A!6;zFPiMmm=W*d4s
z?M&p=OI2V6u1cjH4^v==g|Xu?6F_CaIm+DK0d|>*fo#n6lin1iMNTXyfYKS<SET;J
zY+G`|4L9AC)w1;t;JWWJ8-O(Pr-Wb4Y&Y_b&|Ff`O4QQo{a8`?pdH7JA&mIe4#$HE
z213mnb%ScpjDPb(k(nDTv@P>E(@26%vYKOl^a~QGYLT~E8~<K!IseqXE?pY>x|&^)
z-gu!>_+fu*eRkjHyozW^DC-)sy3qVZeSoN~c^Wf|?OD`kbWa!FlSNU0a>QjZ{kcoB
zK<3M-J%ILYG31n0<>uluigS1Teb{r;nDT?R%sCr+Q~E@h$rYzOnS;e+f&GP^(*rhi
zFClC?N}&YKm7=#Kc1$MDWd*0+7{yTNvC|_XNStP8d;R`zh8;t$srV40;MnX!&(4$_
zw0&yNd9{Q~Jeq1;-EG)Z5r8>0l#y!;sC9mL4vLJ-#kUUMb^VQI9z`s)(B3HyDw2Wv
zyJ$Q<(jhYT7p2t<$$E8^3w`VJ!VKaJ?H6rK3<vs~2_xEB+=@HTf{|+I7rZ`~8x_I;
z|1o~i)znuG?j%rS@GuZUx`-GC==Ay2s-kL+$Ld{yw>wtU8Qc@-Vfd+K%JZ<i8~CV|
zq&fC7v9PZwQL5RnLQcWeaHLRuv0;R6T5va|4;`Svr5(5Q2!)X$oRmUsl5O!hTOM|z
zz8P7G2~-YN<t!}K)D&>I0r8E7jq^6G9BR%K0ILuiFY+qq$-O-)=4*F8M08s1JbIhk
zXVbxoou=^H{8FA`8ADx&ESUggtMBdV)WM1-oOB>B*`4^b`mEl~!ud27wo{m*2XSpC
zf$Cj3=#J7nyII{itm_!u5`e&qq+!x=U_WCKMomNy?4~ijw0tOZpGoAQpuA{~Vab<9
zldfZbU;0ZN#0xy0hy4;rbKE$#j}bv8Rnt`5KkgS@<u&-U2YCps8@;b#ddW2tW8SeB
z==|&Q!g-!{V&bR$*v|49S_K?GHDwdwfrGpNFQ(Eqq;U)P$rE>goXH}FGbJU+4C5Zd
zoNKC`ptqLDkUH8sYdRP~K_kxbCW@`IXa##otnWeXUoHhBff|sikt66GLf^S+b5gEb
zFHEG*I4Jo(TLi7xVRs%c{Q}|jDS3>Z?)NYXX*IE^EyB6hV+{O;e-G-bRIvMe;en69
z&s#4juZWjBEW*SVx~fQ>ti-;WyME}QmIvh(JBh_mbZ4TS?~G=)@63L-dY8#%#RzU*
z3tW;6^j>&*tKBC|YBUHF=kd0Wug`EPLxy!4cq>=h!A#yaPg{mXus)`+3U+o1J*$S;
ze@a6c<rE#wnhByB2>&&yVeJl&uvQB$?>P?Am-+~qBKoG-Ch<<{{2s&gL55~bwaStk
zmTld6c2WE|e)Rp&ujRx-PT_@lK&6#6vl*qGxgVn{isy+6W*O#>K#KDAJG;i%%nWYT
zZ$xGm-BqSd#&qp(yE*kh4X%+;a%VpA86!q3x#MQePbClFzrb~o(g<}c5@_Z^dr*N?
zdt}@WD?OnJqQ68<RDL&E3zKEYAXJ*ssOM=zZ6i}Y`|dAridmJhBQ}bucMO>>6JHtv
ziG{<SXW*Z9!}RAY%epmPtwn{@gi{ZV_9T20_6|yfJdBe+2O)Q43rr6tAA@y!)wZ;K
zp;y%d#cpfQcaiI2tY=cWMXM_7r$<op5-6YH<<aI6Gj6!Q@35I%i#GB)_&Cn6Z2OTn
zye#+L=Oj=n&UnO7nrSahw`NV;nDk4i5CMA9jMm+T;{!L_+sms92TeDObsx0LaH8^D
zI^GE(Y9mjs57C-u(hHM)gx$Fi&8jOOdESt<yEnW@&0&jdE2tm(o2PF#&Rv5t2&{$e
z6AS2^h`$Sf3*}YW{q9BkKm3Z;*!Uf;WOet<l>7JQ2@ETqp3DK4jK}$X+7lu+zag;;
zz#eq$f}+OKm13Bz+5R?!;ao!wi+2En$dW~kPT$DWVN3_OUGO%X>P>*Wh`UreA3!N-
z+G|?VA3-Scq@9q2nyMuqbST4ST;oAhMO3{pexId0%IZ#Y_u9i9^{Y-o(W<H!2kO61
zYWaT&{8Loq{(j#z|6U6d(ssmfZjtThU9Drjo$shO3VZ720=1^*3I<d-Hzvzt42Kt+
z2@|Jh0Nzp0yT(^wFCb7>bfmJwB#r8zQ%dol`rNxgVq;vgRr_L`uQQ$%pL|61^1EVe
zhaQmwD9tVf8shKfA_>SZ)Kr5+#|0f5$JR2M$t3$B4}0o{JZJ@fEo)MjaWM$nXxH_!
z-LJT9=Wt{<+|fW-bwfIj=GN0Qi3@garct4Q(j3T~DO~T=Rs!`Kvu>R0d?!T2wS4qc
z_u3c!P1LJ`Mj|jY`j+kWg~weDEIxZ_0l6`a#~Gx3ox(4H#gQ1zm3v};5iCjQpxXYK
zMTxNHZcUTKb|$%2`y-RvTr@r6p7`l-<y3*FNYI9JyF0*X4f8S5@d^*Y-sK1CP<ui>
zs*pEvkhX|1;I%3SWreOwnIgZs2V@W2q&sH>cYOmhC!pbh6XcH&u}5$oG_|<5x`UZr
z_KF7ow6}fiuB(lxYjUjp4<ks!TpO=KP;m!CE8@%)^-Lz@-7-3ridA5ro{FLZ&QKPe
zJks7-=6hIPKRGD4eLtN{y94X~9JuPG{xnFh7;t=0A74cuJ#@|cixTyVJMBB)>5fq)
z>TxG|MEfL6sv7}A@-62YZpm4vl+>&h_^|D}@#-bh*B3g+COQ1!K>vf-gpmF^r1!2}
zp|P4K%)MNd<~8Bg0I<H!GB3|H#1BX{nm<?Yo|_{uwW;q(Z|JRE9G$qSlO<ThVQJ_f
zIo^{i-5G~7k5t1E%iq1Lb7!hzGs0@Y^wysA<5K~J#XH8flKIv1k{exlErzZsn_v0Y
z;k3#)^NYN)?#@zSccGN9hUO{M+~*7E<-oC{DylTUqX#?<1&BX7Tw3DuFOfj=^hj2&
zr(llohU;hL-EZx`=rVw|%@uf4E2|{_rqKxbBv6qdl4yFOgBSOFEvQRglzq#nG$5&G
znBPF#uI+2t_0oRHc4OaPF5~#<_P%!)sV&HmKJe8~&)OA4bX!wJRWIQ99K|6!9o;@e
zW2*1V*k^c%O9pZnx6@@v!~~ldt<Q-w90lGAwGWGneC-rb@Pc+}unIltXOcK~j!HY*
zQPd?mkTMTfrBMxvI`8-ZTe%(E6w{+uB6%g-T_CD$b<H7XRSD8ZdN;(ilPg{j6Z6pV
zyuO%$y)@u{gKcgVlX6{Ak4gaUM+dcZ2tji+8+bEm$?A??SL|4I<xl)P_+UfE#nCRn
zD-|5puCX5yHBLnuN=3Dx6^09xLSNe`*$P?C6Ii*vugz=GJdL~?BK^vz{5VeGm>DAn
z7_@(Adn9%T2orqRDTs6~E)T5$60+NehgmLKIk8eKbkV?Zy!Mil;~v!XxqK(;ZDwj!
ze|yhFZ~69NL$j7Yr?30Z2M%O(Xe#<>Grs6#y>Is1XK<!%#dW*Za~2t7UQ{qrhrfA^
zmppVNeRIVxzkM{II-_ThH>D<M{(VJ+k+u=KS8;=VkZ`EBE1zbgH&8b&^w0}NQfZ!p
zGmX?waC~_e%|;N^BxMjvMuus|%_9CBg{)TZeFHWFxDM%|?AAmO&-PV@1pZw+;vl2x
zd1(KtNw2L^CZc&!Pq6UuX2RXSNzhHo{S@wGvw9p~!yfQ1f=1fYyJJnX`AQXaH94Ns
zSEh6c_y6gkzUzQk#RyOCM=R76B_5|H8Rcja?H}t)YvKN8MI@nLY*IsEFsx=$gSpOE
zA-OTNPtshZ<yS0m;Ql}K79-XCP21xj_u%lAxaOtxN(A}~3G|)-M14O#KNa8aaEY5%
ze{<`UU<^;-HFCoQ5|Qs4T(}%V1KBZd&Z_2AU-%i=rl~~(?*?JD8q*onLb_;i!lYHo
z=-#38OR+x?u^s^c2aL#p7_mI_Q}Rk&GYSMmB0V;5HRk@Csb~NS5@0bvx;+$lMPAtc
zzW7KF*?(QRG@JOGU%Z!iw*h%Wi?NA!Vfh;DP4yL1cc1~c6b}zZ4FYftog@vDw6L(C
zi)P19`Lu6(c|L|#@B>0H!3@OE$_KT{E6jU;b0zrwfe8?OQc_c;H8nNqO$nNGiLJ~t
z+Ms^}8cU1%2Tr_p`>s^>9PM?1$Dv_iN%{dt`7?Qjk3AIrb>-wN`1ucv<40VM__6}a
z>|EjDYcqu30&r)_fKRK#%JiQ8Mb@bSX*mlX&R)4J@ol+A&LeF*d1?tfm>y09N>rGQ
zc{o)yfmV6KI`s!JwCur>!(1s7Ok>iwmGJ{t#((D|t`7DGg8p?xk$_2Jv3wR3MFZGS
z5?&e)6Ak50V=-I@AIHr%H|-%_YCv9{O?<{j?frrG0Nj=W0><UNd%Jcy;<@z+*cywf
z{i5xxu#GY`Obn-D@y_@C`}WnSVY!Xdy&`fJT_mhQ)`+~EwD_rZ0wW*N=9|iUh$kD6
z!2UhuYwi6n^CYZRQeKH#_CE`@Nhln#tCV#{ASOg%tW1kHHF;RU#B|L@wNTNby~uTe
zOU5ds1PmvXi}<M`&G)!Z7{3_W%fXTZ+zaP1i7~XdgC&Q!&F3){l6uS3Us(Y_j@}ZN
z4tE0Xx~H4WyY}rDrTY{_FR;6j_}KmCIFXPhf}Y5Xp#ezKL9Y9GOj-=>&q3|O+{w?F
z`ky1M61g`qq_$AMiA?U)-V!Tv(W71g@;TC^GVc+w^Z$wUZ6^=kT;28;%iALLJO90r
z7V8OLLsdoPtEh3z$x$OZ>4?Hj<y%D9Dx+>0f!3~u+$$&N<tHskP3jp3Exy>r3_kMm
z4CfF#ak*lC6%_)Xu8sJ38j}Vvy%rPeK)kWSffc$BJ#atvFSpke<ksc(Z-^f{az)t0
z{`bnb*#BOW7yJMBYJk%F$Ncmj6V-RrnKcc1q)N9NyOva`Z_daYRA_YHV>5u8$>e@=
zbnqfW^&rD({~vpA!qsH9bqjlr?-3OUmK{B@DZwfmKw1e9ElaUMu*y;pX^beSGywt$
z5Fjc|v=C5H=|M$7rAZL!gb*bNB7}ee651F81WZUkAOS*>Zv$G*xt#I7-}t^U?zrRn
z0|?Kv_j=Y|d#*X>N?1rtm#=O!LVO-riV)hPqijljR-6_b@m5X<!(oC0<c`U+8$N6K
z!WLlAWv#PABH~4Q4l#%|+%p0+!EbYj!$&G9KMRU_>~bhFpAwV8g7DrXbktY{lk<B3
ztsBS!ZJ=KoDo5<(Vc2qcVNc(0H%Pe;zIFqS&Q1ur66y~!LyR$!HW|@mdhfI#SsoxS
zJpqo93W<c{l3(`auwC-f1X_oCvE_!b_z@77mH`>b>^Th6rOe857HAFxWn$yCUq0x}
zVZRT&hJ<&U*)2`)N5+b+{ICtmJ8s-~B){6Jh1PKy4p{&7lpA_q)QdocLoXdy3ps;H
zemiogG&u|6ZlBPJ)%fR^SEmItZ_SqXl$oVB_v%=6FD*;yX{iwd*|IoCE(ZRbwPE}*
z^{;hKpyx-M&2iz6kBadmWF3^^w}4BS`OPco%qr_qW%V@Cs}WtZ7o*JJWL$_R>Y4Aj
zz22q-+Sl}cU$X-IJ&Y1OVW_0Jkm(=8s?EGfK$UyDLL5Jn(D)WQ;9H4Am1oN$Q8vtU
zEMF8!A9MfpmkL}(7-oBu9^=SNM8JWB{?Y+m-b+n%GS1VIk2{MEE*F&*PZxH%+f+MB
zV4(}b-rkGX2C4goOHxEJV-tpSeM@zn&JOcBZL#kP6*zi>ZK)+rH$9?oELlI^Saz>0
zg4f`8;nL$q|2F586>H$EUQ1a{r73<#x6TTnlC@~6)mdnyy!x>TUnI<t98S}LC|F;!
z%E3-bbF(aeWeIIX(VwueP8r>Kc6Y;oVV&4a6j?g^OTfMBbvi>7R<9-`Bl$W!*H`xj
z{4W1=yOyoK6mGRXbGD<SPBvUhLOh?+!rHqmtgA!mbgMR%9sRQu`uyl-cC6c}mpe2s
zUzj!D-Ya`_#Hq()(>_HM2T`B&zFYQa^1W`OHQ^1=L}pANpCndZlJ83U(V8$gkTL9D
z8Y4ZPOSys!G;byWDYmdp2R9JNluz>8Z1w}`Oj441aI(ePQru%Y$}IX8fv?2LkUd^#
z!0d6z5hHyBWN%q)k%{jNEviU}T(P-=l>RV7zYLDl?bJLPvkB~P&sdz4J#kmWE#+Ug
zGS#t8>4NHb?>@=4T_?(X>9J5A6KnMy4B;<HGL@c4dsrf6WQX;OoB24A{_h2n_dk@Z
z-=++EBxVKSg8Q#8Ma<R)Ca0u4-K|}h75XUT>5U;2iULUp^FocNou&w~?4#%<T>C6F
zMXcOqq;z6>GtPg5*0ZeZKo&Hqhp*`m95Kn!qdd^laSe^rZ<jqbbPJBFg(=*Voxf96
zhe|}eoYpBblRq~5JlRF=`JFar`3ziKu=9oh!_~IW<#T!^`Pe}7lVfc&P;y_9k0(>^
z>@MHh*_{&Xi(^)d#FrxQbP3d1BfcC+Y%6g2MOns1+PZGmtI&*0dYFBy)dS#0(Fe3c
zm*m2?_eMv>nx0tZn5TWrb$*d3^W>Kn#2IxJ*lpYgftOT&N-rYSdqYB0MsvSk8qfH}
z&2+o1Qq{V*k?cJI%c2W2%L3&O!hu98@SlYu{d-nov&_niM<kDT0qi}F(Z@oI@aOjE
z!M`(Gt<%G@zF9P(Q9|aF{!pFbH*|W~P|(|1<$RzoY$V)xneKs_c-5EXT3;k^8Ar8O
zCiE}U`Q(|2^TViNJqN;*>5}J0`)))@a2DSUZccdGC_Fk4qjI3S+RXL<?TG<YSI#<8
zAJKotf63-DJ<;PzO0Tx)F5jvkbD^@j@_De2xMdpuNy4e<=6GU1pkFk#)F_nT*v9gI
z`Q0~YA-*S4Rz4?JrfTCA#X!?70K!#X_)~t<=Aew7C1cGXU4}PxNTZGJ%&8O$S!SzC
z6VNkBhXh6unEY|F@j{LtYQ9Tnzry^U^iXZZX&Jx4K(@Dk?D}*OF`JWl!gCG$kc=bf
z4{3Nz=OkQAUS3(e@kVKQZcjvOz`m7W1AsPguKBScW%txUDfCH4<B8h2<aJ^CL3F<`
zdrUv=<M-VbF+eYmPvKx;Dw}}(_$=eZvqt4z-kobIn+}|{<l91zlVg4rRMWAg#=g+b
z&QR^gSFb}KuL*YRA_cD8v{3nUc!qAboc4@<fWNf^A$k3_n$`$Qlbp~N$9LIA^|~ES
zkQD4g=*}c>p^~zXs>Cd%PfG%@XB+w3I<ZbxUs9w+s#GoCmAB`=%J6q%)(0frPy^D!
z)Lij!z+&23TEnm<pJ>8IR)!t8R8eed9HfChXt()JpY`WA#K)p)i<#_+Vx*JI&354l
z^ZRCD?g()y?7cMkXURZg(z5S>+<Cf#i!nxRwY;rUJdQI@Lj~K6PoCBv&+1;o3qH95
zYwv|vXeQf^+lcYK@Xm}bol+U00d6bC+oYC_Kw#B<=8qc(k^Hw-XpH%U$87R%r&RYT
zPV>}F?4K6DANQ!KU2~eEJqhKV#Ppn&Y?QxKP1|Rt2@JomLYHy<1-qEb5h)SAHq4%$
zq>H_qHE=*HX5uokWgyUUJlNOmsh{rHko!vy;exQJ=hd7VI}w+>^7uiZIF<dQd$mXd
z+~-`BU~TDrWhPQK3C1^1clNw6%{@9Bh?<$YMI3-(Da`A`21bdM)~x~H3<Xzx2|8-2
z{*CCObw>+>LpOifMBkZVg*17Qx6E4!c)}3HZGho9M`c*bVG8^(BhZ(XIE!1A(m*EG
z?XwC8BHto8HUvPr>B<Y%g1#>D!NGeWeRCSz%oQJj1~=F9>ej76%#E5uPyMaFpZ^z^
zU*DACyP`=Mj~g8=)gZR2ne<k0IO}-z+f0*M%Fnet9MgCseG0l5;5&<6_|Ep;k}_t1
zXjT3u94$?aXQZ=5??;IHsA*V=w1IQUR6ewaDVc=Oj^HocmY}todo#tKn^)_*{~}TC
zknwHTiD8i|=KuMt4MPU}n=FUWDXvigZFp;$Lg~jPH9!FGn_G`I<DU^;1ZLIqqSF{Y
zv<o;@{3Z}($3bH|K>Z<c0;!BT4Du1~iBO&hp#i$?3#IqQ)Tn(djYA2?n}G(`FPqb;
zx65Hu*}{k0@_5)lG~t-+ilCj32*l#$j_6H%*{?6T<BzowVTF?+8BJPY=Z1qN_jbm%
z>Xsn`<;UOD2mSpyCTtMrJR1Xlo$yxasXJ-LKPIoNFbfaMnX2ORH(c+(t!h*uDYV7!
zW3g5rIB?*wRq?)Ee50{!>M-v{6wp(z&EbxV8Q_>tq>VsM3!UCfW-aXW)`Q|~!ArR1
z=h`a<h9-0iB?C}=!b_~Y59udXUp$9Z+oojPudoq##vEkg^8I4)pw2ui7mfC}_V_fM
z?;>YP70NMD1@Mc|^5os6W|m<N=pLr}hL>6(XkGwa{hTy%*_+uW+vJoVluVkol(k2k
zv0bZty&VE;1If;mfqMdjBZ0;>A>=O_SNZKOEwljle4JPl1H!rKXbg#pd%fjAjVk$l
zaMo!-H_)N>3EaN$bi9(f6Py~}N)h5BChm?ER99?SZajLk+zaLI#@*)T-B&$6P%{q1
zj?ILAapXNwfvM*{$aO3xsz_xet$+tawsn2<XA5XY<m;js1pZ#b3p-lcoGP9wy<m1@
zwl$L7kZshcvTd%*^F<4~>cB4}ZhKS&N(%89?1<jNa-Q{w7pTC<w^*b4;NJ&N6)i>V
z!qtslOW{Z;B6!8-fE#ty&5qv4;dg;**pKYTMmGIGpKqzQn?tz`h~;ga*iV2gGv_X*
zrr|J1nYcb?AgT#YPS@!_k_|F*3h)YwF6zQ&EYHp8U4Uus>)$S>nTZnWYNoCai;P+^
z?!Lon4d)F#eowk_?cIn1$x=NhQF?!f6Z*Q34$R}`!g?Vrm&{Y8fZ}k0r?O;$8}<J)
z18HGdmdsP9{?Gl-f8%YMpVyR2=Y&qjMb&31ig|$LZ$HPA|C0&&==BfqJ1zQi#M$%0
zZKhR3rx@*--~Nq0nf`g-3hC-^Kl7i~rgu(nkX@#)H^&|3hrfwYJqd)#S^uV}jXWN)
z6|tq^{ytqaPVc{8@Wuuei+tc5_TT>IgG46yP0`fDIZe)U@j6Y{IS0k?AODR5YB&8L
zDw%nC-!J@-kpKR_?=N~CV|SKDs|5`EJ0t>gv<7InhK~OmSJ&^I%9e>Eb3)-x$zRxW
zf7R<Q%bh*;6ZzZEd}S8?eQjKQXI0PF$Ins5ld?J&iy3`0S?qK7daw;`jWq8?zTc-w
zn)3afMeaa>_i4b>l;4v3Z*1WYZH;mAhkWMTrcZu0(p+R4KMmC3egVQB5I~{RDax>k
zr~gJ2HlGaOkhj9nb6ZWCZCWfxgjS5X-O+~4g{y0X$6^6U>Y37i7q2~+NA%&v3q(sG
zxLr(?dr@uCN4j|CWspfa4z;Q3^Q93L;$VN%+0gpYdrC0NnW1cixA%6^{y$RN`w`bg
z*KPn~<(y%)M%UF`4Wi?`YwQXF51Jce#aYtr=@vmhl5GC?g%iS=nrVYcma7sWs8Sy-
zJ!}r2JP<tarZv#sXHKpFbO!=w$Ev&5ImgS^Q+g^B5g*?=qTs&#x7*lpPsiEsCMOKN
ze5{0d9UpCk(GBh##P)DE1vfPnf>zc_Ze@sj<V;3>@uDGcm{-^CHx~JH)M*@_3rsva
zfQIhMQiYnDnxAHt|DhN-@$58%QZwCQ_1stV1`q&O*w}Q7+fw6gV><JOBV%1MloM?>
z(Hd^(D0{B=%~5f6;f1=fFC|+U>fl^^FPP6X|1>nwylfISynoTHx!d}2+N|aR-)3|D
ze-!O5tM*c4&$fB7-@F~1{atd_Oh4u+C9XbR7W!k83)Pg3jNvJA@0pGyWGIf<I4N=C
zAB&+WO5F88?V{l8nxS}{Y=_?ov5$_;>+N=9NS=|3RdhAPP%V{2==LaHCmrdmj6y<>
zir1*%0hXw#*(Q8svuG^-cHjuyIRoS<Oa9HyN(|B*EApNhVqg4NpH`DYuwLUqm-i-^
zel+<iCU{Q_G_)NMy{?P5ixGR|QqCnw&N66ZAO{?3Fu-gk8M10p3|!$IK(rftb+juS
z&l77a)1kR#1C^$ly7YhCFl#0Vx|G3+A~vQU!mo?BtG6T1DX((&Db&*#G6yfPyA#hA
z-Q9Oj&NJE3*Zl^wV-Xf<nTLVKHH5j*$ihOyx96JlcUJt-D3`Put82vX2phj{c1&(8
z;?l&M(XO@%zcD)hw|^X2x~fqayCqE)S088RVvF2X8klpHB77m>>sO)5i{u?_{>nqD
zx$hrbZRjk0aJ9KW7`{VU%Qgy?l~cAZE$c8%V4&`Cx{rB3LXVWUSPseA<&?f#y)PJL
zk?!VbYri59cET1=(#1fYGG|+vZtlM+d~hTkq44z)e|zowSdJ_Po__0Veha;Kr6Hxc
zTcC){O&cd<(R@BEXyUp}y>EKWiFaw*8P7;gWd+>u0oU=-lx9Ljuyxfv%fSy9l3(8G
zJBd6-PV8%6EORX=5kbMe%N{NbEcavo6dKi?H}iPQ?Tu0EyK9V~IVuNdMpKAB6%x9K
zig%-x$8ly*zCYH1a4X2$K`Qf6Sh@K-ZC{$J5w>9P*#5o*z4ww!K^7731OAZxJzCi|
zK%`7|4i4<hnb-8v@@1a_9bay(egAmVz3{l{r*$Y5pquNwu6Od1i!e{r{L;MgCmGCQ
zTGBHgqxeN*bTJVx+AF1a=g-TQXiyXjC@bT-(e3ds+G-QV<(1t=uv#cXEBRVqCduw|
zPlvcgCWDa^nA|@T)U(C~!R~~=iw_@~{!Dv!UL)gzFJB*Z==xa@nk5+1_F&&KAeR;5
zz<TO*{O9!q%FbOlNf}PFFI?O^aiQ!;#<C^vyT?5bJaGsPLl5Fh>j!w2=Z@uv`#l`%
z@o_ua{P8CAz|e&3Gd0Gux@+d;*n=1W>knn0MjN;xAkFxz3Kos5d37Fex;mGM6;$|*
zbd^jzIiX+O$}0c)XTGh%@RGfzH*YG#w!^j(e@9%M@K&N8Io_6#Rh1Apv=Me>-^|M+
zNyM%9NVFd8(cfJP&%Sa08i9{qgEpBIO@Xv7dKzimP=eL&0(w>JYb{lQKihOJ*&FfX
zzQNBe-s>okXrujY_?@3<N8I#qScF>kIJP<Khlf&cl{ohS=pPEPXP}QbGZLfy@IP<x
zz^?U0_cYPh09);3=#BJ_2Y7H@dZDzrcA1vOz7>;VhAFW{))hx<wRqcNv9)#P4`u$-
zXF#IE0@#Gpq6gQ!yx0pk<P^Ev1Wykn(X)Ge-za?doPDL;rRzb^5MsCjC`sQcZ`Ru8
z?D{a|dTia;y%>%6SEAo`ssmDYXXoZmG^DHUX3`(J<`XY=n)MQBF%9$aW1p_*K=1y^
zak(c1Gp0d}la@IxTYb-GKD_&O_mq=Bu$Aq6WVS{56XreTKQAz_Ql|j_CGDNgZ{=Mq
z-~TUiR%JgI2rMevz@#g<#|w&%=&9+>jtehs==8H9?dXhz$wN66*+~6IWhG@(Y_B-3
zj`eYDiSKEe<Q-{n=r=CI*%In{L<cqMXHNfof3MUo)Zss`?B7+meL*kJ?dOO-Zuvo@
z%uGT3_UovcJwNlkSB^1;jBniRbzNhHmY^&kjz1^3<we}P=7TsrV)%xp_caioIf9Rl
zLE)d{_-E%1)=(!pItpWk9W)a{&RgJQQN+My(b@DJo+lkYc_d?t$9f`>Xp4yaNfLqR
zr>kY>I{u;6K<XcQ`av2Q-WlwsU>{R)PgM<^eVG3ocI_iX%Dnvp=fF%pVyl_+F|C6K
z^|f!686Rues%d=T_skzQiZlQv9t6ejk;|!!-_!liUR?EXV@Ouf>(~RGUS%pFUv-R+
ze(V5f)pBqi-V!`GI(KB44W2e%Q^yml(k>qZ1_FNOwp5$a5dsy&&h{0FM$;(${GOM6
z4vzmW-JM(X6aRV9L1%&ATWK~YT?l{i^=Q8R>^L}N7PPg&{2@Z$%3_Bk(;<SsE#A!a
zpb#q!y8!<+pfe`$!iQ+Hnu(gp!85O^_MVi}<ikSPu666XL{el-EP1zvYnZ>KZG_rx
zg&Oc^9H7N*@PcY7BGX-#Gt!XWhE3IzheKJx$uHYPb%$r|Omh}JtmjqFU_{sK2L<?Y
zn*^Ps&_C=59(^kL9f5Z;gJ(hrk3BW7S$sa3TzSvBoLN#adeTB|mC1|~F!OkxhjTZz
z^+*U|+g^a}-d7MrB7i#K?~>a)pEJCVEn0-ltNY)?XhXEjSFK-vR{rCyugmmnkW(*l
zABpEsGtIJ%xxU$bIJ0ZcNCp{|=*g}5STJFj@v=m1usKr+Is56&#ElhmZZY#(S@z-j
zZ6@1+fz9^`P(>TosUh<~Vxjni*I4$j-{+}q>^I(i?C3EA!w}tc;cVT#u2JDR_r>dR
zgSl)9=Z`K@J-;7WJdJ7e0O3hdQPITuuOG|zhsv;vzWclx&%di=HDVS`J%1dEK-_A#
z=&$|x?`|*4U39Czt}OVi$m5F!?VtbU(d+-iFJ9l4<8SxSZ_Dv*IY112%N>9B3HX*u
z|7M%;Z8^Rz$NyV|^S?h?jLS0sVK{7xjUSf(Sz#C}En4jVb_sZc)$l7u`;dJr6M<QH
zFrj+2t}bh(XLF{CVpg9m)VOL3VB}cOTu{^0Jc9hAUc4Px$yn_EfBXj*^6F+%i17_=
z(B4KHYq(BBxs>+P!>AbIe?Uel9g_*3l!-JXFhJDu5#*8<z)<h<P^<7`xrJ^kxVfKc
z+KQZdT?!VuBS&TE7x2sgAY~6J9%r8orGK7tCuHTx@vZvtIg%*-8+eC!`{;3plQYM7
zvv%wCV^HK2r8PEn@3}fk?l!I~khU3$s@U0*{L)41ws%7xf1BCY9^J~IMduBV#}okf
z{K0t_8pOu%s6VPYFoYl`8WSJW`qtHnu8536X`8msqo&1suU;EFjzSC+T^*>L9(E2!
z?vg`c7$d%)NjkjR_(BV?#`!FI)0LZ0QZ4NN)O1cQ!Aj3)=5X>$VjuGS$l}{{05oAH
z8}bv`O-Q>Kvxv#F{R$V2ZJY0i_x<Wo1bp07B1R+qCWcxkexbDUulOM7IY$wUqKO!t
z7d?i05miqk+|G7i>73|L{zUPtDeW)8S%N{6aT=`7dAB#^uW@qGWk<x6(@VUmMtrrE
zW}$OqsT>*eCL!nEGWB@d=;)8<Ud=fP_Q7X(PYp08;efCdQ<fC-uXsDc032@TDkm9)
z7@HF3Xg^fqs3>@b`NdI@B=!Oqi!#@X5ra%uc4&2CT2fAGBxKGX_zfTSG$@?yoNRg9
z8Z%MiC!<fqD;F*b^z1quqkRAuv@8Ae1v#L0;l(vS&v4!w@2(VFeEws{Mdrq0jIhs7
z#P6JFrL-P^!0uFEL^Ah&eYf6pMdV{mG3WSLu~rW-(8SZjOFo`?RfB|>iMk==bvwbN
zNpG}~bNXQg$N@bHzhq)?=jquq0HB%~cT*x9%~0gx5_?O5De?#^!!}D3`l1{-TiNO#
z9s3l>OZmW<%y3qw+h~?yEGe_O+*HY3JnIMU4c?-+hr6xq(b4B)RBuBI>k@3}Y$Jr4
zeh*l-eI9{<#y#b&B4lp&^9X=1r7f?#y*Cmd&ZbjgkRvAo|9qWE;*sOS>Pos6uD2gl
zUy)yOck`JTnJw)4$sASgFKI2-&P><TavbUG0Cq7&*@<jY=Qj3C!Q*ZXY}hs>n6bi2
z*A;UB^tzYRlEg9h0xs+nfH(5s;xluWX{S$DNZ&DN_%{<Barukx#oqjZU8>hxf1p6N
z{l#vqS{;4!>lZ@#EubQ%F0(U$vNV?6Q&d;&)R?k7Ra26U`1I@CKLC)>J@V+!r-v`^
ztBzAcJ2Ei0p?4eySFfW=-`#HB!&)r6&_4uOrNc92!RO*Lww*uwjDOMyIP)#|vuW42
z{OA@4$TlnA_Erb}5ZyW2<e(`wdEw^zAh@Xx73-A65G9vDzVGkDQVLrve+3$GW?_%}
zB@1Q53vJ|aCi=eg*7T91YvD(_Ipae23US&!I6AxM!@;p>ZTzA4+<v}AY0Z+uvhYD(
zAa*8W5RMd;5x<x3eolTYqE*fQ;e0q*30xYjV3Y<Hcju=Tt*wp`?l^YF#ILGWOkjG*
z+=%PAes`yPQO=b}2<wo;4&}pDtHY5}!lJ8ug$E{AzmP30f0Pfs_53vG1D~yr*JckS
zx!ENIVB4+vH+_D`=p1ki^)B7mJHwpn++^XCZ(~{l^T{C`6TK{;{HugPjZcIsy(K4j
zk3JX;ZKd7P&K}+UnPu-?OdhZrVA(W!M2LFJd`B@_=&jbJ6l^Rj6Lz3G_1qf;$aC6S
za`<q#uDmj6M^}QXG(PdUxauAGA79U)(ALS6hPuv`a~+rIa4FFE1b70-aI5zafP2ii
zxi!JkA`aqnWVD|ZR%lijQ&E;+YE2*VYIJ_~5T#8{^mQ^%tR@WLf1WwD=>2$UdmQj=
zJ@}b&%g93AgYa0a0_Ihr-_DM(WIn;-UJO0ESEt0+v>vZ|`#E*CgUWMioMp`R0-r*&
zx5XB|S254hsAGdS_wsb5@40~ZJnc-~BsRfoZz8Zv^IO88BUv!fZVMGypi}Pb$k>(W
z&`z%)E%dbl`_L+aZyp?TG5mQq44UUU(`%B&K=hs@$-d^2$!B@o8m~zswHq-FDIcd*
z)~9-!nTM8+*MfBN&b)YEHczz~*yU;ni!Mw!=uxIiX3bg~BlNc2=_Y8=++=4Xd5iT(
zK||71b<VNv$JCtG(Jlo_$VvR8zX01KhK9FUzcrik-8Gv=(uOYHqJEk>5Cl$D(hk%6
z#2(__N1~(C2enN*5`WJkaJU5w*{OTsVM&^^ab5UjT?yJytwWb2f~4?*=K72GtZ^5$
zHE5kd+rzV{98+I6H0izYL68fFqBcpZ>XCeX)2Gk@(?J_II8^;NknZdrOOSZhN>$_|
zGwxgheqRi4yJNujo1AU%3k1_lP!1i|`K#o1_js4g#i29d+9@j)^Q<=itg`s~u6h|U
zY-^rscXx-nGNCHGNVLYX3V3qrcWcpVmucU&@Es(0GJ$LVE<gakhD<IiX{y8VxTlAM
zdzHTL{}JN59d;Rj-(XCNBl^N4AUFT${;A%ps_fpCsE>PW-tXdde~#$N?0)m%k5k)<
z5ae!c`t}$h<cHWu#029wLO>4`j|@%{M_)IL9*#)xjRd2UB1WA;pZ=bWxFkAU9<hzI
zx#Gqfq;8F))SQF3RNbvj*-1{k0KEI2j@6+U!NWIP7j1+buV|0>nRSRzMnAi~%3t*E
zn2N>sz*iGXrH(<(Ow&1<qr-J*a*Vb`m4_Gr?Vo^$t!O3#p!&kRv6s`rZB}R(foP(*
ztU7C=s8f6kR`@lqYSP;cu$y|5;Q${z%_fJCER59ax<wp~1GsQqTk>7ky0g#HP=?#x
z7!(N99-lo~uMI)ETobNz4ToEGTo^GbEcY*eDST9tb_9Dbn-Dz=ze7IKxmdP69x@Q&
z_t`agvn@dsXk2#j@9hHD0kisIId%YX*_j9IWcx}>?`fO9of`x5n6u85>jms5N`>xV
z@P|nFNXpD0mrYimd@-1!s*#`%@$rTkw5qkdB_GJ$7(0k^LL$TJz#O3o>ESPN*Ot(M
zUUoIj!IOebw4uZfr)bs117E{tw-pSp`ZR2@e|N0{&x$bDKXk}-yJ#tBA8r}j+>@@T
z$E&wcvy(+s{3B{I<L-lynCmAp_CF*qURBvuK-1I5)^Ka?p2)Bp6#0T8>n~kE!pFg2
zcX|cgBk-IN{CP^B6wHOGckyY=e7(UZB~D%{F6tReO_LAH3)_dWdEuHPp4Z>@*)*oC
zdcTR#Ez>SI%iJiMIW_E6V9vi;-B$_~J|hWdB?l8+Dy=HW=SiDczx*nabyepMax@7i
z!ZLyDy*!Hq{}h*fP*NZ6>i)g!yMR|4f70@Mxf;3TR-zXF^}#{t{uODHo4K0T(%r6|
zH`u^6EHcpbun8d=1Z@8#Oc~Zl@VZoIgbsS<?v^ysdfT67<>J!8e;r;qH(;a7e>J2U
zYhJx{?=71h>3n<~dr2*YD76{yTDRST??>#MSkAS!;B4?V4X^EtRCB1*#p`D|fAm)$
z+15E_zs8FjT5obZfC~IWZg-cbK4niOP4>60BN1oz$tUuz9com9g?(<&V-M4FsW1Ye
zlR6fwA&A=s#t?bWs<@JgD{!t6A3jW=I2AiUDg0vxtSnyL<YUqB7|Kg&S4G`a=ySm^
z>NFS``J2|IWJ|6(7jy*8B^d|dMmlh5<*nWI!8*q$6%J(i^Q3!8O=lKJ?Jv%RDO|mp
z;G_os@oZ4D_WFB7aP-J4`-aL(GInQSSDD6?2%_rs$3c`-=<1P?rU~NURi+Um9U0ss
zrZReMS4E#c^^P;3kKF4wE%fNCj$pntoB8$kiZcH*EgRii5qMXEYF|}_dRc8UEZN{7
zeFIH-snYpW`4MH9(oti&`}6u8XC6MVK0Fw|QV~qfJKBDPVG<ycUH39;n)O1PZf`mF
z0CgI$w>My2a(K1kT*`9JxGhb;gF3?K-?QH`C&UMQFU_9wY{5N>UBPWNV+75k#d23>
z@QIwkhZKk&*~gf7DkH?^dZJqv)oSoZM!$E<163Wl^Vz2C{M&JI<05n+#@T2K1JHT4
zRK(?_lnEwc-6tNS?zK-A5GSQo{B<ecV;RmsBirej6%CBIjBO|%Js?N=o77(hSEeEq
zC=0TP9)m>8hVrialtbp$)+-6P?iFcrp)b9d7ma68XDI!NexuS$<yi4vM_h$tnJ-wI
zRS=5k5?y~bV28I94L&;Gh`<1?88}=~1)snuUlYp1ZX^k7?iMz%?Sd`yd#QWzR~CKX
z5x$~tiu1(zU@NKydd^Xw^6Aqh)Y|RB3@!8p!MmH4#}}T^*-Zgq)D2I|D9Or&PTw49
zoz#i(tVE?sk)oPVOHpG%3(_^ya}0q$jvK+p;mi_>lW*NcmQr|6HPhV&QnjKpY^L2k
zJjO>bKQ6larIFMo)!Jx7{wkD2G}0o}|BTyWa5@REy^L!*$-Fi-wE`0~zf&LTet_{?
zDkTP%vxBFYeIxj6MdUTVwdgD*!IGlYCviji6OJ>FW3-patRd3g0-|hauj6&Ep<e9_
z4wBAOGvxc-{a-@QlmdxO)0!W9N&$9YX<4t@1EtLnY#A~xMpO4={DP$N)zM#`k7en0
zG*8BR<@Bcl3ikJ9{wDh{ZxHyqckd3a>-$&s-m9IhFBYh+XEGOAZ}pd62K>>_1DkN6
zU;N^nRkvu$FP<H77M6P)E(FA}BLSKYMqe1Lfi?|wxKu|N16G)F+Y2zlZuiu+e;t|9
zyV%ow5zOtu-0_~xy<UPMaZ-P%035Cz4L;7)FTdKN{q0|VTi9>m^X(&9@+}oC`Icjs
z{9l|3_F=5BqU}{-)6b_>7mi22HH#gcZK&1pGgt(-zJOR%y<SJgolq6R?Q4pKxRo$6
z2~v+U!}p$vMBs6=94O-a`Nts(RnwAk!Qn+}-Dl=qiuTo2YXm(hiG7q&Z!s~1LR|}L
zooO(dWT682@-TEQd68-<IefKF--wA-iEFt;SVEL>T+J<gA*lmH8~9;loelxvhQxoH
z7#gFO24jDCvfE>YLfV>e(!YwQvEze>aM0+;EHg?Xj-^Z*Ebr^ZBYdQ}L~;H8*wjER
z2q-mAN5Y@`c9e_8KP$p!StZk4$-Tvv;I}cZougOv_k+2rI^?R_GM<Uy?yuFFIPHsw
zuUuyLlN{eZX0XT9&3XmQNESaenlw4OL7bV~M<GsMN_l-1TvjZX!%4n<TGA4zsel_F
zHyqs#oypD15>Aof(Irh#Cw(jaDAzZ*2ZkjBrdrw`rH}18;HKKBz-}lzjo?Xs$&`|d
zuNU}-Y9izhkwh;rNW|bR9vY!`(!n7u+xH%+yQ~H)B&k|Qv{fB@Rh8-mC7I23ZYc8=
znh;#|eVpm0ZM$it(KnJvsoXr@gfPG4ue#9_pZ)3kc+u@b5%BXnJI?=JOIoUa(mhfp
z^BB?#FG-a9#w%#+SoT5`@SX}<VoXh_mg$fcq!MQe-#RqJ9kg4);>V10)=G)%;1XFV
za~PVXY*MvbNy2)=j>jU##hug|j)NzUCzTwUx>yE<B-nfHkiMPe5X*Fg$%cT4^tV^G
zks!on+5EK~Y}#0(#h@$m<pZ-_$tL_&<A2;KSV~9~6Jb2$w=Hb!dB_8vPI%bU@g4q0
z!89GI{!eL~4PX2U%5T|;C<lz1M$e3I=yromha2^Hg?jPGw()DVVn*6qc_aeHPJY+W
z-$sOm-|>AnN$DW%9dTt&jm)M%h1Tj3gBdU{Bf&v`Ei_p@gOM>wI(O`$e>uJ9I=B@f
z$<5H>`y96p8>@FVUK}2R$sYN$Os4P7bz{;}^=~Gpx_xirV$5Gf=S|k6nITg#E=VMS
zO<HelEln@goZ8Z^*&GZ5Q<@e^S+n;zvo!eHOp~<+erlz1qfdF0)oE&u=3UAOX%XFZ
zd4nZC=pp3YIr`#B->uV^r^lAO{v>L<|7OZit$3Nbowf{#a=6y#k*q^8uUtdP-*_{c
z)Tu&?*|#PSAxVfN?w)F*tD%D{|16RNX?vL$s`nfS37&cUes(D4R3H;^)82aed&}^d
z7a`oq^GpP|F8F-0VA|snY2$b!)FX|3Kg6f&7vMzksUV{gM(Mv_n{M;l_Fcims^ZGS
zmYo@ep#fo9@d_#9;wlcXbvsR`Ksc~pEmu_HOb9OT4w*B{&1TaTSd$8J;067|pEt_F
z-6180)zoGKF>WOxEvwfUd_J{dvS}{6Juf9YL|ElbkqoG^JMMQ{kIkIoz)kbqbMXIo
zpPDVs*)6Yr>Tdn=+_tcfJvJ#9o~b~OjJcKv!L&CX)EK4uU(8`8B~|7m5*~YHSo+@b
z3JPtzpmugwQn_m)cdcCVF|;uZEx>H4=<p{$H}OykZ|bU7C)f+?|K*iDjSaZeN1Oj%
z$`3$lJjS2IdXjfy6C#WEUdMMEA+?KMB(h8oM6MT`QeHpyt@Jb9?k|)yg=<Hgf=;eh
zSgKJ^pa=#NM@I!ib}E=q426O$t?7fRCite$VvH=+GPwtUro$DeBk%tC?H}b_FW3tK
zA45G||G{8v@EwI*w@nT&hnrv{yC{&zu+i5FY%mk~(lclMV$IjZ`0twI<KlSZ^>%=H
z-CAy^puB9*^p$697Pk7k@R^FD=B5KdgIacR9qAAa=j~;(DW{p~v6nm11$O5dtHk`-
zX}5Eq(*vlor#y0-C)YJgE8-e~OOPHL58)E1dt@^=a5lw+`k6?vZk9<&VVLwvqs(r8
zMMizq;X6O>iu{+f;Fn_g)k^qwt~o!#=##jh+7W&E<M&!}mMb&##ja1PeQ=lKg94Q?
z-?bR>DU}4;q^jQdhJ5)h?zsI7VqEs2ys!4=H9$-JGdSB&rd#eE5wd7C&Ab4!cOj}S
zXy(SjEcvYGoU?k`B?e`>E-WG#vBC6q?y`UFQXX|K#+_Um63Cz64tO;cy}kT%=_&Z=
zdc5}g@y1N?4G&3`iR;-t=fdn#a%629_h$K>jKWjh{Uw=2`ds|l<}_L76G4GkfonBg
zLFH0OJE<~zXZ-9mjk$1G7A8?LaynUxJo5I=(^yoCu|yc9keEO&e<nML(i8888v2`_
z@w<1II5T1woDtr%|Gg*QN{JRBz;irY+5LWSZ<#$R2@lINf~K(K?AN2oeHHbXzi+!$
zF&-q_u9YxpwjWA3Y!J|n0Mme_JU2FhTsT2|*qrQrDM6!Z`X7-gRGw`%jKQaw{mwOh
z=n2v7kFy@X`Z@{%Y&PAxcQdwJd`2rYG~hy37&R3z_+C{f8^c9a4=(qFG*>S(5|cX{
z$EKBN7@|XiBsHj?hTV{9A7EF(x~u3luU#E&PE$KM6i(8NPfb!qsiCWVm;F<KjEN|!
z>+xrvhPG3!A~xxp@%{Aj^Q{$O@+4Rh9px5VUl7fwo7oYlb);595?4YGuZNdXK7Ejq
z>Jj1|?Hzz!f4IR!6Mf;3FXvKD<}?PaHVA@CpP2TVNO{4mjZbNR>vNvDu^MCNVOmK$
zMQoMFtH7KCt+TV-{FHT@w9gzA1wExT&)V6(%i#VdWxG-<L=B!wj9pc(R9Cy|%r^D#
zlRFc+ZEY#c$X^QH5cKDRRG2zWSy^FY)r4LRF}1;%s2NKz_rl-TTKZo*)2zL2XFx#k
zs!}-_pKXIW2gTw{ZyF%)e;j>tskn~LqgR;*gpQ<I4|K_MVu31r+n?Dg)h<4P;@^aT
z^QBel9o1UlAvcUnjs~HEZ@hWoPmyG|Zg@4jEGTE%`79Gr=ee_)NO?8HjWy$VX3?LP
zS@Ku0)`&fVZJ*1=b@AT?EQMsxT+ApRoDgro*k!wOQfasa<dH!|FtJ@q+j>~bB}b%G
z-k~8m(L&>Ua0Ykpmyb?;&xb>m&rDQPdyXQ0$StY%iP?R!%-+Fr*n0zFgu7c@JSFrn
zGALS))}4vN!{It4>l2t)#C-*SVv!b}AJH&F5_P=zq}dxV0@`}&L*!_?gPX~T!lZ^6
zart-Y7l-0gUq|D$pA%e~o5s^7z@9XL*y@~hYw8JYC~*=Lb4R>nPFZ=avvvb;Uei#z
zotXPeKp08@g==KgcSg`=fX#0MFL9|FMeZuY%tzQ|%<Fc+I|JVdPhqm(e%wfm9;HiM
z)A7)rnCw|ctQl=qqPlO-5jGjHX&J<O)pA<zXt|jaPNt>&w;+m*ceU1uxN-x9+;(Tm
zI*j2AfZcH#T!r9Dy7Z$n{NfsH+eC)W@(Z3!gkA4>s@A#Wo4Xkgre=D`d^cxb>9p6b
zo0U6_mZ^GV$oGsFSMcm>QBD`i&o|JjN5xx={8kaf^-yIQW`=t_)hWC3wyh3JX?J>c
zM8qCMT~Hj>U^2zCo}k?9^C^5j(yAnOHLo#Lj`mKYq0!HW9MwfEC$&KHcDbzYNm&xf
zz2*^Tbf!#7#HGrzP|cgo@(lvDL>&r*ERR0z?{i|d*s;VfDo)q_dI%cXTD=IQma@}g
zdh*ny{>`0^;5`%VtykD*S^M;bgda6ybKJ=KomJKSL`u`3_`~Fuw8m_dt)?tD3+C@T
zmfRDX+l;vNkKb<qk^$q~2AqqBi_p>YQe29L>h+TMq0=<S&&8bTOe|?PI=RPy(n%}M
zGMvo39@?<#Ko57qiY_tmlq<xLmzLS#RC5UnsGqQU%td|Y*wpnzmKh}y!5Z1ioSG~T
zDY;J;)y#~>uOJb}y7zO@XwMidylFa-J4USG5YsFRz_;~jREhzmEUvCzTUynjRopp?
zn4BvpW_kv|Ttc5me;|(IB5gdUf~K4rEVFCM&Y%-A(>%YT<stMcUTB)H3}S0`>$E0X
zE!ErH?9_9Ac!$!~^esMT>~S6S2_L=aR6%jh^hc^-{>{&H1cBTz@iIZ-=jMQv1GAmG
z2MQ0Nj$KNjWm=&9!ad!1S7wE$s@4~a^{nvar9By2b;W@<Upv_l2AY3}Ko4S|a~pjb
z({(}z?r%EOqL_el{fu6vsG$Pg9$YUPbJ-y9iHh`2<9)B?-BfYBcIuWorY5rIRc&qd
zxG<dDWYup~_weMO;3twbtopOiH+WX+I|S8rt3^U6GE5#-b@J3nw-@7&yMt7LQKhKU
zll>-EmD>8MyItw0bIaDw2iyHe4VAmLOQ)_=GDgcQI5nRmA$sf&l6F6_Ad)BxXvW%$
zD0QPof+)R8#XggdwBErgK~x{vtVSR|S|xO*5mhs#scz$)OAhMiN0dS1Qz>>3F=bG0
zjC1bAYN2UX|Kw}}F7c25vKJZQd$-?_q#5N@6p~MYu3y9y!qLUWC;p$$0DNvZUbuSx
z!H(75CY8|hrCP$2i0k#H-gcHmHr3TJfq>gBiC3<#v_DWk;Q{}36Za8$-O02o_^^{T
ziK+w&|2BC1N`R>!w6a5*cuyv2L4a}9jz1G_8S=V*zl4Vndi1pcsR-Y10X#?LY2UZ}
z3YhymHRt=y$a?iK`-IJh9i<Q8?nC#XrGmJ=wD9!W{qZW4zF1Fo16d)<=ox(uSf572
z`1(`nl57^HX3$K9(i@z6?X|=VL*e}kxyv6I5dEbhsS2-uARqS#4lAn&vehM3=CM0t
z0UHW4eoymK0Fw>mL&~%)I5Og&ZAI&?BZMTjw)R*5MV}uQ>ur6a`kPzx@luuuzk7<D
z?H5+XFF02SoV1D0pTDoFsXQ~1ouh8O87VAMSQ9txZEbD!P_ggs_<}J-Fl@2w_D~@>
zOB<8)({Nu*D9rYL!_~ia55DzCzQMoW;NK<xH^aZ*+NJ+Hn!+tEv`i~SeYa|F*%z3+
z=f4(fNxPuHEmVZ-hUUt-hsG1rJVEa~0XMrjKR@4i0j*uKgyFKtQEpo{LFtgf*v%Qm
z`O>hqS^nVlIfMvsRoT<yT&n$iNw%cC)h@n9mcT<yHd6gss_t0Kgp4?|d~P{EGs6U~
zLYrnq;l&QA79>A0%L^f`mx8JW)$|CCsd4&Tp*q-V_co(*42osvPL_;*Xbk2b4j#?Y
zLyU7HXFU_N0o`HEq^E~y(&zYM&zIC+?FS}~0=q+BR<-twGy=6yzbr9pXvmU)pKK@Y
z66D_1LJvhxk~o9CvAY3QYsnwDWHSBv5Yx&V62H@s--3Oq+MLySD3~NQpV(C&kHT5;
zA7@#iSDB)ODRU_DU%)fZDu8<Ka*K-5ZdQRQeP-jKG41W`gHO`V3q=K^eOO_10@Or*
ztpQ=M%SNb-Gksgojiu)Ot7>*rk4BsHLO}>3{(QX4s|!cVYE<>p7SNx|@vs#(Wa-)t
z2TsNsUDTw<?3pv{zWW`d0%BWMkE}vd$4pA<lY7TVDO15S^l_;)cl7%{AGYjvP<CP@
zF$8&x<H|g|QcM24Psw7U<`YK39z8l9NvBy0gi({hX4C4JsO5SRcIT*6g<>zag4EX+
z|H{23Su9j~{Q!<3h^zLyt%gq4L0_BZ7t?BDRrOP|kuhbv`7NK0?E&^M>}isGza#p;
zC||(ks?*KA4<2C5Ec<#^RI=(Bg@B7S2W_{zh8vEXE<c6uR!KvJt{2r)maUO7A$img
zxFl%$!4*!gUiM60H+wb(Amo0_KiZX-`LB9@S8_W0AMs81Yg?uSt9giqv{O~<7JF6~
zgXHyCgV|jv@d*6veag8|l56m|u2{Z-VGsOzL%i6W*Oy;!iCM8eV4%|EMOegqwC?c)
z<MB?OWKis}1j-o8lMpv^5sZo%LdMMm7B5>rXwLKw4P^<q3XmEEQ${<FjgS&es$lz&
zp~uczJ~42=i5S}8t&~u)Mu*%KCK9G)qX-kpzsD}x=n)TY@UH8A+Y_=%2iI$R*#yjf
zh58*FZ_vw%%C*lj6;@R%!G8YIQut7JVVRchU3Iki3JDEMu@G0g$@Z@CfxyydOASM9
z_X}Dza9KB?Muw7z?>s3G&}(45WcM>Xvr$=-$+7PiE7w0+NdGBCPyEZ`?FUmMf*z8B
z(x5=5*)gZ4XUA!jJO@JRN`^haJs}X=stD1On7I~Cc2?Xj=~T>%!6DMcW-{_q@_GDv
zT>u=0hgF}ncy&|h!1jQM@qQh5f02{X&xUXNQ*{gdn+m$=?(02+c2*0x#*8!125&HO
zaVomaFptl(p6$?N@d6EJ=*cF4)z{nRM`ys(821b!nrfpHijpx3G3gW{DYV3ueI^-U
zzTKaDh$+nr@voVhDL~>$PzI$-dbgD<%#Tt?ScYjeS2D@+U99FlPG{2+()3g4J>AOz
zqJ7oVttnd#40kkXYE*!Rt*V~+l$Y~va#v+ThF4!g0NUI8RTkV7S9a|b#_pQQ-0jCd
zrPA0IqGB6yKr8I6vPOCyZW;z_(m^Ljgr5;yKJ<CI(OEqHnWxTZtkw~q8ST3+RZG@5
zlN?S8U@RPeFAHk{aZiG!Xpp$Yf+oLP6yo^gkV+a>UaT%salf1B*TY)DqxSYfy>+%b
zkjd)=${iiu%O#DO%ruJM=*LsROwBN~zNKuMIQ=?ADr}Z*z@9u=-D%~5Sd?kht_663
z6#bOFI#E=X4eK|Mlbruo7+b>E4fVdQACP<bJW!t0?=)X!0GzZpUVNXx2y8Xj4+E4f
zXF^dEZRd$x!%$)Wq*}pPhh*3U*<p<-e^e2_hM`v`@Gm&xY%~A)JQ;pRfy^UnVtp(l
zx;Bl0;L5SVB86t|^VJ;@eEkr76o)9G4oPFJoi>l?L*7?P%)LpelMRuTG_!gUAiL>{
zD7@@ecSDGz62beWsu4g8AMkn3>5Yq0G(S@JGQD*x-s`WRKs-TSSv3v@A8ebQbQqLb
z3hy^OxXN^-DXaBscIWPjTpA0|!;8=LTbkf{uvZ4Vf&@q(CSoR(S{1*K&LuVlWxaQ%
zj9DwOjGmaxq>q*OT%zi1zrJ*Hq^**Dzi6n&Y5o;kjyh<ghvcJYCS*u7K~x^YiI^Ie
z_hx`zsx-oUrgAs>J88k_nW?^6j$S|j1v1xn;Rsa*^GCkdf`WJBd#3jmeQeH{Bs#D|
z)-RU0kgR?H__s(xxjiFpO0aZ8zQGQNZf(=0U-jvMZeVi&NL|PPimjb%xZ6X;QO8jo
zuzOlYJxPd>L6oi=EfWWTXzTFmY^UOm1-Ps?=rIDHC?&R94k9>ZjaIMDUihez4xQ_*
z;xchJ-V_5OVfn$CE5#X6QZ5y{Q*t3Lkuw^K+?F4{+k)^04jZdSJ8HYKVC+tUL-~QE
z-xeY0jiWQ7BB>sSy>Ci#-KvqPP!sUzV`{M%Ij^G<4&WZc^Fpd^>EU4be2B7SKlQ0e
zkf76@z$Q{ELM#4(Zw*m`yI(qk7_H#qj~?I#$a-w*>&9Vn_4zd)-C6ODNMXmew_+X2
z^v#wV1ezxMaa(U(xoW0>;PgG=md%{KVy~g1mLD?$Pi5@%a^^i>sx2k9r(@$H&M#6K
zG<647m09~Mrd1-OZz7oTkubEGjwu{Ez^)Tac}{n~b#_bAMgurnrN+T6y#2QdS6ziQ
zbw>sJmo0mkrv!6GygKs^2KGfVd23Khx7jO&muaMPAlTR<qDqfr<9x$<-1`Yaw5M*!
z`8FeMT&)^6#VYPtK-GKE(+GUAt}L;4n`IXJ9Nf}tCS+U?=2zZ3A(NrT&g=&%BHNuX
zb0O|QHc>J;G%N6S9o!4GL2z47^x;ev4VdGhjF|I)hH!i+ro4$r;es((uB6cR-^wH2
zzmEE4A9iN!yGG+)RVX_7IOY^8Ren6{mKA!YaAoYf0>t3U)QypR&hEOU%c*4h9jM;o
zRL%j6AX)3T3o%-DZgBX01D=Jk#LVQs{L!;rXIT3F()m5PuecM&_p6eeo2c$Pz$Cw;
ztIK*;cqvYduD7O<(Ue5CzbflGyPp;@UjLrJh3wbzj&Ii2KB-2QlWkBscm3zO%%Nl$
ztSaNrhCrV#m?3%U-|%G>-C8mq&bsNAGh7W`Zc17KTeh8Uo<*N^19M!PHS}*{SyfKi
zSz8#Mov2ichz^6F!op9E42zp28OepXOXiRKd4XgPCI9)yeg*pgK1wMu2-k)=Jyf)M
zTx%qz{-_Gly_v<<0bT4&5|wVBRUqFtzIArLPQlnE_dq`QR?SddY<@F@B)n}OFN9m=
zrZ9Xh!wluk>lQ`VLek=KEXZjVl1i;(P)9DN_DDkAXNyUB)@fYrV!+=fFcgt}RTuk$
zNK|cGwMQ3k>h}e@0S@&Az`;M)3Hz_LYH0H^fqLZ1-4$L&<DvVhE?T-MS!mq$h+a=+
zSgi&<a=Cw&hg)&6QnSy56PVXF+ZpTj<s3}~t!q#fv|0wp4lzleIUE&==+l!dh6IhP
z(_XC`A7af*KCUN0WSd#|KCtn!Roav<72asQMhWI#Z_3zIm&E{$#(uca4;$ntw?p>r
z*^gbUe0Zr#lI)A?t_ggzAB+(U7o7?o`?w!vJio|G%-(zk`8ZNRT%CnUh(7dJa5(hI
z2T+$#M}OA0oRy=0k`sy_Tur1%iIW!|@2*!scs1MTR9{%8DDa_@B$SlI7I|UCn%a!J
zqy6=fe#Ww)FQ=sTc8}g4&RRD4h5GW~Vv2v5yx|w~sWH&^kn!ytd#gWc8|-{~jsriM
zpTW~+*#6o2^8$;L0_OXDEzP$sJg{%-W4!jvX?EY}R6CJ0)Z515M^7@#B)iP6`K@*(
zn`<h)%@G@T&sqpOmV@9y2J!<EI?R=@_MBJytVuXzlr#lHG>1quRnib~VSu&L2svD(
zyx7O;;?88-`ZC#~k?sd9x4^Ci((`5Bg$e|I?r3}>?l{eYNf{60YXpgv4=|ov@?|Yq
z(`vsbg7*}a+1F4iN&X4Q0hyDdJi@H1`U52%lXaudDA>7ni%x<6gv5^N<ff<snu3B4
zb}A}51^9wqhGGF~R7OSW#8gt<jRg~e7(B~6CWFNjZC#8d{<4lpNofF1X>h5rB~Df~
zy<n}q#LwmKB6Q~#7k}nF7b_a`8r*84#eHcR;FhCsPxLv~Z`*hXr%Jw`=>=K1g5IxO
zESA0@$KQD8f8qLngNy!mz(os*@Ecco$v3e28(6($$^Xr=h5spXTmndT;P&KmYl6~x
zXgo*L<A01C?@&m?)MUV7&+XvPc2*6?{11R*<8DCHxWl77KJ}>M1qUvw`BDt!*y;C3
z#h@DU*Voc9Q=kI2wK!rWF98%gq(RH-uvIz{#p-A`x2Zw;;}LvT2csrZKTRwW?40UD
z6B<q|R?@WjuSJua!%y`C{?x8ABnu*qAn*R?e1g23*56Oti^7@Y-@smsHRh<)3|7dx
z^S5_1;VY!jll$~f=QSkR*EAzgQ+Ft!oC9VGl!+RO5Ml@4BAn-FORD#Vvd2Wc6rqnW
z_0FPLRsF-nUnAAf2gkowss9at+-qvVJ<X0yqAL1?PVX-_DYu3enCMqg3h4`&MY)Gv
zO)C-}J=xxK=G8x`Q;E`1?_fS<l#WZFjE&`Q7kf`<Cncoq#k)F6ef6tk+p2MX)oBQm
zzNc(J#<ss}y+SI~G4={x)qxY>$Li&i7JOezErzRn3o0o_))CyoeofD4Nyw5q!1<PO
zL0AA27<IJ0H~Hc5VkLMykmP2yCeaMsgfZEJdUNIYzmu%l1rX)6=a}q&Qg_=+Azwx|
z-_MSZaE0;mCIvC|X)uIXGS&4@fHn!DO4vm_1oDdBzU}?z;NHd*B^A?gjGOs9@46Wa
zPz6hGKo|7o`bzy)i^x7Yv13^yezI%xKokk1Gx9xL(_9JWW+o(&*F6{8A~~sR7ehFX
z+XS3Wrt9y?`8Qg%5158s5S^e9Clr&H3+tp>j%`h>jbBF11-6|5)p3y3><h7lmONb4
z^@LISibxTr#X>8e$bC&Qu8wJv<lm<hTEN}%R^%A*^}Jx&=_9x#O^;jmDe@=`EC6lP
zF|JeYuzW~Dg=%R49pdAhcy+N#`jY{lcy0fj#GyaYzJ{J_pN4UDi_tFs@yp{>p;G^W
zpA$0Qz%SDoSzFmX^WnSosyJC^_J~k=%#Vq%yf9|YZ+33!y#J<4ZfEcQ!vE;u9~UE#
zoShvoc}L+FTm*cuje~z7zI6h>hF|RgWV{vs2V@+K{}VDkV)8ca%J0eMj)r-|*1Sm@
zs0s_m6J6NuK4Z9kO{C@;#-|=nyF<+*#!&b>rOW4<FMbZqVO+fQ>de;O?AJ~msSJR|
zCcfzx7aZ~(F&yjBW4w2!`+K<N9xoQ1@8C5=DA#(S?=?tc8G4u<gVK2!=VhLE3Iab?
z#Op=~>nDk5+~W`-=p$Y5tG`BRX-?Go%B{}9oEgFwV6(Garqp65N#oqI78mt+O5e<r
z2!RTL=aTqlQ6T@85)54MH+zSZzv}=DV@(z^IxC`2g&@BaPkO!62*@S=3Nc&pSEYUa
z!kcD~*Lx^zJ=^<K7n4(AnfKJnemoN}A3u~N;N{I`M4HCO3I?si7{%Nej<gg8#&t6E
zF4<ORo|2s7L<EHU+dpSa_oRNDp)-<9<Q8_yuC*m|LUZ#e0~C%>>K2L1l32ZMs~k;F
zn(m!1sy{e^-FhH*F)}vL+Xmzl?{S%7vyC%D5GAo4&2aq10-uh}g)g7KRa*`M3aH#S
zXrix=GhBb}23~uxw{s(a`C7XGh*DD6@Nlshd<Fv5vMCIaZm)0I!z5T3p6CYT&a*Nw
zgEj_=n=N{Zwn|6N)a8s$v!kyt9pS8>txdRuPpazZgPQHFO&$|U^rb7fM{#z=9b%+%
zTDND`8aFtM)=7aB7aku(1?*c^vGH061#)qzHHy#Y-PFA5XlI!)fjKki8V4xXv~<)_
z4Tnxb{-8K;wpx9kvJqoGb-x0h$3)5p$cm%#0<nQ5wg~D91q?q9$B)TUKArGUTUijL
zFr2Mh_|Z?#Qm3!|focaXrFRS#QL?V1+Phi*&>SD!*SG7#1v5(>9lD|>`FzgSn0Si7
z>}bJL1pYAN+=|8S`zLPX7VNYW@_Oy8^|(IepNRXK$g)YoD8r4xIjNpV?KN+?={3>p
z{g8ICIg5}8D9Wz?4|{JK)#TN-4cmHGsxq`%MP#yC#X10ri~&-$N>zqh6-9;^6%d(?
z5D0;2QKBNm7L_qXML}f<f(QXZgeZs*B18yb3K&d)KmsHXLZ){Idf%=0$6DX>=UeZ$
z)}y~%E1E0U9?pI4<2;VzJd`=JIQ{z@(OkUjS46h|)P90uA}80w(GlCqPa#0a(>*8m
z=XN;eJaPQ<N$L2M!7ps6tlBl-%jT5%aeLyv4g;EpZ1n!k`c>0`2dh(FkvNmgGf_S)
z{u~(mGNQa0Z@7e};82w}A^FA$cp}GJyMyX;o$_-W;u|ZWSqC4|6T8}-rwJLYZo$8U
zclrb;$2?CV%jzcjO2f=L-uAEt3SlW57dknQPOu4eR8_&%sU)Dxk4*^$)m<c_JTcw+
zj|j3}WZG)|_8p}OU!4SQ8bW*u$z}AwO06;UUZ_+)Fp<M)v8-S~7G+febFH5HNw&<v
zom6c^!g4=LiS9~vrm^cez|i*YULms-N=AAdoykaX`f^pv9NR3ca2ufIT?VA@^Nh@x
zY5p<Ug_ZGNO5EsUE}{$d>e`Rfv4R8|nK>{c<Z8q>oIq1nyC?9(>w=Vyv99q>r(VGm
zG~AGTpfBoL$mItj!P&Eq*f_b+`mW@j3(bkPqW|0>2^r*Uv#&qhRdHC+`gkHU^fo;J
zWFuLyf@hdj6A1dB(tJg_mZ-IBzvTsCRV{cKra4c}&7wE6^OmM|6S*0h7859hDUbQi
zK%XA^z;rIfuyzA9(|0<alKruyG6L}eeTrHv5!Vh?3joX%jY)(UZK}|<^7)8es;w&b
zI3s-C@&0?0`zq?lxfAPCG~CLm?xHn`O@&Y1TavOCWb&dE3@QS)ABoD$gPB;<!RGvg
zdA&bEE_{^1SZ>WjEdG+jEDEVkejdqqu9|5bLNp`^!#AL}NB>;+=1zhkS9{&rk!@%P
zjW!9rDjOi?X~z*~xl*wFKrJt|i|7%r$B|@VV$a`OUp3DL==m3TT5?@WPOcd;y=tB2
zT+QI!eTmKJVD|PM17TIe8cKhNY|V}P>Kvt_o`pPaqZGP!%DKA{CRm`$gYpOR*h{EO
zv2YnQhiQc|Hmu58!5mz0qU?qxv{u~)n~gOO*$%0Cku(1&Y<4|1vc&jXUOfbALQi6R
zN%X)Bu5m0H)sjMBX?~ei<2V!D^_<|?z*@YB5lFB&`)(JP-cADcWdbk#XM#Ve+7lA^
zY^ljj?1~JGQ+nCYiDR;j(}DG*SAjN`|N2AoXtTG;++;Gl)D9PzX{s_*1_#hLe44rj
zShD#FPlHl=O-!f#pC3K|(};LluFlLklV;^BAeQptY-|Zc^RD?HL0$zKt-<pPuEHU8
zdDpBwEqbLlvlDC-0($a5)T}r_)WQ#pL@3c`X6iQ0Zp0e;Sq*3q{S%h@dB`@~mgfqj
z+-X)<o9G%Z%ahZdVCee#enMleDysJY{wk686W9KOjE?t2^q3z4X-x~n1ktj$)CYRU
zhfz_VFFel8copH?ZoDS89_Ur3?&|+*Zu9k{JZ4(7iB|g?rnm+YW=K?<_@iI%Zg@78
zV2sl+1P*)8BoYcgsU<Mj-_>}0UC=Jx)Xc}7i#1FJPI3CiW7UF<#af{Q8D7U0DAqVj
z46x)WZd2g_VqGD;^3*3J>?Z3rlMIeaPGuP!d;fZFr1Nw|^d|k;9A_O3=Ut7ACFr1Q
z{*Dq*m0Y}1OLBNg(v&kkAGnyjTFcRFaq!{n(+7vA2;IWegE4o1Ai<(L2QJ7g%apY{
zXn{aGhH#uaHZSS>4$K$O^bOSLxPi4<HnM_j_q5$pCq}y0Z>^3UsN!2;VCc>LgXKxw
zShebn`}$SC&O<Xhzb-%>F{B4<$E04rOFXdPAz`T*az-@RKpvmnEUlB}8%Jha(I@qI
zKAj`hPzu}>Scd$EXmLN7_EW8AK#4U~n=~+w*_=}QfpF>-cn!FM7iW{ay)q>yY@Cx)
z45q=ooY8pc-@Yc|ShKvmC9ZL7O`VL&CQZAWb8<vppL79xlf_GsdqQK)n(@{%GEliR
zPEZQm;IR?DM@20MqyDtz4NWo!$2KpN8N%h7*<Veb5}j%(y--yCPqOy!<qt`a-2(0J
zUeidW^CV&5=|$~t8Bqi0RfdD2{^33`=Z8K3zDG5udJf3hCdOB2*Cl3umJw2%4$Fa(
zRT-;H76yR2w>FX2({JC61l}xiF*`A&A*?ZQazRP$d$nZ_4M-Q>XhC{vVj*Igwb)&^
zdR4rWm9v)Ks|NG$YVxV$e`p5KW=?4QMd!Yh4q8&Iv7L=I35pY$_XG>l!twxvMeb)Y
zD$myZ{-)v8zlq)i`Q;0M_Hvnm+lxVNEr6oTYclJGs<W?-vHkGO^iHG~No8J87<v`Y
z_Q7(Zu-CrYhH>djUMM!N)rJ*e+p9|Rp3jj`mZPJ+kttwr$IS*rDdo|kmFJjYU&GDA
z%>U>R41J^V<3C(e+!rXy9%;|ygrPfiWRx)-#5l%jAgU$!Tu8>>EblP?E6<~$>E)Aa
zt_5zgM(@z0eml3OUPk=3z}M|XY9$!@)Sy}N<?}1AV(CJmY5ucCLkqp(d_i~7qRN~F
zu7-cL@x3HiXkE>(mqqs$<eq<i<mZ7s|9SajeBLB}pF_dtQ1HJb6eI{x)1w|y(n(*@
zyxw?G|E7hG&fyy=)VR*rXRE9L`b|}3DOV`{w-<y4T^;M4qpS@j=xIWLrHbn7ty{O!
zw$AUMml4MoNCA%Yw#ETj{R%-3$-*L&V%^YZ4$A+x{+lXb-PL%wQx)z|5J%F}&Pe6j
z*?kSkoZ~L!YA}~0Rpiwjn4aa>2HK9)i}D7anSj{s_A@)DCGK_cD*L6GP-8CvE<*=!
zz(-eww7`1%OUo(Si;LzGEDQS@nT@(?;C^_a%YuuEqOiCt5r!%SM2l?;fcJ`V#_ahM
zdXhF%qevTN#0LAcA+pP(^0hB!O=zu6HcN@+##o;szh=c$F|yO3<`!UKMHqP5(3-&w
z;GGH{q4dW{e;D-=voKS@U4avZ>?!rK_{2=&yf(6qbMc*;@vMy88WanCTR*Hj)t~sf
zm?~A9*25zr%4}~S>vM2awQ)*FZ(pF#Iaar_x^6mzy*68Q7FemsznPnbklQI?;%<<5
zTO!NF%Rw7aPtI<v8mCb6W)jrA87Y)@s+P>KJJbT1qj>B8gW@f6P*c2RG+W26rA{<A
z(+lpUjW^-goYe3)r^i__15;7n@quO0(;7^P_1=(VP6b8ZO2zJNF23@K`0)K7&S&wt
zlBvijeAS(1w$s26kxL0p1DXAfzllG1pmPqVBGeWyDuXpTQ1mY8mbq8(!Z<;R_^I|K
z82f6{%18>1NYjtB(XL|e62?4VSz>KF6KHK8aGt0tX+{{&!ufqeu@WooJ&~>t)D+l2
z_Hxg}o(egUT<mD@+ll3Yxcz0N;6@GjOL~4$VY8whe}cS*g^u*|&V%ZA82N21C{lel
z?NeZ_vid|bzjhPFD}jWW-DsaJUSD~x^ZQ&T|EhJU*UJ`*(c+5KnD3DtfyP>b@p*HX
z`ExYn5>Pb09YM_m2qSEiveB}`#NfVSDxjRshL>vSX;WLehSK_whwKk3fK=S7z#;V3
z0Jz)|R5Ks<Nz_zrcCzvWDne!s$ZvgWLZQ$P;gnEji+f_D*{#xRPdmvJ<_Rf)0;@!S
zll<Y~tEff5d;K2@`Mw&4*;8H$GGgeeKgG|V98$gS6KucjK!RaW&_$%iU+{j9Iqeel
zA?SYPS~oE<VQ!MXb^~C8OX5xdX|wQp7wr(ECBe|NGVoPnep0LPA!{5y4Y>J4U3vXF
z5V{OP>-|2r<4)D;1jDC!e@kyW@S|9Ns@aWA<Y?F)>u!KF>>yy2(NR+)$BqnSoX?oq
z!SPcTaFsWioHO65O@PdwuXYR>Nlu9ls$<1S>U4$H4P@18R`1t;A`Chz`sJrhEwhUI
zdEOWMI8OU+TG*cWhK*28tSE(r7Wp2*OrmgtcVTx$G3MQm>qp^O4`kdMN;X8ed>YeX
zq}?o=Kis~1(OFFWQY{vs+AViHe|o72#W7Iua{PzUmY5*L0ccR4F`inV-#YZF2LjnP
z2pnH=l_!RDh+2@NZvP>JaSQdKnR$)Os)5Z0SY}n`z(ntLh$C9M&R!Y^HK1kftWy3r
zdjDr&q_9_6A(cqEs9hli{xHJ1i~gDI&!n+6e?V=&yJa=Pp|`ZBui2kw<d{~L{d$`!
zx|A-HcjaNf0JsWS0z72gi~K~W0$V97g|Je>`*0cSv3J!<u3+FMvw^A^k1;9B8~KQ7
zSmLKOU%UHBzsP<x?1F)hF!*YjnG>0$k*R?YyeocajUDKJT?$(D1*xUr+0ugRM*s@g
z8$VzxuIV)<P>%ti4dY>~b+QLSc1+yXT)cPcA56;^wf|xcC9aR#J(i&0<(L%*2?%jt
zkOTVA)c+=l*N*N-?6Q3Mu9^pk-@oo|8%oxaHBvs{c0%>71q=@F{dJ+o!|<BoM~<N_
z(e$=mvWEDt8y8VuKIRrO0?{=gXa^rrI>ABM<#uiFw>*v2zctoqYck(uu?NdsnRnYY
zPdHbbLEry+%4}}Wom!f@Hgb)%*ktnK`KW0Tf2)pN7M|{bLK<TWkBf|9J7)F{Ys7P6
zK*8^WK<D_{NSmIepmLd*XhhfnN*kzu=-a#B>v|KJJvDxjz@`;mOS~WW@hMoms#zwb
z4u+vsS--1#5U|;7JXOKV6vfwzhsd1$(Tr2LhS&SiH5251Lv2@Dp#|Uc3sDE+@<&JE
zgNRKT5(05Yj#Slp@`x;GJC$jYoT*^xYAt)khYHgyhF>;~BQ%8iMJ9$dX~;1XN91Hh
zbA$_qFc$~!1%rJlD7exdimc;rRZfHh5}@mx{=PQLkquCGLxb++IGB+oG06+nIY8Sj
z`sF6%zI8QK{=4QUt2VePuC*7YqjqYUM1F1J1WbP3<Bi;@)5b2B^<M!ylNqLhIw<(p
zt)7jidWMpd_>D}HwQMP+1DX<i&p)FZ!QaKRV*LP`@9LhI7Wv^Nzg9=4I};G{=GJ?4
z#eF3B?|zryr(H^+F~B%v&~|%NYzwC492?7Hzt2EBD4G$naMP5%-!g=TvE1s~d2Zll
z{%=(66s6-*EF~1{uL?1;fK7(*8#nAk!Gu?O;<u@hYHH3f0R~>$d=wqqI0fuyvj5{c
z)ZA_5FnUY>?6o6a{s;0%Az$L2y-Ak`$~N&b`csmIxD>jU&CRuK)ZFQxOHqP%FQ2s1
zn}9t_b++`bqI$W8c0&5OE?x_XR9aiS-|r<HReWl&j_!&z%%C??{F56Ij!2^viq1-2
z*EOE?n?e3?pUzD%E{>|0`72{6^$ynXfwfu#^#@2gMiP5|saqv~X6TOPb%Hw2MxD27
zpy$`CoYW6BNFe9YjE}N`DN2*luo#XV`P(<BI`{}xTg7T|yATYu$MpDSH;9s}MvJL`
zm;WLn;(j?*84wE`#5M<1;4;Of`n}Rzl*r=<rV$z|@3Eut+~Nf3EPlE_i=G)RFF-T_
zzVE;lT*v<SMI8QxNPI^EAw~b2H~2uX+NC||@dgN*BW|L<E{vQ~#+ia*S{4H$>G0(n
zO<D5*w3p8_fDS5z8CBogcX`O4=`6QhRt4jsV&j=k1Zh7uPSh0*o{^OW{6*}<Mp%4P
zbI_-A<C*HX{9Ju1_x-OREx0-<<q@#Qi9VLALN0lwI4UhwPVod6t0$S?V;Ri648}83
zwu~=ttdBh7$BA>rA7<Da8vN31c<s%JNf7`te}Ul|@^}_&FU;7W<#9#IVSYTpg!dc}
zyN7zrpg)<5>@&W{;-Gd4>0sFFSC@V+X=^39Xzl)C@v0-jc`P<^JMejC@{3j7%7EE{
z&7@~_55o&Lf+J5kYF!4ya=2w#J?}uN3N`fFsDA}ZPW~~h<$%qMVBm3|uZH20Pxn$G
z1UQ_T)Z7;IbT1=C7b<lmEy=btvfdGIChJp7{rq-x)~-8FE@RJsG}3>6AV+r8U{P9#
z55bZ{x1P>C$)eLN1$4(Vll~JkqA6W(Iyw>4S1oEODAl|4OQ2>}J7LGJojeC!%ydC*
zd=~JeiF3`8N$H6V1a2MtW>h)WS#jAf?1THx@%hvvl)8tijfKF2qngRZmU-<~HKkN$
zRU(@;(7s<J1<{ZDoXWx*qS$!+%lF0(W@!LN1y!zwc<pYLwf_UPf5tx#@o7U~JlANJ
z8<Pif9@ce3Idg3ttzUxD1lh-|I$D$)&9t;`eU;6+;<!p<k)PGD1#-`tjB_0ggKJr(
z`jQR);E1x!KtPX&)=qhyZKY?%^bLdN4NGW&N0_J3L8zMWv{R^4X8;a;{qhwLJs}%p
znd!W6tCzQ`mJCN27+9dTmz$L`m(2yjU{8>J0p%$9FlDT-e)pTm?mmMnN!+w{-9tUA
z2Z+7n4~IV%0!q3h3A_ESaBBPHn#75FRJ=xd9OutZW-m7ZC<JM#`%+~3aI@s=idchb
zjzz?20Y2^kyV?W09MrXBqov~?UMF(T%{_Th$PIOFgp1MB_!2cRP#+SdI4+7mVuF>y
zMVO2#J8||^#S==Q!%_`4F`)`+d}S(i_$TUtA&hPeYU@BFPGcqRt<BN~1->{AGhWIl
z2*LUiOv77?sg1%+V5?QST+>eTlfC0?IE>flJ_2?psKq9RuAg{+&;KH88khv?{VT?5
zI}$x0J5_%B4e<J&ok{D-83vXij6qylidaehcG_ggw$+xHPLN=3A|MWT#MShFVfe`;
zAE6Blr=H2;W~BQ>RV(Z;hw%FA{%uu4kA6Jwkg|x`_kpUtp#d{`b!@IUn+dV)8NOPj
zid&IQ?$mL@SZla|0O2M`=)(g;vAq~Sx?o6xFD?UioM=7JSZAO%XR5eNW@ljCu&t>W
zcyZ<fS=*eO_Ax_>HiTW2aA&Z|@7A4me>iaQqbQ58|6?Zk#X7dhz)Z0*&?4)|6x1*@
z`iSsjn2$l^2@j6|pMg!Rs@Sd7lD-%V1b^BG@x%N8G7l_p3)Jkj1a^b}ulM55XxV49
zEdMiF_P+!z`&=IQTplR;9Ps|H1iVT<tJHtsV^b=PGJkAe%DlHg0hk*MQBo8jwVxbT
zMYkQTuw4S~Q!@?Fy0>o+isrM&MPa=Q5znfF=NaWq{$sqH+&bE~3)y_<;0>-SfMvi)
zs@h06porqsK<tI-N!acJsHU40a9-nZ)0*tMbfAUAs~RvXGP$pJd_0p@Ty<(7hdoCs
zzj6n7*`cMUo=9-pBV5!>Ca#F<SIhT2u|OKq+zq6e;h=uMV*wnx_EhZ%>(tBjv>7V7
zI8KEYj(MT-;rSR+Re`ax#K>syyf$Mwt{jGC-BIU(8tIbP0NT}2*b0qF?=5W|P6OzZ
z_No6!b9W@T{HeXKGi7TMcm2mCuhIhm%gcmhiO<Z;VbPv1MCXfQ#eL)c@jaDfW_4F^
z4x(mNO%$s-EZ&PNIg`$BuNTgl4KDH;J$q3S-(Fg^Gwb66Klv*|O#u;$S58Dl_0QYN
zhKfsL@*?gAH%yh+$7}0k3$7?(E|)2@aDV$(@0Qc!Xu6KG(S^3$-6vEe<75te-emAT
zFbFSj95`0g_~gH-=rM=iE?B{MsJ3%`rTZJ8q<BpUY=sX#L7&C6aVxAL;B0<D)eep6
zgqxu!lmSV&jttD)o$)6R|Gi)24fkr{tdVsB$wb-mwK7FvSZGww`A%g{bZ{Fi22Qqq
zsyK^GPX;ctid@?~C&)l?^i(svQ(CF_u!6EGSLoTNrsN4>z1eY+d&)kP2<ZRwP*;yv
zZd&|r4mvSD>CHl{4uP%W14M$((t}l7zXWpG9sjv?O-)ZLOX9Yp(X+NfV44zTE#STU
zshg_pVMh77d<kXiU-lp88t*-$lvEnUb;TGas~ls3|EP~sVPkSQMq^hzbdsQ*K?%$5
zN*51u07@>G11Is|`d*!SOh4Kz=n+Gk^Zm`TVv)b-<cNgE6=(x?Eoc?BH?K_B@x|{8
z{oEp3r7v09w)T+(t*YaVdmT>%j49Cg>L0!&<~aDgwLsCM+;nKbvwG#>P6g^cqDn#)
zlUF?eY67P+RAxHFym_2$b8Q?@$}=8(J;&Lj!(?m`GIR}wOYZSDUh7mfFfnd{f{Q4$
zz}82^#y5>b`u>I%y)U0EDBnHe3b@*cs*J;2=L@}y0FV9OCf&zfuJTQ~o0DnW>rDFE
zPTGbg83bNNR{*UwhLMF~zu%F1{-X8bRXf6qPrHlI9ekXlQH`o&to20wMN!=8urnV{
z;$_W0Hj?mMj#H}PqVyt5WH1foXViU7w%X!^3J~Sm*ms8QczicNOK-^A>iYT&gFr($
zAPO1$3mJ`~%sz>QC~hvrcv+<$4_40lj<{?_n=$)izegubOuVIAX;IgI_`4iBr8HXk
zc%Q)LTi30j+E9*b)HMIK$qTi%y>n`;Si(V_sG_u0BRASd?W#EQYTyHsbZ#LhMdwt5
zniIh_Kkp|YXCEPt2uIAr0Ly*%Ke?R4evv^B=|=l)AXZj)>BgC%x9I0b{WBP)3Eyx7
z`c`Uum*5JMMa>A&i{%9*LX0BF#R4#>CG{d#W?F3QgL(k!#Y;9O$<NxH*&OG05^6P*
z5~wdhUZA%WxmBi$yrsyE*|M)>Pzl+>#~7AV8}Y^`{<^5in|a@Z(li(7tD9B_juNz%
zZ?hULer3EK$F>l*XvaI9z3B)wj9f#h=?7F_QRp^{Q6Th=zSxjefG*W<VdFlGE#z7`
zd{b>=ktzSuL#rFJ3tDv_SP*<8qGOa6@)-`!rHns-rrNM?8C}yFeNrEq`+ZjM9{Ra&
zaLIUOH9CovtFZ!DZD;uv3fXI|YUz$9HF1&Ij55x>x&r=fY1YM`;Oij&3fE)s&>dno
zs5QdTB6Bv}AyG(uc?WZRwLv7XbFo<^k#?L)S}7z<#YDCoB?ERjqG;ZxNmw*O)n189
z1ADK7U}lW%4c$**7lN8$u_{<3C1ZhMr(%3|$Ocg9uvBG8C+a)lBp{lniDqzC3ms?|
z_o0qs`^sh3vqjpAf#t|GDU+gHrJUsxb4RLzOIY^AXAZruLeH*cu$Dx=2=A%B8+hJ~
z^=nP>kI<Yy)O>L%^gn!YJ(uw#M*<1m6$Eep@bD=3t!D2{I|-RZ{@lLox5AI;<4W)J
z7}_Qm9pJ(t6UXLM`ZLeW%vSRapcc}eo|A1|b5rT1(~dO+wnsj}s2E&i3GyeKzL?bl
zO9HzL0?}Mzl&m8gh<I@|5?FR8BjZl!#H`BpUr61=SZb07J4zG%z8^?gMl=sZfHN%M
z8Oy*T|4Hn}C6PtXpsZ8be_U0eYV?xVY-C<@b;Lx;hbj<B<GC$^BsXI0bvTx^S3ecd
z^YCt_fY$q_XBhq)7v(khma-Mm;t-)M-Oogv2#o6E!Jsf=^!C(7P6ooZn{v@+ZhKzh
zKTv|^ttNTGM4%J`CstgL6ItChf_U`~*PnV3Ak<%Cg0St}Qi5W3TJ>|)0(naAav-|j
z1+#Z?FVA;S<sO>jfV1SSkDoZ?UylKA5oF3sFkEJt%;9B!L@&lXPw)UX+hs}T2m7o^
z)2zMAjs%+7kD8O}hpXIrszFmxUaDT$8lRlpK9;|8pVo#_BG1v96!Yrn_PLQSN;K7k
zXbO@+`PovOLH`BNBqKY|8=d(hE{=ya+yf1=#1xR<_@^#N0b~Q&+<SDorAI9MJ-ojJ
zc;}+Wch8LE6En6jo6eR-_xwUB-t7o)1*_;81zn2yHf8Y_yz@AH&dQ|m%YItrAw*x*
zdDTHchSire6=9}N)qTVoUf@Gg*iF(RdDTuy#-ohDDAedUqJcRX^b0(AMlaDTv-D)j
zexA)q`Zz_mZ=Rdbw=vOjS~sW@hc&b^wfrGW&c+u@Y4KcFZD9R|>7cq%v>b)zK`MW7
z7gv~bCI?`#`w1ivA3*nsN49UZA<km+l&aqrc2#J1>G{&{wy#Nj5fK~;e>HCSs%DIR
zIG{B8{p0MKm9=;DJD>fLwd)4YvWvJ$5}m^gi4{)_+q%X1G1fq1W5-3$pFv}jrDdAV
z#MtRedq8Zhj?V7!Jyl>!IHV1EB}%K2{cJB84&3SXjD4q>#M6Wd6$~=Of!Szz^E?3%
z#-(Qqx%Rmo04`PZOibj!(}wwTc5^Qyicg$TZ61wt7^wwaH58g^v7J+yAo%J7w+v&+
z7lf;d!2{oL=%c|c%I2Kj@u(T)Yop7QApk29)VVNFSPBoPDK6cxc|6df9r_O+_%7{o
zV=I`Qi+$Z`wZQ2g?K3Q^Y%w)YJPR)4K}Ax=fM2TRFW31(v&GD|JZP^hD0n#f6`}>W
zn^3j$F1Uy%Dn$`~>E*CgqsTQ(8_INRcutmjc9m5t=q6Hp+K!v%2qCcYQcBh#$UCPr
z5&}`Gj^Q2Y5pw`q*!FE*oyP3la6cWv52KiF@o-JA+MT5XwA)w{8!3#=4Z&>f(a%=R
zC0pCWC;6qV(Y^^AFRTWbHmE~~lWiLmW%}8gfxS!Vaodr{Rv%l4c*0bd$YKw?vhT4P
z`DVM_=)1ogn@If-)kj`>a3mvIqpZbW`qOyVx9wPDzB9bV<$~61_!;})b97QuowzI2
zAac#MsRq4WT|Dm;)+X#XOZ3nkLuE}Mzi}4U-V-!WHHS~J?;J^?YJ_|1_C&+a+pPCt
zk6zj{E8F?ZI6crxD48!p@ZkgX%BaWvKAmz0AdLt#D*$SZ$|+cKrY>>hhw8pK0~D3+
zy&S@tX0tX}34)EzCTs&jGF0e{2*QuaQ&o1>G1X#4O=eFfiAx=8KU~;{>f9RParf7S
z-o2dwDgiTBS6V2lvol8@z_D~k*|zGai3hf8yR7m^^ByAA$Cg`9V{s+t5|-4USG<rl
zaB6=CrXh9p6wuIwt*Ll&J^tTBaM0i~y4HA`Dt1LOPG>Jhl2=?)*_PFPFlr|+Hr~nO
z=92XQByZJe;H-o!iviSZg2UIDDw<}%*==zyn-cU~`9u|3S*jC<Ug4eqty<~6RdGLZ
zf%}iMi+_3c^(Pj3?Z-+J3?1d;)Z)s@E`zT^4<O1f`3f@TuY%9dq}RvI1VV#)2&0V~
zhe3M*%-JQ&{>=fu&#un0>&$v==@>W1^InT-fUBhO!;)lhJAxVb31)AVzqiywPe<Kb
zPtb6aJavepEEKVGl}Osd6U`Vuhpj_?Z~sjMFI5u%RTMIQ0_$0xf;DbN{V+GZi++85
zAxUurv~*JF((6D=CqwI*;{xK<DIvHbrdup^f=e3vXXfB>?z`OppZ+Jmo04g4+CNl2
zHa?n(9S9NGBtc+~YdTo?*wY?|U@1qOD8ehWM!KL^ONYYJgiwqvo!&2WOdg@n#cgy(
z+Ra6ae*XQy&%2KXyh2w|#?N-RnA?ty59bjQcyxj_^m61_Z;j>JkhE(MyZHgFUeFKj
z;_-h2>E7lQ7JKYAo2T=4?s_MUr~&SiZ?l*-Ld>^+%8t)}#l<}fOy_4*{U6<dKi3XE
zLuvn4ptSi7`dL7+=(9}ZvlwR4|7w-d{}5HreUSnM4c%V5M@`d)&H6A#31w4B#Z#xP
z#VY%N<O<UN7-T;x;zo2T{WkWH5dQlCfkIWj(uWd|Ou77_n7Zz7joN&KI`MwauchG;
zOH+dexqCqlOhX)(>IkP7^7`(2?+?4>1j)-^W82v*m;WB5?5mEx`#<4(uL{+!<0|C3
zE*X7HHD=rGV)6g%m)XA_v{h$dGg~GBby&?0Z)!h(LG9yBs{t}sakk(phdTzyUGp6?
zvu1$;)7ltY#jz~Vi+p_hKQuLGErqd>Gn#a>=2Qo4^eSkcLKR2?;U9bGf-CL1)`*_(
zKk1Fx-`qFmIY5^5EjNaq_pvFPZjH$!_AY}}2l+(hPXl5sAx`NONTbZ95|_#azzkG>
zy4NUypo}p{#`*g;HZ&6quFzs~;(UY{E2<KXRvoZWGrDIQ=qr)FC*D>l$;A=ZDPIAM
zJy<Mdk0xtue82(Nk{pBF8ye-k=}hPSe>VJqL4&e01tV9dU@aK3fihI|dZmakrXP*O
zegdZNVzxAy)fJu%8(80~7pZmurTeb=i(vdKea3YeA8s}_3#*+Fq#QOQO16#ij9l34
zWjYCj7!v$yxT3vDyi5iUwAtLM=FaUkpoAumXqT>VXJVGcm~~`g^#9p!u-Oy&T;B*s
zv0q}Vm^_4il^f!XXyh#>a{yQD+VB)=fR@>#iXS&^(lYg`_cV@-5>K#rmW%cO0><w>
z2uD!P(#LE@3vsAuGR@-{4wPhCcL2(BWY>M;7YWD&HFerFy5ahf21)@=&~;1ywhh^c
zT!o`@=^6u@?;62204s~%F3=JTPyX3mkr6G^uB#tUT<-@IoiUzyFn?=0VCcl70scKO
z%*dlbNZNfYI2I7E=a%`s$`lwO2^EqWoAq-CtG9<yRpB;*Vd}?DGpi28PB(_?=fJNX
znp&zh;T-D89`$P(6LK*oU|Ic>{fJyKO|XWYr#icxNir$SD5$BpuJi0f$S1}sN2m}D
z&9yWB2Q<EHS@P8;qr7pYE?g+~_ot}_4=)$S8tmUL^&k>-p&R^$?_W7<SRSI&A~<UB
z%`Xg1&1O%}ke}8=M$MI@9>j*hj+)LXy_2BhW=D9bPiPAQO>=!D3y%w)$l?|Hx6I!n
z`VAw+2+`NlzY%Hi08nQ*0sTFhS}bO4PnOi77){-Ba`g5ya@HY-7s{X$pz+(s28)&@
z8$m<L2i8~3M}KwryNS{kWe?~kdVmNAWiC-U!>pB)13ZQ;3S8v{lBt%B*|i7<&_fM=
zOUH3$9eS$$xXYbDzjuSKYuO|}<yj8IaL*b;LbZvWxlIiH<V@5|mvsNk<NdQ;A83=U
z(nmabc%)Y85?ih!NKj0TD&*0Tvt-ium$;~`?sx{%Ps?JP-KhDQ{{EoCIZKrp$zcE9
zA|FAQ=5}=Q+)R##tYv${U4gRLyU4^kJa77Hg5llhEBn>hMRi{Jj#%u>aziz&%PV_t
zd=xC=zbDYy5?eLEqVQs$O#^w2P<+S|sh_U@0bp%ARB|4aii0XUk5<)Z_(e<;<rZ3_
zpixwfZQL|!Hv7pY>77gt?d4Ah>jV!!OCGOqyChqr_F+%zu{1c?VaLz%JNe;c(s?)S
zOS={?2gr4YjjVZQR?gQID#Kc9wv#u_{rIT7tSgGpOr@s~9yNU$TJ2~DZ2TJriYfXp
zj9ypS_bk<kJMrQ2w}pV{skOC7rd**@iHl!>%)Zh~$zZOAe?pv|%8I)z2uWyHv4@(@
zZ;=>b3)#u8l(^6$Ae?_C=fhOXFp9F~aw+5otKyVR)749+-{ZB{TCgqh1L6*s#^&lR
z*0}D$vj`0)5%7i9ePgFkV>xFo@DOq48#)MUZ>0GG05@vXW31I5XUyDDv;!3@uywXi
z=8RShS2<{EGUJBHbGRAk(|17Kql(FjYz1{VJLTm`RHvsndUdcCL|k4MKQM7fd#k1)
z@CIl-dDrOne=oK!9espw1hmBreL>}0(_|qahS`Syx-O;Ma@0H|16LBp!PE^IWHhh&
zS6YOlsN=4TMWVkgfOxxEg|9hvq2dS6t<L~ye&>lDm4ckwD~QEQkI2GDdZO`kk|G@?
z{4##!+{lOZDgaS9v#+NVrYk=(=&sgYC`E^Lo=MOcDfEUB@r-4Vs<ryXW~z4kk-wu1
z-&NgvbSN?Owog>WGhOY>%L!_OSFS3{_{vbB@ajJ7o1&GtXTP{ueKgS@b4ihdsnc}U
z@!rNXr)#eHVeuEhM<e7a4!TwmG;1$H+NV7s8TqGMh>PDB@3m*D(Pp|{Vfdt%iRAQ$
zsV9(qnm<ssuAIl~`ne-&lcSoRF1iyfQ1=|rNo}DnONn2>_Bs74(4TxF6HqfOb@!!!
zQm2OA98Gh1#O)Y3e+6zZnhCn14#ev?ehrJAI(Fpa+H{r;amVO4cw^KXV+;Cd)z8g#
zgy_H~MbHIujw;(IDlNi=YiBJsKlAHpKnW}jbf_%ihzxX03zCYMXUSZLBYO=%V6Eu5
ze)+~ey4Kv_78T|ciyjX*1)GYCuYWTiaR7T19&BP@MWnTX=7vL^UaIl+bC&ZW(pgyR
zr$4`_7I9%e47CFo8dIKXThWarVX_BA3ZJ(VXzKqKv81<F8(H_}16QnOS@qSywbK@k
zk}Uulb-Ib&+Co;T*nJA|i*S7G;ufcEU)mc#@;Ixv#^bE#I@r3$);<?^zf6FixSp@O
zvuDjtgXITre|I5x`QeYy?|Z)f<NM{dW<R#ve>326<F-G#elp?<sJ)h@eb&GZ@iqa$
zkGpC~OGJzJ_wL&vM?e1IMyc2t#U3;|MzDyAoe>T7@M7IWO7h^IePV}s2P5C`NhheG
zJ@gk1isAN(?pk=4f_H9HxfpNw8#EZyrb&_@Wy~zp9eEKdbQN$)_Lv1bz@#;r{`;;^
z^&$?f`ra4&Az(7&j^W7_!CsBku$6_yTSVEiBW93oz3Z!fcK;EAmVQhfu8r4dhnVG`
zD$d3ZQ>%$3)NzYIs;sf<tE|us-i<i_$2T3TB3%(TzDz9IKFb=NN=}e78fZW62Hh#A
z`vztiP+~{MalaG4M!w<Md4t*`Zeqi>Ez%hlIll!yIdh)w$249WTb~!tKN&}#o96z-
zQ>O!W1@gB?-F#JX&KwhI%^$^O#o)LORv!v}4(~Izl3;xF@w|O+mxWF<xB(L%PVc*Z
z<gNB=#-Qa^$nzh%D*lY7_z6eD_<2K)JL6$!KBIwDN|lRIWHInr9`R1?W!;OrJy48_
z?5}zJ&AV54!<Hq&W9Q?g2p3I`({-%i2;qn~`$1r0+-e`;&j-$yXqqFuE!d2o_3po|
ztJ+=Q6`5v(osF49u8w7{WLh}pqPE*nNQ8L|D3#W@yX)zVMIu2(Tt$^*M&pzfL%EEO
zvxz|~vVQ*wV^WA$%Cnpf-K0;PtfZ7rF5cEQz!{N;jy+rs)q}C_M`cv|`C{+>tbcr`
zu$yH@&MJGZI6ax#5so6qm`&v-lln^|9K(2b3~NC3Pdqch8TpINoMvfi;pKTK(@i#P
z!<P3CZYIk=##Q>z20Q$fGc!Ms<pB=8)wWUW(te#3x7h&M^bnaWe9@*HV0o`fmQ@Wa
zlSceiU565lk|$h!uOhb{y!j*KJGT^O!RWg`&v6cN$(b{ctke1B8me)n?Hk$%$zkQN
z!d%#4o!VVpqIQ|j?qD+WbLOUS(WyT;7mFkErW@)R9x=sG+03<aS%8HxVuR`WEO||)
z;Fxc`d3kwTZRSY-uh6lcwT4y7s3`~;r+QauJ(Gph+q~tU&(6JOtr5VlV*R%FHlGbW
zfW^2AWp94=#e%jZxpAru)wI7l8VLu7+Ebn@6!?o4sVh;|47{+Mk#bxls0zD1Enqv7
z1DT9jx<ZP{Ld)&O6wlMGLSRHcH2I>aN;W*eiNAQUB}-UPP@F!kdw2bMuuaNprR0^L
z?2_+9yu)@gz7~bY%ZX2TvtzOVp7eo!msebhFP2y{yTt(|emwhcgKQ5=+Zmz>(U6vW
zD7P`t9kd&M9=?(Bl}SUx@DRp~Q7ejn54Cq~2+cP}4_;sm%uey!w_arLuLQ~Ea0-dE
z(`MdeauGN1!Z^&$2a;i!>inPiND!>OYnbg2<F1&mI4>ICac+TsHdo{XjY0*VKm0rh
z>b?1SC43%)`566qC4629pH~9s;QvQQB(UR{8HcMaIi_Iq!~C-0USloBA$sgwq}NUt
zpVE@OhaMztO>aw9A#Y{NOYT?+wUq-3k!z)j>e|(d!6&N<v}dkqHEClMd>CVj&&>a;
z9($>&XrV7U-B;}lK@#%<nMJQj;m|Rq@S)ud=XR4eRSlpFx8^vt&-*SeI`tLC=ute(
zicvpSD(#u-Mz$%&$l}=wJHM~Y$^%Qu8%q>VQKD#V!owta%~)!C;xi6g!Vu_482C)`
z2m&?CMd}BLQWY21crjZUNfXj-nMrw0?Nw~TCJoipU|9|;ym41F?XC9q>sCc)Z>*93
zUdc?vcsM^onvBV(CsrG6YSOj^vQZacrET|DmeUemL9hzdD6PoqLv;-ktx`P6qs4cX
zib|Adk5hmUMyfD>(^En{Fruo?Mjas>Zc363H^L*Le5XqGngt7B_p~OHGYsU+6lStt
z!^VHy1ytIMl&Uh=3hNFAx5Kqd8nCrj74jjKX<m+OYq0CWy#?c=l=1>%W65?SoD$3w
zLk!rh54R%ob=MZ3qa~R++A3J)1fO1_58110#<lx4U9t-n^iVu19A?NV`sIHVX)=`*
zN?sF5HaF2;aZdiFr0&e&11g)3C#j7CyStQ__8>t=B#KqXUWaS?iY<KPyJc|$CB#*C
z4fnQTa&~WiGXK$(v4p|5=HwrAqL(vJN-Dp*JK(iZ!;_Tb-NPN@it@M3TQV1VLAGml
zsEg|Nk>1RZqY9~=A}00&FTRHzqY8dvnsz+$jTF!X8Jq@OehySySUPIH#SWdJR2HD_
z^94SncT%efe0JP%;r(yE$<8mCXFN1B*?L-I#KOE@c0O|xndO#wY{xxmx|cdtFX}zy
zNl!gzaFg!4A$aij{YN8hY=7}=w<T)D85P=_Zq+Q2@}DvzvZiZ{$w)=`SjRWQ11a$;
z)MQudXvNuo+-^Ko4Z{%Y9u{8l)s7?AI(uMGha*^7;P(QccX+J3yuHe_-0H3NGo8>k
z-bGmxH8uGPu|%cs!?{yy;`(dk+)!U4%|h90Y25<H15WLM5oYE2Qg5mSX~KJL2a+MZ
zb2;VWICaRr1F1tdO8NGVVO)2S-*tTp;JATEi&WX3%#m<zNc@I`56cE-$=Kn0@4jAe
zV@U!(SlJ$X0XqiGxOm}~k*fh>tjj$^vMakOh*d&0hs8;KQRDr)r&^j0vEYur8ou*v
z=(Nb&(@M*X^4{5K<x9-WFV`69UZGXBPF4oFy(1xlrk))i248m!xoWnbF<E`bu$~ta
z98;?~5oAgl>hi}wjCaaudtFF4X#eu`!-clHB(SlB+Rtp8ezhaf(<7C+#TT0uS|i;=
z227xn1Ct&zYvKX>d5J3er)&$)C69@=s89!nFw>U%Ab$Ef!r1rpEzInR8*XiR$N6Wb
zyWUakin3Xf*AIjjC(CBbT#d?`k(M^+kFH5;zQI23NI(JHe>V$GihYV_AAJUYFj{<$
z_+<_~TOnCZF<NEYBJJZUWou4|=lf1Mu9xmu!ECE%L`x#pM3s5d290v#H)dNsyE%dH
z2@TFm<rPM^xpkLg37D<H<TZxX<1N~UqSIgGJ9*p7h0AoOWFaS^3RA1L6zEgtvW4z0
zT!mdA?&B3YNy<6Um1g6YR`Fdp$@}4`d9yzu_HK=0y8HZ{Cf`VNf<~RF>1+Q6?&V4G
zv({ovwV)X$*4R6|yw77DMj-wPHBO4g`!;FciKb48er7^FOw;6YIoVX=t#yTYOA-|B
zampRtJ0YPAIvyjs;Mx6UTREy0o}GW;J|y}?)EL6Yn9w~=31r%q`b=f_8<@_9&R1}n
zxT88ZQ@28PS@uI+W7s%H>#VO_Z4Us&i7ReQ5oSI^`~$fia?|+!GiTQ6J*vFcGf)(+
zrRC9;g+gdwIkgv(19P@5f6-R^9v|qsgi^hq@@`YkXfaATT~2uf*5IR)V;9nlnj1b|
zy>EY;V6<81UyFy1z17Y+;}13SIdeJeCm7>u$zFxLPLyqN5q2@D$xC(fW0Q81fFWgj
zL<A#LN!us-8UyWn8E%X`DnCtL^OWo(7v@zeuS;I@%qjEkt}h#Me`VdLSfGA*T_y_<
zx=eN1$Dd&)G)x1S<JB`Ehz96C1$ZF7NVUQ^sJ>l$CUm+@!4TEliG1r&`{C$<k)U^`
z!)zt#S+-SqWLt|HWO$&L&w%DX7!8|H+fe-y+Lt9!9dOi4p^kW+ySC&NLZcO-!I_G9
za5>z(x`n%X<!MpE#1Xq+BhwTix@a@7m*tiT-0yT(pL{V|EO*>KYrSg)toz*uzP?@|
zCbA|^s8@9Cj?tt~QpZWL_h2Bw$4qYhTdZ+n%bbtpCEUL2CDiaP_9(sx$<Gt5U5pJt
z#?Fw)7lkab)VpGIJTi=yhkI3V7X{#w<wdCdD&B}r35Biiwa>#vTgh(>bSmZi`8Tq5
zJ8VYWH}ql$0<pFCX}0mMgJ!`hcd~TDgHiLXI%x>5)<w@BA%harHy3=d8#NThQLKS#
z(G%S-TFTF|;CEu5)^H9#Ktz|9P{liPL{P=(lRbMPyayz@zLWO;Y{+c$v6}g`z?Ar<
zFLuJ%kvJE%xi)fo{SZRvcwh?a?Nu-umX<9KJo>yMG6q6Mdy=bfro=x#)~j`zAKj>N
z;^wXekcuPq;H)kAqlln<kPi;-U`070y9bXZKjAHoJpnk*9Z^y8eU=ZCl_<=;_7Z9q
z-K#I_ek{IZFD*+L?8AaL#=d2u?lw-b3rBC!eM4#o#0E};9VB`RZ<iD$#wR7y*rLJ8
zxiCIn)1+-O+~4{*=$&lf)HrPUO=HV%Q`ftjK@4&g8tr(!kkCi*t5n)E;}f$(WPX*!
z>%<ico%KJ0)^^iI!iRWcL7oh%sT6V)7o$*aX>nbSijQ|oEoY9&CrIa51EqAwL+m(N
zaM|0BJ8e%H>0F7g4cY38{VHB~Z2K|inU_u4BIcT>Mes+$3n0dRB?*uV14hUq=1Fuc
zixm-)WrT?oa6Z%xJGu$Fv-k$r-fEX#z0t0khpEFnrWmdf)>9wfY71>rVoFm5gd)V&
zv2EzzUmxzu7rZQ<Mxgu8`(jgSCyuq=N18c!kP5@59eUYY6^j!8SZMd#4dYA#IAUG+
z9h%|;{q>In>wU5Ri1bf%(wwMl)}EI=wJOLR!_@^<Pf7a{me!K8-x8S#eL4Kihe5AL
zuQa4`Z##XpkLo4}cfu04DSiyIWUr&(VC7Exyhh=JEw*0L)8iuF8amq4V&k=w_t)FZ
zFDyrQ>Tk8@`Z_>SP}V=hBDXpVI$Te79=35(<$2~rw2&Zm6H2Zvt{PXWczW{G+XE;m
z2j7a)-D0QN;xAMVS)e?J@5xtgb6pS*+sEmCb=0GCSJkUe5oaRn3xv@nH=UO5Es*A|
zYqM%<KHRc3J+JAA^gCw^6E@p!J96)95U??~j4@`S8ew%yv*nV&tWaqQwM6!OV&7O+
zz66gK$`N=)^3yZC(WX&JI?nj)WRHd))2CRt_gMG`M(vJ#0d#H!rh2M)qGex$MTk9b
zitNiVSsh(UuyPaj^nq+gmv}*Z@r=EO*T%!Ff$?E-cq?3j(Y2Mub&(uSVie)JMyGbP
z4fa#At0Q+oOw%_JJy`cQw$|Tq=G`RFk31?*pFWPxZSj2;#JbPqKLYAMcVT7EhwtU1
z?<&els81Se5qIurNbX$UJ+pag_w=a0HwofD1ijVtEp;12oLmG|x+ZXA%~r<zs_JWa
z9bf7dw2rceHSm6?Y*%z&&Ni9w)HLL6V7p`^#YZKHii|5|i~7A+#-!u)$g6J~a_4nO
zi;T%i*UAVxWYwJ)u&;bCg&!C=*(IO)AUSEQ+987U8Ls!_?pV!vKrJhq>AD*>iOxUC
ziR<FOb1qpZ_gFEjV!68c80Mxx^K}!H_6M+ZWLtgGi0<(_Q@w&+kNe6+Z?$0<MFdJd
z6CUwbAISGw#=fokw7!rYw<-l^qF3Kl+3DuVJ_fBK<7X{mFQ<qn^QJC}xF@^NFHZR(
zFWh?ZwC<IH>}TYP@nH=Nwnv`SSIP}<X#Z%K+dI|D>zRFW{l<R8f^h&YZyL@LE2qv5
zh|s20(Y{zZRApaCBp!E~N9kXF8LHjpKJ_@koa*$G+~~U{eS$n1x;aUT46^B~f)$qD
z(KhXZJofO4j7onYir$_+{!GfL@+r_MqM!;lEd0pylD$HeRvbLez|Ajh>lTO?8D%*o
z>pg#W{*mv()pxE29fm>};*JZhxLd^T`y~@P_K1lHtycroIyg!;XsLq}e!ZiR5E+ux
z;i+%HxINvZ-4Gy@wbD_UvUt_V-G$=AK@jW77fUPexHynAC2iAv{1(NYAK@jV<L$>`
zkDPEu$Ds4SVUcP6g8$#n*6|<_X816k7cbUfOoZ>1l#zb;HzbYz+Nx&ehuFzRxfjXP
zu(LAh;>H<ysh6HX>sQ8aqvvmL)y4~*eDA!eCDc-X`cu_kcRa@>3;naX4Y&yOn*03W
z=Rr^}hR-YE|DS^pJ<j%f0HB>g(T%su^Y_mma&tYE!0+wd;c@WXR`nf!GJ4;~tH*NY
z>xTb9*V49ae11obd`YVxOx8U;JYNo*#~VO@J?FbCx(=#OPqfeBXn6bP`9khrVe1}b
z)1jro8`T9cN8;n^;`6uXi`MGJ0{*-+OnG$W#R^bo+eA2i)aQidU%=D+r+^gN_}d=O
z>i`u>p>N${R<QQ3`s!aFUu-!0<Nd;)LC)UDTXjd*J!u}7`;QaMnUdS@+&{ne=h+79
z<?|x{=iXij;y$(DZgKXPN2g84b^d}>+RkX$yrIHyPmQ+?%-`0qY+PT}GtwQULcGq<
z=r~mM(}I|H$im+o-7p-6F$wMg36(dbsh;8cR=Z*N6yJ5kr8?9uFaRMewwtD=EChmk
zI1M-Up1bQ#k3@NpIlWmAQ$5R{T3V|yr_{Yvj{ypU+Y1@?BiiBg1BY8A-2uWWDTalX
zgS=lc5LbyPJs#_0)+-O$5Uec0+=t$Dl+*W@wE(`xQ`ECd3x}N7)eg>Pgpj-#4$Pim
zO98j?=^Bbr4bl#v3&U8CsI+a6!TG#VG4l0%e6pc8JvVG=9SO)C4V&x+IWUp*cr>i9
znwTn(&+Nn>b#j9oP!O29^26eF)`C2f{F_TvorfeB22OwDi<M-45>9^{I44wofZ4|b
znAHXqNqkX1^?U*p8|E<aBKLbk!MO_Sc75o&!z_5(e^TZhA6aZh9s&KQqx5tR>i!0I
zTVjz96+0xjBW~=kl8Z;;rsnfI^c7d_HY$U}!i$CMWX8lq^{cYq+r^BQvFo<I^Vofb
zyJ6{FE5bBmhQqFQjp0;PN!qx?((-dQnX9JTW^uCNsZ2RH<)Ro3Dde@)jg7sBjg%xa
zlw9#maQ9Mt8s87vpiq&r)M$?rkv0H440SPr&TK5V4wrY=((~f3`yArqpw(62_6Qk7
zE2fyk6P$+j)~qew-#vCzz3m;LSomUh-kxXfu6qQ{ewf5aXZdweUgj9Ox2mF#o~npI
zZs@gDG~4=O!@uC1JjqVtMyYRzPtoJ_-dz!|`pKTh(&5378WFWIf9zuZN$8eC?+sOz
zwp-jZTVllCKSHX7pPZG>OON<s_4NU1r0Kb5jO)7xCDcYy;c%7RdrYlX^LJZ*KT|2F
zF4^l!WF87-*A(_m{jt^SyH3jN@Nk#{|5kgK6KnoJ{0#*l)#i+7=ht2oVeZrbPYmEy
z)yLmoXO&PZE`^(nDSgQ9h)FhfB3kvro3>0NkZ!ohD7O~_I!4|_L9<2IX38~&2Lhr`
z$A+2u<K#o<Sp#MmX;a(vKEg`%F>2S4_}B67ywr)}=q$l>y2L1Qo(4qR&Mx>ha!6$p
zyFA?!szJH8gyQTJ{46UbV3Mn2{5wDvsDHD@pr_wzJOB0<QeWEkve)DZIh+Nz6!IU%
z(3cQ)np*6FMXqtITaSwEV##-8ZA`|i_n<4zQ}zrhC$^*-CS1RvHTAPF(^;Jv3>ldH
z`{?d1KFETPCtf%++lf^rg}91#2D=zo$X{0~Jg0JWUVRFM|M_%Dr|psW(Z=6)k5#xS
zg~A^VqIr^=qJUXh>4^Wz3x9lXAL5s^h1660HhSOdYx{%*`;fY3skWp486Q24!Bknj
zkZ^ocnh14b$<mst$63V_ugCqVLNuViHMsi3h{??m$|#<WfhrWLO^%4PM#atPot}Ro
z;pg7XBoAd)1*lSK&%<RXOK4Sghae;uBQ(xM##X*AzGRNle3jDL+d<z7c^KFA#K;pP
zIJ3=4Vip!>#)zDHoEUZ2=EB^*kE<^%6i<rxj~MR<yYq4Xt(Ar_w*|&!U{tqxv*@6v
zKV=<awPHR0LE>%01Sx14)mF}q^DMkTx|u*@_+C-uAOFx+JNV94r;Z>0@zmp3tD{7p
zHEZGo;(mw@iTJQ_6&0?0w}rP#tM+(xQLCjh!O5|Fme12Gw2Z~;!v<AFH#7j0cMA%Q
zQv?-`%ic2WLTUGI{>e=5%8~cJhY==Cc~H6auaMMqa-s^aNmG>1M+vQbwxz?{DWSwv
z`O6G+x{W+>X5C~UV6^>a7kzA%R#D+2TAt!!mQ!@74x<*w&zbyL*!R2Bv1X*4&mB^f
zzhE>)Z>UR&>#A{(%llp#Oi+J>Xd+r4P|el3=L1CbxxbRbi_~SlSdQYkZFr9yb}Qdu
zZBC-;Nl*>zW-IT;0cPX*vZi%L&Te|CUye{eo*VKiJZ|>WHj(Y#c5{a<-!*F%AGal@
zUzWbyBGg%C;9>N^%;6NS<x#HBsW)tPp2}Kox~XYVnJ}g{dOx&@=ZKL1j?g^xjis)m
z%65eRTD6Irx^6Ch-}9hCemE*B>BEc1Cw8M^FJ>j<a11!cY^;ZdjIOxbvD#4p&*nU*
z*ls!EOvV7{`Q1b!{&{$g9pxm7?cf}v8*s|N@yT0V@^dL)9<xUbxR0?d@9`TP3qa{f
z_eVR;|K70H1;0A4QJro+9BE^ww##bEv<`iG40^V`IJ)&xRBeCEt}BlEKDdN;rO2fx
z&VS>;EBb<W*YK_lalCU?AGEaZ3W;a!rtI1G8iyL@h#jbO;=#pI*^`d94cqPLwQf!^
z4ih9%GAzm^aoH-?S9Iu3yN0f4t}yJp@O<E0I{#C?Yuf}IO4e@JhaGODOyZ3}ZQuqy
z(MBqywi<pH5}KzRH2m)FHi&`1!I{7Ot+tyFF=|gJ%pDoHrdkDy2@qTym-Ie&lxOvk
zpIb-783Qmm?O38VRh^?>Z<%=T*WH1k^;Y{^U#y2eb#QjPVukiyrxEJpld?MJ+lG^s
zyDN`Hp745qf;4LG6M8$$1_mi-&~3K=cvOf7hSMSg6&eh6HPh`F??v&^u=^t8Mm8@f
z7pITsyei9Hy^WhSU&z`v{Bc^mEhwwXjjJQw5Iqika}akGBik8_cXU^}ccBi)uvhqE
zsd<v8prd5on1VSxLm$yluUt3%;n`ssD9`2l(gPk;gwNyd_GiMpES-90+!P((8vC&Z
z)~l~2EAgCte_fYy_1!mbG8lUlv$CP^x5yJ0Gc!zq<Lw*qxMEkzSC<|g`hus~m^TPH
zsA=JOsiMahd%uxy>|1GX?%{lG*_Sy8TL*f0G<+i3m1w&&l(JfIrr4>}^n!BnN~9#M
zC$%-^Jnu+V%&X>;b4xSqizKk=pkrBgSG!9&11Yt<<G9N+YW&?2?8f@967Ow3Y^$y!
zpa6i|-%~?{cVW!!2!Z7q4Q4Zt{_w`rvB>cvJ74T?pG-=A&s7kDoL%LKek@KU{{aBs
zfImU3Y^=O}OX@MBySSZ3e#Bq5oIs63g@?bcT^B3J*qU`bYIK>9;1OEa#x%$<b#N})
zVKmM*#PeqSIp3>B&*-lPe=XfqgzTGGP7LVlYu^>E<do(55UR6VRW+SPbMNzi<{+(>
zR(e~1^>s1ppSjj+dHdf3n=>mS3jA(9Eckxz#pi#LqI_EFz<Tec)bqQAkJeNe*619a
zypx;n&&KKRzP4qrTROMy>n#xs|4fnPyZp|6c|Y&_8wGpjo2mEhEPESwD|+prusg1&
z`0m$y_O1H($aU)>_XT!m>aGVlrr3M{9+okw{Y`N57t7=O&Tl&Whw*=;<C9pnw~0F!
zSnb!2=drK;l*b$DnegG)@~=z>?tGTv*mu10Y18NZvn}TvzGQlCAGNzKz3Y$c`FYQO
z{J5OGdOttdhqsWHe4o8noA@T@-?lq*W#g9D-=8<<u<p!GYrmfNUK>B?iKHKG{`tIg
zQry3a<yA7rEM416=6sV6>E9WldAr%Ta{nU6y33d;Pt$tNl4kj3{cl})H;3O|9G_*m
zy(}o=+^Y@1X+)$+?kwflm3M#DKU@DIT$7dOjC31+yuI|6eztp`<q{3P-m9FyA6Snb
zUsn?Q@qlgoYXfg!H9kN3_nC`UUlVJ;%CbJM__lL1^S?ivS6B$HH?H-K+~TK~%2T|}
z{qrdfaf|;8))bcI6wF(G{cGm0k{>@0+JS2g2kp3=-n7)ctjxd7t6$cc21nPhx8{7~
zd+nXN=*ETr8?|I}Y-ZM{w;z0-6AC=qPG_EMpS<VA-P6ndm*3lY{+!D6?oEe(dzAEv
zLwol#^CFhMnQ-aj`q{z9&)F@1t|DtQD=+o#ub0lNt#sncUci?i?ChN6XweY2QY2tz
zq_GC`lhWe0qwTM@Rs|*%ov^d(4?U-CRrd7tthzJD4Gw?id4H;)Z{=&rWnmU4KP{+#
z_g!$#@0F?h{x|OY9e#5D{NgG(<JOkX@AhoC1w6lL_U88%R_8a{Mn3%wUN6v4UIZNS
z_LIMTq5J(byMi6N<IdhOyIfau;?=pzo}<8a;w%2?*Dt?&f3@3G{eSI_4>JlcKD-rm
zt8S~0XR@SRa-h~l%V&R{>eokF9exa5)L(KKIM1{>{#1Ho^_Ka4|78CD__05FaXL${
z@&D;5)8~el{eUkGd06wA{UbP$9Y{r5N&cP5ukX_7_;<@+U-ikY-Coz<3LB+=P<vj?
zPCn@xuyK6ZJ$+61<~`tsZKBo6wAz<XEWXaS^C@3dnUG&>f7Mz-ef_r8l?jtw+ku-7
zfTcyDQQ*df^J>;WS3u=OY<y!-@MC`O%E#iVZ;$_LN*5@-j=ZQmA@b^6)@OHSLR!B&
z*7RLIcx~=W_XF2AeM_kSRw$JJ{c@V!vtwQTJ@(t*7<_o9kXvhByZu|$y88XI<IdlC
zQ1kV8#vJo%o5gPefT46vSX8D+tv?vLAe(nJFoEW-|I3~o{Jyj3@9({==YPK32JB;f
zeRcK9`}&IhCl73`YM;2i{V2yRcOM)?3^g)+akief@89<{|15i@8ot#0SrKrl(!aA;
zc9+?%{lDjtytsaFnZ&Qp7GJ-YU3zwR>ql8Nx949C-rCJCk2}TZs{$I(D(MRZrXe19
z`L7pLCSm2b9vFPD&6nM;o+bVI<-^O@jkj<4Il*U|?7m;GR)2YMaq%nlf_c3qPbVkt
zOjLUJ0^+C-jw|@z-}Zns37@A1UQCn#k94Ukp@2^t)wFbVPVE7<n84#?el95BNwRN9
r)DB1^YEB0VIB|T|iY)~j_xSyn57k`i)#i6&D#&I}S3j3^P6<r_#eo$7

literal 86119
zcmeFZc|6-$`#;>N4!66-R6DKKRzs_ss@A@xGdiVJbf{8W6h-Y@?2@`osajh_QNk#S
zl7^zTgt66{swE;KwMHa@1d;vuC^K_^U$5U^&-48Ayq<4fT|W6FIXUM#*Y!T{>wTS*
z#5?B3V*ft*?|u9BiQP20Zn<yYzeM-#`(Ex}!oZzljqG9Izwd%Ajepr!JtRX0o*eM_
z+3e?i`;ch|xevYvo{I*U+z$rcdV1&YcQ-AS*7ofal-#`j^WAVKDo>F1l9ay`L)Ut3
zV9CjL%z-F*-zW}Gzp-__QtYaU=t(6BJ5wXKt?=L6bk3&13uScNuS)#ndC`Y{{rCI7
zUX1%mT}EfW^3Oj!Ovfv4eMueEA5_v_QPW&;*7KFdn_c#6NKBCyTC9bH*+2-^kg<kR
z^j$*BbZ+*UYT!{(nfr?aylM4e$$p@pJCBd&lvEGwUhaFQ9HFpxc}KtZ;CH*%k7sW>
z{cG>`Q49JndzXdbFOL7!NLNP74|})I7R*1{yS%pkzfJGk_y5Qsi(M$Gl%_LjT}BYB
z{d$#<TI7TCrG}RpFP{`aHMT5LHp&v>&-5obJ-U+0V{Sn3_HvXk*(79d4z_nbug8T%
zD5vbr;)j<l@5_92uAD4Z#Y!g%1y2V!UX(o~8oh8WITNjpA2p$Jf1&A`t{`Y>E5`Az
z8~MGjn(R<%tZMfTF;}%KA*bpmJ{D*ojP~0nxK>18sLhYMLbGr@zB|>0jq^`h=MFZz
zD%GjwZ_W;FFgKFL;}2}9I~0$mM9)SWs-%g?%cu0ZV_<&_!^{XngO3ao4$$QFZBQxE
z#Yoja$K7@wAA50pD|rEQ@|^$EGfldZ%~9kdOfg+4$&af6>VAUMu2{r;b*)mCvSuUR
z!=%v5DNSiD?s>POXtchJy(M-h_JD`yrHBV<wm*tEsro<Fd^B&|8qMbey}32c^?bvA
z#H-9G>G@E)NHKlDs_rZeDWPucm(Lz^>`lKm)*>$IW+W}pyk|h@=y6@2?`Y*titIKT
zd6)j@^ydH+KRB?O@5)5bk`~HWS48v0L^le~beynq4cwlwPQft2Rf#Gf!(+u6@dxOO
zVJ5pD-skM3q)HX7ggWW{l%DVObs~|CX<+_Y#L6RM6=L(%ELojwA1gI_TqxWDeE0(Y
zJcKpao4)ti&vAfJ9&_e;*~P*wAFpN!Jl}BT$GXL<Z$-n#o;728*c>`7(Hd;x7#8u4
zozO(#8_i}gBEYskGC}pK1^R}-lxM%zKb8R7xWn4gJ0WR%*qaVQWWev&aQ#=)RzQX_
zJ?2?3zur%uXC{~vJTp0aElFk+O7kNHj>tBioxm`=wh#ytF#6}MXx~Ia8c9Bir77ph
zYf%y4qrBiGF6AqthQzog6W-!x@rbvK3cD9N`n$aB(+(rW1-RpjD%M={ToJlT_76`T
zr8|*wa_rwW)!e)r^dK+epMBX}EpJ>s9fmdVDB;{=Y`)N_HIgpDZgn(QGMg76ld)t@
zRAG}}5BxQ>s_GS=Ed%x6waep@7cBwg7+&vN_+cs5Fx__R2_Xd@1Y)|qvmo5Sp=|oG
zc~!W0rcvO#2jB*rvLL&MZJ?-zTTA2w;fh%3h{8C9SX!#)`&oMtmw5-?@X_Ta-5*_z
z^jvwb6TY`JmON|lCPS}3@6TZ5Pjb=GA6Se`$OENNVxFO0JKdj_>WUawHUFb`SbJ@7
z!&e>u0nCV-(auJL6G<>utKM91JqR5VMf4Id?)k)A>CRP|X`Gpe2wTAvY!TBkQO(sh
z%rPetSlU>x)It!z9FnRQJgm4InQzDns8qau(ZAI%qeP>*Igcp4J6z|))2a?Z_NmyF
zKb&L}2_t#a{jSWUg?@Lv;}+1?b<OjOFcaqp{f&@XN34Uq#jhPsYp)XyWc9>><h@&3
z?>Tzb%uJ0}Xs<=GG?#ND8MaiPn&-sP<KsQCHhL{2tCG6gK6YAu9~(Tq)VEH{g$0f9
zjZ-?#Zem~^Km0u61-9vhGs9hK(WqdHZeAlC<XdgD-}n{+5o!^$6@A(QH#|e({^^;l
zp?}wCZ&CPM{Nnf}q>36hFrpUuC8^m%0$iZG(AL<_q4*Vhp@6=vo+SI#8WLp^4vcdt
zOH<IrCtrLAXP)j~k(-;ZwCJX#nje0zABuYQv7|C1kTy0f)ttU<x3x{71@q!Lqdit?
ztVPdcl+FpOaxpGeLjCoXQ+1DEm+C|cqWIaMsrFL7&r19OU4J*5&Ct@pKoe)Lidq}9
zWu^xdOgeaReZs?QtJ&#7<Gm_5qXp6!;^zb~g4;q?G#GFy*k&l(tUyc`w_n+#E=kk;
zmq!&B{6bRlrUk<jHZTGX3!C*GIgfGlpcoXhT=Qfi&j#{d<2ch){=9n3Ygz8=EJWfE
zER+Mc`Z5dE1XIjKD7SK_p$rWQt!{E*)gpfsbIZ_QR=@0O!`Ptr;cqj1%#r?Q$>yhk
zj>Y+na^dq}i~OL%uJ|#oxTrer=_m1EL7I1|L=SqFrh;5`%dl|PnsO2`+7F&Mp_oXR
zq+0Qs(He}awTu=2M?tpR!GiJV$A2^~AbH_Qve`jaq#Lts4Y?8rFIv1?7_AiXGJo>k
z#d}V8d!pT1qv2{RnZ9m&NT3}k$D4ot@9x<ELfM?XpLR^N-=YHtQIOOdoJm6W+JW<v
zhZXU2?adx?%#C2doU58{MD-$Mj7l<V<-7U7<=z9x!e0-dj|y2S*JGxKKIn*x-p+2-
zly~~E-_*NlIhLRnT@)RGinj{=0Lxbu;1im{^HtxUkq*lw@=B#6Ycbw)aM8?FGyh9j
zF#kOLp=G#EdCF8@u)UzpJzHydQB4bfw5sJ2-Zyv1ldtUoCJqXpp1e6N=nMY55E=45
z5dXH<wUR4W7cORlZHOZw$uX7uR1I{>F+`#`&BF#`+JJem5T?9Iy0fe?Jim1T^Kh9I
z%CuXdGt2(yV!R?W#B;{l8S24kF@W#^kIy534;OWk54c-s+-Wt3X1JG@OHJpZ&8_Dk
z{>~<+kYrhTiJq-^3yP|RzoeGW^$+rrwwnwj{diC-^wju#wbk><mlzr`s}oq7kLaa^
z1$<;#W+;bYeYhBt*Y$<H!DH_i{pG54B-<*YuonKeu~4x2>=8jRsJhiW$l#WlxbMyB
z(B-E;4w^r_>>6<qV^BXdj_$>rtMXs?bYf&tS1AB;QGAq0L@1jh@1T+6U6g1}V&_Yv
z*}zM0OQplAG{x=xH$3coEr+;`FW9wiiyr3Y?!-KHWGx)OaK@v#XKFsMF7kHDq=E4Q
z0`u@rA&#et9?x@}=0;FLt?F5>-aHv=Ifkc4S}UP6h(#g8$7mYEkyUs;wZ*R*Pa~D8
zxAGR#kj+@HX43hh${vUDcILp8{~icze3*Y?stkGg7yvuJS6LTT1x7Kb@;Q62YAZFk
z(f&-cq-a+Vf;Y-}hmc8)U*W}@WzyrVP`Q+M&pQ0GRN+;uYuDNAk{(XgC`7Mz2dF=+
zKhVmG*$+Y|a-VefjQhBjJow=!e7d;<V*HtHFp(X;hKvdPx;_)1vdT^=8J&CxFGPdF
z@>^n5vk@8I8{oWmkY9QbnW{3L0^OXs^lKf5MN6yFl%-1#cxjC<3LvO9JD({H{Cn@v
zh^ocBqD<1;8ILTiSIFC5_plMpBw(|o9({&~6Fc-gGYSfjiaus>Ex!LzsNe!(vvV<O
z0YlG)Tch%6B0t*c*Sp94I|s*UMYK5iIvJ^qm;b?vgskN*Eul08xZ0*h8ny8HaZE%K
z-k+y8N%pS9E5wRB;m^tHTjcX_yD(~cK4Rv@eaQQDD@L&(S8wdvuP#DOxNds1xTunU
zC0RI{gZBn0rGHGRR7BP;YI<2~-lIcWW3kQ=?;^v&htN6Oqv6%LDH)?IZds|z^4#z7
zd1gKv+%X=K_bDLR@vI{#(C!KQ3>^A?kz<819Z6XoWlXOxjwE-(Z6d4mUpM(RW3TlB
z5Zx_j!KJ-aN2Pkp<GJXe<F-jD%{e`m0zD#2_`!%6`b-|~Di%|Kp))L*S97PQsf>vk
z%DWX;bE5eIL<evPGdU0YK^lxjpXXKTlN#ZfPDJw$;XY<?eBa4IZcATqj$Pb+mGMXS
zMpF1gPDjkJ%^7gV;X#OpU|@NzF85I&XK?&|Gj@3j3TuM;Q@=)d@%6;NMX7;e#Ej%k
znF3V*jZlqg+JNwNl)E4tjC1>BT^svyJF;IaF|;B3QNzzuG)l^O=?g5sXn;HJEYh|k
z4`RPCe$&al7IF2V-<To78>%juA$v%8zLMB^(?GQx+n-E>Gb2+#2ppw44zAIo5bI8v
z-qw2#Lmi(5oF&(9Ds$C7gH&5AHXPdE$eg+jYFL0{KSt#73%0<fo{{K2MA;cQo#r=J
z7j5r5Ub`L`dI}=QJLEKP?bJNNM8{FIuw0}DVcvH;LRKGShCPSC<wCY{JtJXyY}`YF
zL&F<3$vz=(#>^kxyXtu}3^7K7!rYyVWv7`f;ss@;d;Go?u)!A-+4>f|(-a}V>py*^
zmUVv{A@KQMMmXm$zEIg{-i*+y@SDDyhf#<<(LdL&NFK{2gsb|O4pc2On)D%1uBFxS
zZkLUP=h|Bgp}?=kJtv*?HuJPoFs7AeuXhZRFK7{mW+}vuzW}hClNmm(H?fpA=2BP~
zSRClC;(@T|>x@CpA|OFldYdiTrUr0=;bP4wT;gg8sw^6Ye@A+)>2L&r&6)I}-$Nlb
zWQHP&!|d2d>jL;_qzx)d(CQT>uNzsV5mj)3f@AC4Bp@Ei@~AbXl;<A_HMt<;N#2n1
zswoBQ4cNmE3e$~p%COV7s2kjb>;iYvPe9mwguS1+P9}swltM|EB(P=eb2Z0WSPSul
z;wJna0_R|o-_LW<j{x&E>3z~5)=A9^la9*B_2I=3AwBR#uz!8I6jUjOAe}M&GKX;I
zmJbas9esA3*nkr9cIn!{n8-NJ{uw}RX%3P<e-hrqPC<5ymEFtJ#=CCM(Z}4StK?1>
z3GEolNjXVZ>HQA~#`ohsbJaMWiJz_Ju!~9IOlY6LfYXA7`SY;eA&r5!d9{Y9+h$rX
zGy1&o?gpd}_la&|=MjU8X~i^5{(Rf7%902?IoBZd48`hxf;+RgckbO7HG)Rf*a1^n
z$A%j#jEN1-1&mdUijzH|^<FcJN2Z6VAmT`m7NKD*&bMgUmvjLLmyaN%+6giPOFAw7
zGl-3-BRwEO|5y)R>g^aHGTqdto-~Oi1DL6b(Fna9<HXrX$-s&>9q}g3V&e;mR+n#g
zt&MYi=Y1Pz23LG+XWKCHKH5JS6_2_H`57rg+@5wmm$UPZhQ1)-U>^gIv_g-RG0Zpv
z71~^PmRDmp6m=$$@L>q%v;!`Gh>KqAq(sL;n%i++b7&`4{Rrma$|7wbmjv&pySoiy
zo2P#FE1glZL7_Jn-4-A?T(|eioGG}B9=ROUQAf^&tmzM+#{JzC>uY;7E9qCI<G(wP
zMw`+6M|yIA3^N`Nm}qzdGB%94L|JS)#{v~w3LOdl$fG$`5_tRpvPK^HYv2dI!M9ZJ
zaY1@Zz`C!Qj3s|=X5vMY-w+#SO0`5oPu*LnW_%r~wLd;~QnNB!IYD$;EBzi%KkzA!
zs=JxD27c>fJ397W!5Zh>XDzrYaRwl?otTUF8R4@{5R7ruRpz7RK_J6gt2mR#?sw(Q
z6(^4DyWNfSx311O$vqpxo9R3Q8+wMed(K(yCV5avn|Wj2F2u^jD9eUKD_lUuc;pdf
z)@1I<qz<A_kU-!S^G<MU6RjtRSOE(q87QSk=Ub$7($E#H`D85X2_4JOXaR5<E1kYT
zYKi0E{5lX#I9q-`qIoc9)t|pvLDSS3No(ZIRkU0p`I{6*=Z|&=Tu>Q*M+lxCWNTmv
zzUurT^{i-fy$Ihd#K<-;zvYZwOLY^gb!oB5EE7I5IypGA)f`n7%H)g?d`Tf|+LLJL
zH@U&N7f4n@>{GS;=;rwo1ucFlqh$iui{kT|-q|a+qQ5kb&i4ZZALbxW5d?<mv*-iy
zKO1dItAcetZ<Dv*l_RB~(NX&0#5g2xeMZW3K>9ZJ{c|ha-Ck>eHd2$HkFJ=db|c}d
zsVdAWKv}?J=hX%t7XLvrG&AeAU6>N2$v;(2OY64va@D3xWdy=#^)H0PZIoR%OvsH*
z&ANx)&(b!L3&s}RpBldiRpESY>w|DNVj{vHu!x37MRMOc^3Iq)w0dioM=HfJ<g5Z|
z^BKrhq)3@X%KHW5gt>`+(yi?AX(tj05r=L2NkTMaOBZy^m0X5e=c|6u@wqmnFR}s!
zP5M$vn$iimGWmtk#2bRTmLW5(5t6G(Vb{V#f^TZpYOrz|Ois(*!>@L1@&g*Y0tuG(
zt+r?JZddu@vF??S)vp@E%Ub}~dJS-`Uq<TkI#$A%lgj{G9m}oz(mv-YV1JW59-q!`
z33y&IB$y^gY>XtdAb1}lY6qvAs?V5%(<hEbo`tym$K3m4D}B)l)roN^(0&b7ekgr{
zAUh4p_-G-kk)wU<8e3gbr4skyex^3zmp{tCOlAr^O4w$v`{J$Yw(_*S`5KT9zq^n9
zHD(tH(U&27-Bl{n{pZ0)akacNfyQ|;3w1fKeL@L2@T-i?d(s6Ab*0NudSniP(CwW^
z@~oDw%Bg3f#~7cXE0dI2%}Er>Zi=Ky#77!T8XWhPGtxpI2iChM%g;5IQJM+()!kg_
zW>^DwynSwI2_yCmzijP#4<5Iszh2y7-klIDn*43*FW8trT6ylPca!;)A^I7k>iNZf
zUf|cwQ2QR-ifjR{trr^*wP02dZKfGL&UMxv8uRbg^gTZ$=9l<mBL0zTAbb)6xpD(K
zIv>r0)tyd-Ts?cIVP*n)?T-xy!N>K+YGP-(&%dlGK1V#v$eTDe$)=*U6@V`hMC@rP
zNzc%e36nrr+`kq~`1(4=i6Res*n!Lt>s*G6U|gFKHX)6@xMejPr-O-u9Gd2hJpW&8
zoQxbnq#w;_u!T^2B3d(VPL{l(d7koxjY;ZbOs++%JC^mW##G{c`MIxnjFPd(H@;aX
zjEz#Jg;nC0mbW}Yb`uw$EzF;tj8`#LGyKpsZFgE21$*$Dp)MllpCoxkP@#cr;*|ID
z(p{k+caqAl4*IN3?q=3EH6&-!4Sq7o74%8-BO@<jA=md*%!3%J%t>$V0m!}3sOIq=
zI?k-nL97AW3#llTK)~n@Y*JQUq=%At^aYrcz_0452rT;I4PrU$ilk$Ha<H;QXWS;K
z!vFo5dYKbuAVK4qyeXyzNIT2$Ne6XO!!Qk?5w&?<wGdQdEq6M9eZH~^o>m?f@9IHQ
zKmKj86f(V{lzxAtof7SlJPPAYWgrbBq7J%=OQ%PqV2sP}<sl$;zUuDo;Dah!Z;rtD
z^UI;NH9_!6f!zG^Jf`fHioB_aBtR{6aq}ufgmjaH?DUO=F#l?-H(T4(#+MW6yP3NL
zu*daUX%sq-Vs1)kI0h@Nk7zC3Gw2OAq=3t)jq6eSA9>)hiD$NqsukqwI+e@jxNsWA
z2i8Zn9`49&+&LOKcvC(+I?ShWRoEZlTGI18|2V$~hpE!figFn{Zi#I@jKkcQmoThH
zaD&~c{GLrDpVlb$)4Pv)dTFrDdU39&>ATgXh&)nVUXo9%$Ewi+^j(^UZ-Nrrp%Tlo
z5$~DUsJ~yGtw(g;(<6^r(|>uK`o^58w#BgSrR<3=rk5fV0&1J1V&3@&yFFWwP&SYI
z!8Pzy*yNrxT37~1@_wPX#s-9W_tIz71YM%iC4%PNFWBNw7-(ln{qXdaF0ys8X_qQo
zvjC+3DQb;EtY34$MLDnFjXgEl`R3`Jb_n94uccG-!<7B@dbM=A_h4)uq2M{42Fo)v
zJg%4qlDc5ou=lBrT%b09q$0*U<I+}=gyK5X*mTdaUO+lsCm<K(?{(2fnR{<-J^#O@
zh<*R}3n-qyJfVCkG2NZTeV^OXuYG5#qQ3W3O7g68`IDVu*3QFd|MwT`KJQ-db2ce_
zTypH>-m^oZN8gYBBDDMLu|n+rEIZG=mo6*aX|p`Ii^%)-#UJ>$tyj_B)cx<5{eQxY
zPfkj1uXRYwZLiO**TQ)hw`c@uB4?U8JCh(^ur=YDW(M8@@p}YMk2{4wU^o4$^0$-R
zd8!cWbNMS=z=C^-?AOf{6L%95`t`!!uYblG?Z*kj>WASc_buHRj{p82M~Tr<S|gXi
z&EdMI>SkbyHCBh+8VIZZcpTf$<ZCN)I%zd#HLBN)&q^nRZe3+J{_TX<z+$2oHy&U6
z6)o}aH-Gyn+ug(&DSY5OJL1l%e|mA;Xg?gvexwhKrFW3~<Rx!_>cO=C+wmWxq$IcH
z1#>VZt@Yy_80c0ly>$uxPge~<N^9NH{z_V_$E`x;+2#NA!9rZLh2D2zePhI;XSy|N
zE{e!;3YtS%fp}z#x%u9VUIgTPvpnza-w{#h5r3d2Sl|)&U7Ep`ZBQ}vQq6a<x3qN6
zezq_^V)A!fZgi$!3!Q(=k`ezjFLK~K;ZxzY<*mrNzrVQcg@h=V>H#wK>mxqBC(RC)
zJ^CqLie2(wZvaxMt;g3&8pZ$J!kms;uwPpFXF#T!N^AXK{*}hVFvTXF(`^pI-`5n2
zQVGpk1yDA*Ku+qPo?PrrZ@YkKc`@}n`Y_?pzwYKz4u5~XrYIJ0g`#NhR;#~9<~*Em
zz>v2wvOUd2Wlx2`D;t4#Hx2(Bdpk5FmKwNu>mPIfDlhp-d<;J639pns@xxnn`MCYK
zuSSU1|M_Ri-Y*&LXG9#0Wk~_AgX8wM23(K*XK+BC*s|U&-8;D1_zDBqkD+#U4I|)U
z7VPbt{uw2ZYVVE`IbHQ+r=6Ye@xA!)zd9l-xt$GTWzUyJYSkFAK|uG-Aa=`97ix;X
zdjkzElZ&Dv=@$2M|3{kv#d)jwOCLs=LuEEj5hdJ=V`!jfrXJy>KHJ6$s=cwx#lZYW
zb7ihe<?T|s8U_5w-yCI<X1>hH>8oY&BYP)`%*IFoW7SgT4!cX`<E>*5lT&p$hD2SJ
z%~r-E1&hSb)uX>KGn0;}85$Z|ocRKZE716@ZBu%(>0*}3g=P;}q{-L1IHX@;P;0C$
zGh<PD(O@PTny;%Bh@^#{iFd{4kC8~#u#>5YTx@^$79ziOsOBNw<!qqrE@n?&ReFD|
z#_GaF>+&bG3mCtCU*YMqd5HjP=OOFed77Q=wTYC?9P@ZU3T@8=BWy<XOE39E)zj)v
zLWv{j+W5U~0*gJsOi(B5E;JSy!oPK%yb}Q|Wfka7!|@0F?8=zcezqNBQ5MPwyoK%w
z$yecC!H4GMm~COg8SuBOKQc6x0=z8V4S`9OQL`a@%ESNBwi~?K!uKtv%-l3)HQK(y
z@*u(Hkm$w6L~Q)r*J(54%#gk73cpL2$_t=wySRwilxU@JZW@<Ke^fuL$kFPWw7%fF
z(!lKb#2SS}_OytJzDx$yDr&xzWA<@EJMUyD8cRv;TI7}d-`<k5H<U+|@u~&*hwYMM
zO(tGi2eI9J>=&$Sb+?PaAd?h^n-vp%J#jef?)c^^cF|lfz?iE2{3BGJ>|;`hzPROj
zMQOL;%OGHKcYw^)@2_$6rRKiByVEK#ln*6RlCPe&uO1rU59py_O5)S@tr9}^36jJS
z&>_(&?Y~NB4A<Wh4%F0GOq6Mkkxa`<P+>r<m4Q?QZ@N_LKL6@s;#{QCp5E5!?33#_
zlA|ax`i{}IDD~PT8VtXz6fk#aR}0%WkOstFIGbFSk+;k!8L=el!JQ%&GU*m}T+4%w
zO7RDdP0mV)CWDYgO{L(e!yUK<4~g#hzK5r!B~p~p4e~(wJ@CIKNJ7-jU<XrEHd*On
z3L!rM-g`PK+Vs9xQIZ=_6+BYh5g4$VIuSwyBI7)%AO#t*`dmsUkB@2rX3%x5yuIeR
z6e4<M*7nc}@v!u2PUD~$(NMB1m}oOM8=lN{vG9R9uMA)W4E$U*c@=lW*y)|$?`ddT
zmd@#(9+#hB(kjXM+EP)d)(7l*e|f2=c(eoi1k?AKBPYS-9j<I$x!WatD^NChu<|@+
zwZ@i3h#TYlHV%|p8Yhc>&uq5HR+Udvl(?%34kflPGYDU<C(B|8#HrxPDF>ADRk<?w
zq`cnpc+iTFo{OM%Z01?2=BW30VG>C@${>^0QN8MyT!)MKG|t1Uo=1y^b@8c?NVWAC
zEl)lG@*Hr?R0qOLsro8YA$Hv=4k;MUAuU$pv%0FnvI>eXK}owL>6!aY;N2FuGEHpK
zEU;uO7&oc!I7gGLq*RobjSqLZs&Vzv;`qe-ur~|t(Lu#Q8zw$2S7CaMh_MqecjI@h
zRs5zp8hf!_J0?&(?|rd|>qvGbY187Q?TNgCw?2Q8TIxaLr9j?V??FKcZoHm6Qt9Cz
z1)=I#^Cmlo%^KnMoVrD?1o13i*;@`}PE6ueXtWK!u4NNtpcaJcsC#E0w}n~1o=hbk
zt;A0yJQ^J52@ep{+wS=E6R_^a)k~2k&e7VTgb^B(KW^I`{IO%^$T#1A>|*;uA@<!9
zzPi8?SLbcRI(*5o{K=&kv>jME<w3}b`NFRLSU2M>grq4CN+5_If76lCI|;1b;7Z=~
z2nXABtz+|xFpVJ3LFYBU8$3hpti`)5T%)l^{POvCIV)n>*>ZFICb7BGGLlk!zU&yD
zzg#e>c=`6B?!_q{LhfN<<`a2IZC-|0BG%ll#~U}6-bU7@4>ngc;EI}WrB#k7T7DQA
zG1?E<%&23nbKo}(U>O{BGDKr`^X6*^dm!O?2j_2;^`>k=>mZcz9ir5q=10Dz67qb&
z*W~NCEw8Own?7rMdsH?uhTB<Sn@u#4pOA?&6YmG3e``e3i49dLzMQm|5;@Szp)oPo
zfr!oi8Z&OQI~t+Habr#wX~!6baYz<DpQlq6C}N%0_=a_pzsnN9QG(SeDUL-l)+f@u
zMe}GgtshpwuAT_h53^;S)+4m-VBV&OhTjhsonOO(jE%QvQ*%igz7G-(=z{yr7TQz9
zu{CoFv58WXc6z5SuPvA%Nw|?Z#7ZLd!?8`{YLi1x2Y$l<X`3p+2^!$x4DpGopf&2~
zNs6GGuV{T3iPYZR>Q!ix6=HI#@~rXEPr?VLqgGq##decBXyGU!`VMz2&b1t2ZKhwV
zLwnUa_z+3<R3`n384{o_3XfBt&TxeT<C==a4!uj_6_-;6Uxg$6JQYSlj%<ZcSzffr
zFu$atQszPk6|L1UGrghdtu>J6J2{k)Sf)wR^wR2VA6;utlx?g#vGnz<@If7=&jdTo
zLji7n0VdAe&)vC-mQ$Pg?c)jkAaE1Ln_gDr344}mzOk6zcKr}kk7;XPZ9#b7b=<T8
z=l0KFqD@XwGHsj^d*q@nHnOHAYOD@z{CW7m`ENldEx9yGp2*;AaN9s)%GFkaT2tZi
zu69ijjGHLdxZ0?w3CfVW_NkKiFTZ+G`>570!NXo1y1~E-R?rHq4d$wjr(dftvghga
z)AT0XYMZw_g@lMFe1bi<pGZr->bEnOA&d^&g=#9@;W((jE-1<wp2VZ(hBqk9(i%ql
z^KTg<7}`n!F2=O860+|s2FVwNT^Lp?*xi6CByIw{u!Q<{KTer$CB5EyR@z59g6_te
zW}5E=){_&GACDTb7~L?f)|(C>rO(h>h<>VmdNaXX3qsL~%aHD$3tU4jLni$TJiHPV
zz9iU!-zdb^0^ZN^aMP4KX#qyKZF9l~ocwV$DS>YKF|8*KY5-l0#;n?9g<Mfuqt;Kg
z{j8NY2`TfH@(ldQ7vG#sZ(A#F#NITfSn0j};`6teBfU%tEkh^xU_L#2T~$C=9f5zb
z@cpd%#OJX>v&9ZSE=z-cuG}&DVvLEii+*>HMGqfgqjfOPcYd5t6&H^IsY&IzBZ!0X
zQG_z86|-!yUQg!`tTe-Hxa)-E+lWgdnmx{yMy*(x?;hOu3IBt2t^Ge@q5}~s6yZC5
zK=$fx`$vBYpZZj|me}m`Fr(Ncq?!pJkBGCbBYdRU=AJ)uI*weo3&Lm93uGier)1hK
z{glH1meKInhH6qGOxjaf;`9#}Y^sv+6wGn(@NjxtleYGFc>c+<t<F9iMWVBQE|H@C
zTdinb6G(%!j8+_vDckrc=j%Ub6WowD^}#V&A$A0*90r<{|GsF8T?IOgx@NThU}IKJ
zz0rP%aY^`BlZ=t8>chk2L%P^u*X2+8Vxr0Ni7Acv`@C!HO|U<swXr=(I#OnrOz_&j
zW$z)+LkyHI8%lD}t-5~syaMqMb*0bk%8hVBRx>or*ub+{)IJy={Bp|E>34<L-0u<|
zBIy+T3u&MhNo$PR>4qr!uWsaR2S+&QxQe(s(zZs*a+e%1Vxpr4uwZ(ZteOL?dG~w2
z2b5|+3;-4-fWT=Z74Iph9=p6oy5PD{A%-Lm%Obr6&!wWhw6|c&VflFvO-`}2k3&rg
z7pK;PLwbq_hvfZDPWhhIB>4$$VC&KWKj)h7N(mb}yPCtVU)IyxSVa@ut|{TKzTU1q
zsyAF(dK*{ua2r&DLTnw9G#{Rv7UuoOmS&mC+Bcr#7n+;kr*5E(R)~GmgeP)u&+0lZ
zZ7ReLqD@+-@i%#fyInt;^i8Z-`iUF@bSP4Y;v(>5NB(#aq_oS}%q<Bl(vEYb^9WO+
zdI$BCE(eB;P<=g^QG~EBWyeh-oguU+k)0?^kkQ582FY>+s6nO-@t9C1A{)9@KL-K(
zd;KJg&Tn4Pkq{jv3=vn;Ml`_wt_5J#d@d@)y&(QTcUoun)QXS3x)TSo+pW?Q-;|e2
zFNWd|ENkM>22w}Fu)P##E7Ti`LTnSGEethdcoa{E*IN+aiR}~WWq3ai1)%mm_cdk=
z?^N$)5hhL8G*$o{C|Sz^fv>}i$~^<=C|xGi$)w7%6zv8gOukyao^Gv))a%#me!kvK
zQMcku5{|~1rM?|5FOZh}7^1uNu3~!VT7$9OZl6Bh+?9jAb*hQ0&cekZA8cuwE_$03
zLgTh8^L?Lr!0QoEuU4f9*s`2!LIc5RWo3Gu=T73o<C;>|0G3~dz>Wgg>*2h`$yUMt
zAcE0!LbwZYlJs0Y(kPo}3ML&m(aK=ey1dVf=B(4eu2ToyI^}zd)wPjfWf(9S1*>Dn
z2ocF{0e{p@OG|#DS<&j$-m~IG6n4A#?9*NV3_LO^Tqx^RcJZV*HsUYlG$_QH21fd?
zDA2Jh{<X*`=-GP0{YL2h49-=4XZ+Nr?#Gq;b81jSNJJ^EkwfFg{M5YFXe%3KUV^|{
z4nAOv#0k$OXf8sBWRWYld!$?h;TnA0?X8;vwx(DimR(2ws(vyDpd<^zKhkout*ka?
z)35Qo=c>QxhUy+tGn`)5*p*%H$m|MWNpEU_9;tCHSQf8|V{XOaKivo=Y2E2x3^e)r
z1*eEqgFWSbiNSG`&LmWT^p4`k-I!6QgeUX;oF@aQZ*jk)E@&DKN6tlZN1smwVyetE
zi#$es_DT3WYcCc0p709)+m9#52<FQ#zuz>D{}>#6=uE110C|?Iy*=3U-SgYEiWVWY
zS#u>E+@M{ky3(B>7vrhS=II)|W@kahAO;O%C#Gm%SKa4YG6CFsWjJ=7&jgB^Fqn4W
zt!J;1MH7swOl0-ki1Dt~H?;nWQZlej#(lX4$hsPso$Y19uJ7kR@nw-S{R?7UdJSFm
z{-kO>pa6Z^74%5v`p3;mY1baU;t2EM+HsC+k5J{@I2|iYGmzrwP5qv)jjEyrW}yTa
zY*%Tkh=*U)<nsx1u$9k8#yF1h;iTkS5SQPUt@U!%n;a@f_@Ozq42>vhvWN(SnhS2!
zkrvU4>%H65r60n8y;WBNqTcma3EErpTvmLqf~CWqjj5ktzo2pH*KR6%H&XE4&ES6X
zgafy{(hgl#Y?jFLz(%{bVtEnJ4*h1;sM!sD+5e!mfY+V$q$pm(I=B4nWu@f_^2UAM
zpjv*O>s&701KuMYVX!;P8%O>!$U3mx|HierAgyU7JyowDo*G(D)&Vy1mKt9)`J@?S
zJQUU-su$>D{H~fP<x3u)7!E211^$cw`kwT}qa$kD`P_I&C4|yeZh+Jcg9%&avCEBk
z%7>yGKctc<;&Vh`Yd~e4L#<-Go@4)|_ybqUBz&J`I=*8$5~o1Crdxz@hYf1_HFqE*
zpgONq3cGY1*~`uz=K)(<0g)EKy=W8XZnCmw*D_tX+gq%&53!m!vz|jyp{BPbuV-*v
zM{FwLi-<t{^27(2+t%kL4JIj8u}UJR;Dmf)oISQ{rAjl^zxjfukAYN2xzQ?CT0Yg0
z`G!Wh=0n*s*?%Msc@8Gcl=TsW+nr2{V=nmlI|m;sHB^8>m9}8bk-x{=HFQrJi?~u<
zLJ5BH2i!k6^v7l*ACT@us*oO;INOPMLn$^|T?HN_m{3bu!htKtdt3rY-Dl98he8cK
z%Oas&U%kt%p6`0%(TrW``HV|+Nch&Bxj0N}=Y&4AqRIXUAh>9UEiq<>*P9rf9;AC1
zU-!`Wj|?a-1lK{gw6JJD>7qI&MB7|-hN0<kWG;|>oyO7*)a@pb-*Z;5L263l`ccWe
zkM|hW8&k<Cop<<+y-q!IrbzSMZ#Kp33g<j`?a*i>u!$p-NbcMN^L<Ub>tn9r4iS7O
z^Q-fv1m<2D#AH_<Hv{E7ZTFWLko(mccRZ{!fF8Nq>^_3pH~D<f@?;tkdqy(5EPuM;
zlCXQ)Pwy|@TZL5Y3F*$zU1)YcC)QQ}P$-bo<ck#GM$`h;Ao-?h>yvk=tnt3w?_wVU
zgnA}%V{`9gO%K1V+qoq##YE>d`L(gJs)*#w$_iSOiePgG4cdoCSN7gJkg{vqtS<pR
z(k6$#Q`h*R0TyGCoLuI&n+)x9rtMY%YCA8b_Iy&^>MZ5t%4ihMkLzK(yN#JWpS_Qe
z@q5Gi`139fk~MZTEGPG~d-rYk@Nw@;|KBSREJ1&|`9f*$56{6%{)5e~$!N-d-+nQQ
zDcSlt>d9_o{Qv6%T{VIrZbqRkAfZ|OWu_NO<#5xLxUGK!X!g?X|7^iRK!`l@k7buh
zmYl-YKY>^P=pfwWZ_`?WvL{7Wegr1{Y~5$x^}p}G02F&ePXrJI*8oNE|2P#uCJlQ2
zcbDEkFV#s*UI0Apj#m*U@$dUeo5AHVpPe264*kDF_}^Lh$HD%I>Hk}f^u|1r&m#2;
zO#g0ix$1woc>V^U_<SqwP;s>jrhhx+o9VOv<)}A_p}dT6=4gOElN($+z5Sl6F&G7~
zk1i_<18daXzZ!q^T{HCbrGh9$r|r#Qsr<9x`csZc<%tch8}q?x8>2<qA)ejCV1wfw
ziXO8oiRJXorX_lpHj`~~NWiuU{16?LATHMZnW5Tg>R*s#QT-Bu{ndsaa%o!XL@Egi
z3E47~i}X5MpvSUI&gUpKp?bZo*4uHz;WMlvsn@M(#&b4kxzy)Gw5<HV%mY>H{|+cN
zvYd(O44_FYJ$n)p{IRhg&~~BQ9rE3k2$UVa;;&(U(bsHk=l`M~7IJti_7XhJ2AYo#
z@poD4lvazV;~J<mA3IKrR%jqkcMg+gOV%y=h$E$y$6cxO&g{^jR6e^z8#`m>#G1RH
zf$^VoK7=;BA9=kMyoc61Sd#%=?wfB`i~NHn{}_;?tJw;%&pHw$dPeWcr#$})hu#$U
zvNPza`q`X$-u_h~N8YI$X=D7??XH-vgJnaV7dGJ&syb|nmP-w{YLGoo_4is841>FZ
zm2b>u647^=iM#>l<k#LzmcU|mg*Z+~BP+_ZNP3vJ{(zCK#Op<QDGfPZf_Z0Dn<aPO
z;YXb}*52k_Pz#!E+_oNEYa0%yZHA^rl6Ro<8A?vlR)_v*D#q(PKl^I-9LFD~slr(u
zpdZP8K@7fX0)QKt)t8{dG9$?Ud2VM!qmQq3DW>U5qB(@sKI~|;x`xh)jMG-KW=fYV
zcQ&SVBDFQ|2Dw~}e5C$H8LqlFW=oF+^K<Ewg7Q~_m_qOM_NA+1@W2f%wNPYW2XvV?
ztX+Ut2N{mf7PH|f*^W&7g%go1hue;-u&OVaN$XolMzvU-kc|>;;w-VDY)IS&5#@Q-
zNe8>Fx?yt(C9Bx@AfdQyNK3FGsFQHn$eDt`^p1&1aM*WW!#2bHCT7BuV;uN)X<0iU
zAETv|E+)+FkFuFviMR20wSWfrMn3-PE)A3ofzd9U0r$g;OAG*=!^TUyaC$BcBqtxe
z_A^UQof+@Ssd<Qe=f+8-|9TMD(}85HX7#AR0%0=HvgrGc89fa>75475k>r(<DBJ}V
zI^*=-@N=<1C91zbS56U;)_t(Ce=$aWiQe3Zmwz@aktcsw(6?;@!7MLb2t@T-E5pFD
zy4q}fn0@qDzbixYnwx28oX=WeNGnc^SG=AVW*7atKm9gsY2TpMCZdJ&3rqKO&kffa
zatBJ1bfY3p+8FGo7XDdihb#O_JS6JY@tuG_Je9>NuEdt5T}f{u{Iml#<OM^iD(=tY
zC%z5z?5TxjBF|4>z-26~ewnhHH(Nb|_s|lo>V#KEKd259?=%<zqfg&6c&{feA)C|z
z(v>M*P*shIKd={C<?u>zQJEAlug9~+K+SqOnSbiW@c{X^?%Ff+&BN?ML0#S^nPb_6
zt~HhkKLZ@7`2{xU&P!|<5&Hf}@z~7o9>A@!_RMgm8>{Jbhlf>z_K*JXw@#!dVffd}
z&h*>*Rw!CgadC<9MDw?Sfq{V;i@!>%DVA{S5H5%RrnIv_w>ki&MbATqnTAt<boJzd
z#t)zSb_&}&*tdEwOl9l&fg3JNtW8%1Y(lQ*(o5BTjT-N+fgjqj5)B5JFJIhDuI*tR
zyRkeB{Omav)72HvSW=x&Z>!UWRYc-D{Oh!zTx;yLsRT|Lv}A1`XQcJaHqx;YvXLdc
zc=l-XdLE9F*4fy3Y(?1>lZkes-{PpS%fp~nt^7}o-R%k9q`8tltb|w(-dBFh%_dF)
zs5b#@Z({=}ndd%klrt#2kNfR=x#*Q3I_bw>0TOXj)dr+=Ih<P(&Y8{JR)S3ANPe5L
z9}16H7;N5U9UF~k{XmYnBu>-{2EQ;qv6Og7bdT9ss@Z>1h~+;2PPci5`cf#9Ql|OU
z8#PV+v`=GU)yPZ8P;UyhKp9LubjOCDAt+arm`6k3W+)1cPA5EHXR+S1etEeQF$Q`-
z#Qf_ud!9kJnrQKP`ZbYZGY5ll`aV$67$=ky-;41~|7<!y!M^lA1WwbzHXT|nvqN7C
ztTrX`=Nlj00xE?L<=;Iv=dC4xl5M~-b<w6?V1F@DnQh?U*Pg-Olz0LH5xv1c#vf*f
z`e=QmxR$x_c@6)YK!Kafs~k2L9{@)!32aWLZrO7-Q{@C@{yhOKc5m^Cd3ht_xI=aF
z%PaFw51LwkvnG&MRx19>&1j#%H^eJ4{nTb_4)w!++}gEDnxRG}cN&_#)7v!Q2r1Vx
zPe9O(-VV5J2BC)r*J`EZP0L$!&%3!-W1_N+_Sb^^Yl_jfA%wAV93El#<`Nu1=1j~_
zn^~_4rr@=#M-|32W3$H1eD25SoV{RBu|24*bf-4QFhL=<vuw>fHQfNX>Q?-IU}n7`
z!a*a-Cs;~yi5jS4wzGx}XBh2|B6>q@p9)qa;HS}LIcWWeb54Fi*Z)Hu6~bYyh*-=H
zX=JMz?N0)GjleF+Ma)0nCagkjP}U{hb!3BM7Ep@m2-rTfpRK*-&YESfHeQy|6FguI
zY@SC2SWuyA>!E_7*pD7;U1n!&SYP&npK1xz*_pl|wEmem#%r<XUmO=We_X$FBbYn~
z90Xm<wJkvN9yWAV2%y^C)tWF=y!<&Z^Z5)JKf&Bwju`+mr#{od-}Cv|Q}p8$S7t9)
z@zpa|iEMNh!QE&Kf!J<0Vbw%u7A4tNfY3A+JhE=Xfxn_M(x4Uv;%^m&2ZdJIZ?I9e
znax%4?r11X#_yGh3dAL9>O~AsUi>BPQAw|qL5;z<;MbLyzKeq9gxLT-)KOc8X#rDV
zZcC<6?bL#$!61DQLSheMzpv^6<Z+aTA@DES!|B^KH2y#n=$Ajg?0PYP`?BAi8tVdJ
z5S_&?Fjv>BXv>gyFm*!1cd~&ES73QVkH!m9!nx}`b8!-p%oMjr-t}aIA3M_SiX7N#
z&4p2_p_r^5iB7h?yXHrEcKgr-eNbcFqeM_0O`Ks5&PfB6JWhkioasn5Q4$u)j6;@e
zTn#g9fo^$B2N7eXE`Y<#V;q`PVU8?ZKd|v1b~cpnY^op5gq!($66u2?{n8^<@dpZ$
zT_v!7L0W!avi&&neN2If#DZoQ`smJKj)qY;^13;gpRHH?;E?mOtxvf=wj5z$ctXOQ
zxCk5(u8D3W!I<qUBbWs`I<HY)B8F!=*8UVdEvM)!sMnO#69g@?CHVGpTbv1Wbxvc|
z88E_U8o>_q>oWI&vkCxLxMv1pmXJIsamdAk3miy)-1<DVt=&zA&mLGHD?EEQ&irjJ
zXe&-gURL@X@y&~ajrI?kUky+c+W%vm{2Tvf&obF>3u+G#{m}9hzVs&Gv04Q%K8tQ~
zr*0#?ZK8eiDx8jbXicWG8&;N?Q$hW7TfyA$q}G;}hp<QuLM!@D1K^x>`5;-?iFn+*
z6wNe6E|8nVhzBjYjYjg?8>ZUa*WC13gLHQu*gw$~0*AsQAw=q2pE|HJrdb2Ih>%b2
zK#&SD`!3@366T5o&{H?0X|oelSM-8>nl&7QOW}7itocg(XXpV<yiFGh+n=4C7s)Ql
z;V=f3E8&Ik7{33xD#tm7wsq5BjzaRoK*94ZjO)W<bYKRmC98x}|5I~oLU7unCgcP&
zlS+E`_QbQNNA%_j<YkhAH)~<Q*6u92yC4@<CH~`WG2e4&b5n}7;L2cRAZY;OzXkb=
z9}K1gyAp@KQL=)Qz&`Awv3?e6pLb~3EOmjhU|*FbCh?}4i1;rY$bg}Cwmc0L7})b~
z-VzhNZP;4U@*B6L^YOKNv<!hp!ZU>C)|~ZKu%Hh$X|aWgFmXPg%9|SyhO$4iOE@v}
z{2X`=@#tzPPT_YAF=<+6O$w82Mt)Ihw7>tJdc?GyR&zNx`0nH7U`r79a_!IP^Yx%B
z!5?DTB%gU?4ykgH;|>>mVQha2m{oNfvyxKxqbgUtzEa3H5zd^6PiW}b@(C7Xt(4f9
zRR<@y#w>H})H%6;d`BHV>X)cul)?<%em;&kv1fq_cN0KOA$?ohEyP5rF{`&&=PI*&
zm&>a4Y$Q;jjnVye4}}htQ<%lyX!byQ+idpa6kHW3Kvc>~zJ<QfUw_2IRS;|Wr1HbD
zmYE&hs+D+|0!n(@C^FoW%{d8D1L3JUYZRz^!l2(TS|_4k`E5Zz066S2*F1Umwzyvo
zi{HcnMNRN?%{X*rU@M_@gt>RT;(7QeBB~(RSaJed1CTtD`1Dl<!_Gh=(_M?~jV-yd
zmM(U7z-JUIr&dVcTr3lPOIZeX?`y}PZ$R%D=Kjq8*P)S2r?L@d73*`t-uhsDRSBr@
zF?JHQKg!|{Oc!`@U&89k;Z3v@6InB71?Z6F%^ixi8#;qL>4gxX!3w`^|A>~L+{gAV
zy!Lw@TL{^UCYnUx>}f~02Z=7|tFX<aH^A=kUkcdTQ=wtasAc6P|A=>E$6$;eLp`O>
zTSvAt)HFj*h5E-_rkx|Ks#*^<fGTM^m4`s{pcF95XYmL<Gxl*pMZzdT9|OY<K9_M{
z_A)ueY_Z0^3&QaNw|^m{`-Hx$ggUe;UyTzYTL^l_g4KfDz|`n%J{iKRojND?f?3&n
z?t6EOUq27erSFZ_Xg_dxU3DkhjC40C#FYyln?1rWS3Gu<{8DI#wDDdk!~)xZdSQxk
zFz)*H*SP5z#ru;LdD9J3SAspa9>N+=$Ad1xba0Y~L<gpwlym(bYazzU#pw_LdzMGm
z{Q>ObLs*eHs?q1}#Vizr2T+D408ltAbTak#%aVF!m>w`soj-7cuTsl=usA9-CMe)G
z<XHVMueDf~2W)Ry##G4l7H-Hql;&ivB3?&2Z*84VQ`Q>2<G0+Ek36GwG@u{sCUlC~
zV+5KwVNc7`ak)0fJDC&%%XS2TLhcwCbn%RHWe*$efg*i#0x0n<QDbu83bEzkoa>uW
zi-3|jIG5X>-sTW5pY_L%rnHo!5Zi2_afyY7eYh;mOk&k${j$UMK)@EzbyMFB%s*LH
zLq$lV*=qY2!6koZVh|A}<*R%4{Q;x>%Kao1c*|J^>T%_avOQ&9^<<qjEVQFL(oE}$
z5-{+7%xX3Ya_R=abE`+GIu|sq9=Ox!pRqo#Ws2>WzhAh~ieuEeU<ZEtA-=)A!7A#h
zfPBo^#5wd|g*9X_-wg&4yMS4VTX(Am3*1?Tqp#(PMxT6Hd70*hTtvDCR{I^cCm`aj
zuG}?0_*+=WcYA^CHN9V4bQCtr&!Bfz^+|{xLd$LvEo)($x0puz5u*ZW2p*vftQ+Aa
z57LVWt|r#OAL1HxS*a!G_B`YiKuTjQUZZncXesRej1=i%T)DU<cE$L=d}W4M^)0J@
zNoe01^NobPmOrX}C+aO5m*s5ob!x^I#EYpTq2L%@UB@;m58jd9MlCd&Q(h|b2K4F^
z`)xysGuk$|eHbF9?;*I?T=fqB4IfOq$U`(81NNc;N`K;tvSK&1rD6l^jeWFO^Kkw1
zh*hnz+*`r#f<`z3&T1S<D?(;5DSys<1aa(GjaA&_oDsFZXt%NokS{2!ZAFJQ7;Fyt
zyCYzp^A4l(ro2gca0Rrr*n(`)y);bUlrT8raZ>!xNR=UYMYI9rP3(2~)b8QFnDE8h
zD<T~=hH^<H;O7%$#EOvR;mn3iiu^O65~}!#dF$JPd(~*afP+st_{uS>i56{=+2jQJ
zme}p0hMC}h(ruRx!JrTl75$0100zCDc<=Zhg;<<XpK6k7CP)QD9F)^UM_=OIbp=(w
zmUI>eV1@Li_ssSuiN1sp0FQ6WCaH{g)9=U_UI|VW_;j?|v6lAZ80-y{tu>piQqDo1
zGTJ|8K}WFHfrEjWUzp>dZu;=|ei8Gw`m6vp3B#Y7PwxgOMC4Y?AkAizB4H3y&2_jq
zb!|^0ee+X2P+6~&Uc;o+34Z5_6dQXW1Owh;_*IDXnrauzd!awino%xzuAMyr6Sz@<
zl*`mSUzI=^AEOqX7CtdM$qEMqCnHRkZNHVQN9K6p!YQNGZ^SSDKu}3FHtoNbP6qn#
z)QK!6C_jRqGhOIMZ+r?MY!EF0knx$tL`w=xkq<A|#Tc9@776Rg$fx;WwVDsZ4GDkF
zkJ!7zGe%ukxhDI)UBU=5{qyU~jC>i%r{&t3t&OOk#<`A~x9=E+z^ZW;OE4avECB0W
zze5v_KX4{(HA%%jtb4}hNCSsG;QVI|+Wv_5Ede@cGR|wESegr>rEzKnaH*HC3|_RU
zNNDKSBYUvf@a7k1Iu2SH*}GRW((GMKw*-Pq2FVJ05l222WO8bDM?!#;w{7!`RxP@7
zJkB#0JTo|!QTaC0{;egj1;X(@tjeR%`Pkm8>$ewljlqwuh6kLUHn@-SyUG3}WHUQu
zvZ>oc^tmeY8-t;zLvQ<%`7QOksuaL1?>^pzhJ~m^5Txr~fxOSX`G+m<yLKUpNlU5q
zz+|EA31iV6Xy~0nb{XXhke-8KbbIi5BCW=jkKM&iJ=6N=U^6amLm0*wTs7|lEk11E
z-hT(vZsw`1e<iA@I)<9Q1BE-g3pMTRBk(D1i9VmO?7NzrK;zYZe2B|-;V5P*o_GWN
zS^-Qe*Wk;~H+d{>tveilJHzVa0K~+Z-W&ct2;Zxy@G@avDWdkLP?Rl78@s01@Dg1h
z8+ZyC11s9N+GljKANBr>+*EE9&HuhUzM)>{&!7E1DO$mFh5iCD!k^-zoeN=ePjGn9
zNmU+_B_17G5sT~CZkV_T=kl)BG8-1Yw$}kk$~~0RO@@`|yoiK&*ZMJI{f}ESo`*6K
zVN0I_T6m9G2O(pJlx1Sh4S`i)MMc!RLSx0sGO{ZRhra_3I^Xo2LK{E~0K0B}_0H}G
z73C_#P7m*9rE&Yt5qY$*ITi`73d>%+M-dYs<1#G*h?3r+DKAC7&jOa_3wm~KM#%FG
zCmaI%!S50i#8Wl>3c_&n>-mA0<I8@6gC%&w7rs7PFzO0W9Cp+`hA$Hl@52s8&lF5V
zw1gn?3_LP}q&LYV51++50vyEB=)nD8!xwFy)JqcDQ}n70-!$rZ_@$Yw4IVr)a#3A$
zF#caPrA~z7Ou?^^SI%q_Z^bC^>sWcJdEDiwzIglkHwj96i+=f|V1R=w4dtIqB^AIf
z#arZiNBsV{7aLy@WoDe#lOrLW)@=x(Wlk+d<yjEHzyYh;1QO_|kt#|W?yxKSsb%4#
z{lk8LOnoX+EA;P#c$axXHypMwYs)>u3{FE$Esq|^2t1=z7_dRabh`06P=-H}oQ4dv
z5>mr}-wh~H*;`MXS&18qYkkixGO*~waq{Y@1IOv3!b!j>5Z)VkU?=%@iSYhq1OE2h
zdb>!g!LOxHe9oUy#&j(u&PW+(-DeXSXhnUE7>=TvrcRC_^^2G~kO?4<A789L{;m=i
zjM8R?u^VB5>O(4RO}BO#fMrvon5aXPAakmeUG#sk_a;D1X5HE-B8n)A4Js;=1A-%n
z2w@J23Mzt%0y0GbK^bKpLK1BhL}XAzlquMVh)e=P7!zzJ5kgRg00Dvoh!7xzkc5!w
zf0?@bJKf){`s?1Rb8ekas;DyIefQpL@3r>Y&syu*^H|+ue0#dvxa){!t9p1-dbG4z
zT9kUW!(%c$k<bbNy#y2?_2r)7gp!KPEzYt0!|A$el7(D;;!bU~ZQ<-fx!s5L3tp19
z9%P)aPE$C1P<tVl>=-<fs73#3BCa5l$L@RMlu0|)k-4_j-*X?L^FixPaecw*>N$=&
z?RztM*s9_|_~gM&7;VDz6;;^k&hff)j3-4dR4UZ9LWKCzB7S*!;ZN^l^SWz#vtRbp
z07$SP?9z%(Jb%l+;|0{iC{yViW30p|WcsUfwa;%}nfkY8My%~TO4ttL2>@nBcGK*W
ziZzff<QCDstZObnNK*M^ztTEUF=GVpLaMjk7ijSj1@td<=2_m^ZpR&h>5kEvl5=KD
zJ+m+p&gl?x20XDG3hu0?|7BuB?j}E*@T}*5;x%<^UN3L{<~vry9&g!Ru|V3q<}Pn;
zQ{g*Mr)V7VE+=Q<f`ycK)9YtHk~2kH!yE;s05Ta_6MqC*Ujc$Vkxs`c=x1=FcpzMZ
zvRs?!NYyN@VrNV2y?AhFt%VV-zd6ZqvS0Zd<B|jNi9}m^)H5k|rsyKX8LD*-E(i>3
zH?}og?R4)U(c;aV1vA)e$n3(vf_LV))um@<8u5R+@r=4PVD1kIQkNBA39hfWii+oL
zDpyYdCwe1bAHosUMIBq5v!&(_s@;#@e89O*EQdO7^{49t-ZRr_VV<a(Sm{37<>ce>
zUTVKj+nU2GT6g|x<kATWw^_`&&B3d`6&E@i?(Wyuj*HzBzkBPKuVy~+3+Zqmu>!s?
z;HswV#~|vU7L@71mdxQqDBRf5@-kpJU2DdSowEAE=Y7lIiMC=qYyu3hnB}3e3z`JJ
z?@D5L08DjvVS@;EH+RTpma|~L81d{pj{^(B*ogC>_ND|i?cQCYlB?%{^Jz=?J8-0r
zZ#MWr{NQ0d1R1Q73_>O+%ddYQ24NC=x1I{@d=SX<M$G4px$am*KmQg&{Z?Wm{5mjP
zn0`I*@UN3U*X3~9m$vplI=m7Uev&RP`Ni3I{hh#x2Nf^U9QWw{Y6v^Kd)p!=@wa$w
zDBNUELYmngLO<qN@u~31FCvKq)9Wuz0Yy814<G*qoH-8{XKwV@JsQ7RluPx7M2#w|
zboE=7WJ+zjwOF|I_XzUOTp$qSQ3uvNY^ZsBhN@s6U6Pb5!|z477Jt5PnZoPEKhar2
zQ2&+&FgF@s8+K?S!|<Pk;bQ^Bxt?hK^OOB^fTVug43wAsW2pP<lav4a+)l|ahra%E
zdqDi!?3eXG8Qecc;l+3c|L%=<e(TDp?QOv)5Ic_io2QSx1<JStfr<|vtwv1%C0R{{
zxU(O8uw7!U@aun?5Fnl}3W#xX+gjf7JxeP0w+a4FKz`ARuS)aaf6^lUhh%pAr6l>n
zALWxk<_|DI|LtEL-LxI3<NHfD{s$FJM^7i18?3oM0$ncUyS$t!R%voI<+M<{=|s))
z$6l5z>8yK7X>?kX!dk=dPUyLbnq>|FFU{p|IX8Gp=5VQ>=7<Fg3u#pck&N1_2&UBt
z@yG<x_$pOnDfjDM$#N##K-{d14jY^Pb^T>{=FiMH1L6D0tcbw1YjargOMiG!XF0i&
zr-X@0P}uTbVdJaR)Kn%=i`<s{>Xp!t(sP8w`)buriOnOC>t?BHIjlX)&z)F#?(M)m
z!`lZlinZrHiz=8_-TQGmhDAgu+!xLj_+e(@@e1<q#l=7E-eFQBy_AM#cx5>mds-U1
zUSzgzP2#1Da3_UE!NROF7uyVO@T^m4Lg@3)bESp6F7!eTnGW$a%Ffiyz0fnGyx-u~
zwHZt83xNPUVX=Uow+Fl@-gC%#B64jbKX@;nHW%3;b1LW3KB4zt29Ff8o=(<e!mTrA
z4V*47J$OH5IrZ+RgB9=~IMguk%g@Q<-=PjE$7O{K@x`pzNqt4AYV)AQRpaf)mow>V
zv5Pra7mxX?wn%Ka1`y1y748kmR?uVI@+FB1=_J-9Ia3d@Pd)e=fB)2SuASG8<;jBH
z05bD^w>{wAuSGz8PE_=+6&m<DXfYcu9ILd{omy<;@~aDhDy^Xn!a}X6h5Z|MZd>|t
z$OTAAT`UE#y1riIM6c4Nz@KCZF!$a<^)~@gT|p5PBs6sHHdU<`dWo5yP{VSjo=Rlx
zU#Vi;1|W8eIjIiU*q>a=_9lRz7?$0oFF+Ab2@zm3uSucWn6HCQ#jNB>{ibir3$Xn4
zN*3Q)_TC3yuZR>IfstT@0iUR<`$R}+(-CN4^PpcbE1mmIWTnROvh&I~uiQ4w;2#tw
zkw~%ogld0n8FVjZrMduB)-^%P(~ePJO1&I#KaBjICoXm^cHa!2Lv11z)*$;^*W72n
ztRXv7^%FD2v__XF_$)v)yf}9JNTkT&?A-v+Dlyz&;YY&?{DDNx9mcC8Hq%}a<Z^nW
ze_@XJh7og(c-V9LKoBuTz(JTQfqaE-qHj~Rd!gn``H>gPMxtMQIm42b45YmPVm;o5
z1S`1W(qjY^0WEvv+!kOiCiRt})aJqPV%EM@v8Jtv^|@yy(6iz?U03q2e3F7-r#FTd
z$i>$yZ4o;9@XzOdo76u9R=l~g_<mbXKpnW447j}Hhj2O;l#rq&^zkwv)k*y?E7ZfF
zwaaU}=523$?fiW3)0mh8pnF0$-vAQog~sn)Sz`b~vq=8{J?~~R^!uYt9Wo~oKn~G!
znLpll5URRT8Eg1$Me7a#@w&laQq0JDpqnoyYnoP@+_ci9B2cSA8*b>``yhx2!vo(&
zt1L<2FmRTFCag65<m-w)s3{5=ssg_j|J7oFdZ{HX`M>+LJ>Q)4!w)~)nm8-$Y9ANu
zK3q~C0IlySoqr!@Me4{Mrx=2pG4Bc;qVuCDd0GM5@cA}ZTB)=QrkiLGQ&<yn06xEP
zEzbnbb4xu0*1)ExIJlZD_2_;|)XHRi2do8xTg`o{F&<h!dIV3(iYsXBwn+MpDjIq>
zteo_fm0duWN|M4;e~H#mj2CEw+Fyf}yi(Soc#a`!E~E)QMnZ6}1C4}T3mVz8NhCVX
zDTs$ps_l-ufOw336KC*2&ajW?!`_tDhKRROURf*c<|`}gIU1;;qGZMu)`QHHyj<77
zrcO^O5l35)O?2sk#&2;3ahcv(m~7>pk{M|UR>N7`7EXzK(V>7Ehi?EegB>52K%sFJ
zCeUb+F;|nJl3)YGwuFm0MAyHy0~8p{Ptvvg`EFg0+-cvlaG7&p_o~^DNrSZV{B>8V
zv{~Kkoce&!YsN;x6)_F-dhqAfW-_XX9h_N;W#%XNs4Y&@6OmHg_EZ7=vV1D#20zH@
zL4=-VHD2p2odDkzcHZ+L@+c1t*QAfOBo#E`CopJ{StH>jaLvoGsq9H;2@{uGnCKvp
zR3cLW*H1^N2VC3!PrbZT`Ag`bZghx78?dhw*hS|@`<{8w-$-B#YK4U~?Es2kvL2y%
z!nQL&OhW7Abu`Z|N#K9LXFwhhNQUTWq{xK%-s6}RD`W^>RjUQy!K0iP_ogj)o2q_T
zeruBPju2TOoR$!)4$~MsObiS24FDoby~kV8XAQE90qptH&E>TswD8AT0n#X5;|scE
z(fsz7C`#)QBVkZud~NDtDbzcO+!VZOO%Tv^2#>0cFqLaVivmnkpI6jqwN|0#t#_nI
zEaV9c`Jxaq!L1*y5AZcJYv`sUn`|rgZ>?uMZm}|mBjaE@5D5$E&yis^Xs_sG?|Im6
z31rm{R~1OycfoZIH#8OFe~ekfygLj%Y!FR$HP3AiwWygIbLs%{WEx>51x<9P3PN&;
z3{WYCBh#*ikp{kdjyj0CHd+v^DhGRWX;%kutr&+7I4Fnw)7zm+(i~DLuQ2K`*1-~x
za%pu$WzAtaMgp6hw0WqX?l(oBzZH0c|3%os%E_~i)kz7n&=GL_$Ku8*rn(`3;(aFj
z7$Dr9eed9X%f=5YBM#;V0pzJFps$c>S&l&-CnQS+QX!Aw4P6Lxl8=3V9AYHQCY%RH
z4(KxrASuzctJX$p)xaSDq;C)lGmpl$Zuho}Q4dH#Y=#9MD^t=GCIfl@!z@4>0>LGn
z>?oY$*#TDDamD!8R5I<(ctBtU7m~OeWRvYkRPG|5>>B+}nHuk<&!a+P-#G@|g-sw_
zV6J?v>l_%M(KD6|c6nIQY+tj)kX>9ht`lY+-UiCj-7Tp#0LKy+I>-Pu!c1w9K^#3l
z2^!kL9Y@)0&isUj8VUP_d8!DnEa86BKNK1`t|Nxo3*;i*fV2)Pk|`X+m|)CLMOu;C
z*||P*nYjHZqNe`Ak2%vE!PwmBeB*1=X;=GWyu9%i7cR!E3)WZGfYbBm+f-lECPOQd
z1~0*Eg47~J0E>n-i0vmuQ+!Uc=DXWkLdN}MlG3(_wQ}gzgi&{O2c0AgKP#E+SGi=y
zT;OBg;C$WW&;Hm5k=@=$bZ0+*`^qaMq~%u#RL!sk0%Cc5RZLE9R|_P~?ZnP+6$9*B
z_wgT+3WEz5>XCwv*eg0?ekMcCcWY*iIR_cvI>|KdqwjvqALJ{cJuw5d@wNDg$a53?
zu!F}efC4OxHp{%5)g52Exmr{D!Rk(=w7L^PB*bJflB(Egzbwsk)uxV$m=O=Bf*+rR
zRlB_|pc(KIb8O(E=A&r4eJbXx0UKk6U2i{oa6`13Vtf)bEX`{;D2qUVJSO|RLmABH
zv4`s1@xx*4&>P<NCbDn#K(U?T2j51DSXQSH?(?~Ht<HNtnHOm>^a_d{N3RSYR^wrY
zk&)AS&EehM8P7fnx9#2I9f)xWSg862H@Vqy#!DZJkvAsZ=}+aSpf0$-sOkgQ+8tEv
zm06wXg5v>w&vJTr+5HkNX3fW&-n=fuWXpXew%J9Bj7^nR7_OL`d+L9f8>q=1%$hL$
zbb-+no{o%Y(~Oq^y>6M6sYBXFd%rE{pq=}fI5FlnZXgHK4oG{>UGM4J!F2)1svncp
zbdQ~)m^TP!%tw#BtmHN)jv+g7ny*|DUO+K#py4RWH3cTAi?}10lp;h?j<(0wKIxx&
z7vWR}If^Li^d^4b+2orcKz%^5$$F^^8i+Q7s8TlBIo3!x#UPJAXov3P`XA`csQuP!
zaq-h`MYnvHjI7M>7-f)w;_<jb>^jH)x<_7p50tOQ^j*}Dd7cj{zkpJQo_s*@ImkZ$
zqYZRgx5I105YX|T*FZR@lG6P5HC_wbgtvPZq#)W-wxqvE%SAI2C@3JTX5n)ukW@8S
z{WjwGY@K@kdZo(w0pR|D8J8(POT<LnZZ<i{3<#Z3lE7W3zxVcMPZ085rS#Pmk0bwW
zr}m3=!X;}MWB1_n;@+`)WSKW<%|upU3LziV1<;ImGuI;ljBnPh`a0UhPM{Z<U7(&j
zU24Tjz5#uMO&%$=MLyaosnyXeWH>EF&LTTOI*4!D5Fi^sbjXqxMCDa_>3WckVmyK?
z$1_0bI-Z5Hi5)FShiD^mVVTbX?Snp<071?|pWU97g0DXu<pVr@#dbH6wxD!c$4>>J
z8%3bpv7jYb`%IMduzrwf-HDG|x?mM&KxGy7b3ooes64@2k6a2EkHYg-_7yuM$X>2}
z!s22h5~YIg&|_OD0yrZSD^WsZQ_ley9M`XV`qNgB9^jGib8?TxKEHegH5ibsIevyf
z?AS8!GrqPOtttKZpVY9re<Fd%zBE3a+_AUX-R?=g6h1dn*_#mK>4h2yn*^+O15Cb(
zpyiOU_d%z-`(UJq8;(cx$s#Ml2Kj(+bmSyW1Y5>h$?=B9xbdvy4)wQjn%`{}+bu<;
z+klxNFi(}Of3u5lAVZtxFb&<UdE8ItHr^mEJ0X&8#<?}i9ZB<k=#~7%^LKlUysN?C
zfiVS@CXg8-xyqpm>kgy}^7Oj(%ss&x18kr;8Jq=iJx@Yvc-7-r@wK)UbV|_m5<gZD
zzv|mK@PFxKxzY>|fCFiwyK0ZIWcd&LnDF1c&~YMA6k0@0m}OklkWbe`UFl|WXVZS(
z+dt<#-9L&iY<fck>Av7_mjwbaOdklAcb_Je;z*Irlxv-yyYZXF^z>d69(xA;<OYrf
zm}D?nW36W#)yo|S*r_cD&;?)t<!KS8$s_&1OuTmYL32j~Ss~}98Cp%mI0s-)#{Xl#
zfQrTK-ea0#q)^$=3cgMfX&m<zqQW4QFd$URNSQaTe#q}$Y^}=PeieOm0s~Tb7>LjR
zsuwo}_ZqLX`f7E_gNy%QkBp$82@?mU?UMmEY3|Hf;}p^XHnoBo;^AFx7KxEr^z_{v
zvH?SMiCx%L^=<Ei?+e$>E7~VFh-tI&PUC3U{(v;i(Ay?A9Uv*2>i@Pgt44~LL1NS^
zazTW9oOAqp3=1ogdlcoGee9I0DoRTN2K1Ed1{z8pbQo8w)_}6Err<4n{F$;p^CM*6
zxoJ9V0(9CN=LD3qBI}vaqq$2)ZVO=EGWH0X=^ngfIk8)fPe6tBOz95DA|-Yy)rxwJ
zEYnqLtPngtuuure>AeT;JO0t)86>)MeGr_4X818n`>%+OX)3li*x-38>4KYHNHi5V
z>9T=c`tPfezZ>9aAO^E+suQc+0;lKmu?Dx1FkzjbfjEN-Nur58s{!W%T2n%K8!4tE
zEBQsCQIDVpKZ4+McOmGJfGeeWF~u`)479exXbZBD$ql%Gg#pVTfo5!;UEsIB5-8_I
zk&CMKJ`l{>@X0RZMnRqry?ib-N8V@85m@JgOYFUgxWcB6sHWreWPX?nte8^$y5}QS
zL4)L5bt}noHf3O()8G1p<%==dEHja`d;Rs@=_LtCr1q@Y?O(GNZ-Obw^gd~zfS_AW
zv%e&h;%gBh75-`cbvoc@<wOR?srT+d8`-YR#6e)SHT~miYu@_+XHclGys&g2pwO<s
zwf0cH0=>q%1Ww%*yRf8=9|oE9EJ95xLG_~7Ju}M94j-E-Wlo6-gs<0TmrS$7j`?OO
z$&>}ibo($ut53p+g$>>%<o!XIBZ~qF%mi_6V0bfbdN3M&zWH@0qbd=~*Xhke0@7*O
z``~$gg<qL@&;eF8!iE$2I;S@Rx}c_KA7YQNW@#Cy4q!SdY-fJOk$l><K$$F#FU>6|
zvId7qw?Rip&1TDfNpGr*ncYqu+tLJ?@Es4*g7QoS(@Mw^7ny#>+MVigEcPaK9R4*|
zBIhKaTh9v<`TaQ+&EJuSI>5Qb1_4F~am3nlA*ujh1;O3#!3qo2AK*&T+z5ck*D$Um
zWC|-fBlQZj7Yq~HARa(gqiHr!UfrvV1s{Q+&MUxtHPYMUe&@DU)K9%Q>EYNN#0%9U
zdeMu3qV;;i27jz(ZB^G)ykT>-Ufci>>5zF}rK%KI>UUe_L)Ye!9k_4JEk#sPH%h}k
z!5oV#zXrV)b-~_ij_ggEZSVr+xSMOwbaN~=L(B)8!jK4g963FDZhcVm_z@aGlfv(Q
z^quO@Mi7Cb{4sZGm^`~gTG4&sE~kwFu%I+o1MXPF-}V8Zp29A!JnR6+pOe@V>b6D3
zQdzZ00{n{Y?}5!u20S4$7`+=DmOrrmrnHf8he1pv4iDRnn)3r!azkYGp&<)*)@h`a
zAu6Aj$L#98#ef_h80YfPVV+};H%PdWH{$ezfZVNkcAqZZ4XmH$8Oz@b8<=jrWpcAG
z#vsmbm*f|_C*G--^r(Og7tcBtepOSVhl({?V{NYZoqYOz%F@f*lkgxZr^h}twIXtN
zQg%T@T{GofXBt%iTs>ivt36ulig&cA3ve+L^?f&OB9v6J|7O^6_0A4>cvT*k6KWu^
zISQi~1A8@+yO(3{$>Q{kQYq2M2J;Ki7IgZBOM=%HXlgT+avBISz8IKy_v7zcW+D5_
z{I|vn%_a8jeUMEaeIahx_+L#SU>iHDwjJq%g)WSAP5hXy(mB1RJ@&r}J8JKm_l#07
zTwVbds)2qvZ_|KRxhBDuUpz_;>(uuDZQ1RVyq}6MXkgD$r;U+sac2b}Z~N%DulQyi
zdQ(mWMab`OC<frN`I^CZ;DEHrQEVk#f3Pz6*p@VmjqfqIZXysc%xUqA)e`c@{nG%n
z**9d<4hw`)KW96;g0+0!t{H0lr!IZMu<N>l`QI7yKk2ioewO11#7RboY^dL1OGUKk
zAB1FodO(0n^z*>q{GxyU6o3^M$!LI>D7A)_4oGl^J##RjM)W#B%EeMzaVEAUu7WlI
zt|ft(H}YL8DzxJQbI@tkV_m5!c6bg@X#%jIOQZ<j-Er}cPBYUjfJHYLy$VnIuO3VQ
z8$Bw3t5rN(D?~pA#r@4$xC=1jP4`!f$0Q2~GiQAFiwcd|t|E-9gWI)&j%onIh?Z7u
ztgo*J%KCX2IdiWOq1v#PLBA!C><Tmeg@U9>P2@8030?(0CjiPu?o8eI=422tOr1a4
z07{V27CKG@fYQktWvIm0zjymm8SUz>ZC^B<m6Vl9TsED?BzQLgHG(>@*q@y)06>{E
zS)&BSH4j3ISs9Zx2chO4#5b(WYR2PLc#|7%$c(~zIF0dG#5e%e1GS}UO9?25>a_lL
z0C|00gLhtmMqkuOkH+m=K}7EFJFyJeMgstk0&MO|*vw~98o)`_Kq<t?D?5Z9@Y3gB
z@wE^Y^18;9PxzO&Xm>3Ea6+pfuJm<-tk<56&XGZFZR>bVV7|{ne|u(TW)TnUVE!*y
z^fKO>>b3&lJy=-_fJEQRURr`VBVq8VQv7gMlEV{0AQw9_2FT$$bMSdhlQUJL7iz{F
zv_NbF@|Bj}@W*1h4y+4jnUes(T+QgV?Lf|Of&6~yAi+ZZh?0bm{}HHp^Pp`p>(yk<
zI{>er)PFw<cQ#z^#J-go%sq_kZq>X!Yb^DpO@aq-l-B21WDnOzNfk}^ydH4@-g0;;
za8S&8IjOG%E%e#80@SWVZh&CG<vqFzb0Z_p)dD_;)}~7}RoJ)y7SO>?!NFRZU|!Wc
zMvt`=R7h?f)GcPkPu3{@*|FuB7T>Vct3M(<Ex_y~o_Y`!6{UsXho$O60|PJb;mnbr
z3D~AwFml1XPFUzuVvUtEbyqLcm??jC*VZqqBR#Qab*@uZbt*|=+xyhOJ%sA!w(V)2
zW7YfxN=IZQAL|X5GBPrvc?+1{v^XI5vU^MVlkEab(6E3@iJAxN3w(SNHryX>8C+Y;
z`gKx&FVy?X;QnIPlS%#kQ0kXKqhi+cN&N#*@8&_LRF`GBraW7oT=-@1!Fcd*wN+HK
z%p!t43?!YzBsqj0;*-Gq{cs)y(j?#_1?q5i3#9RlQRJ{Im~(>*5HhYCQhHG%>P%hN
z3q8#ojIS|vrf%<ro?{L^tBG``9$@d^uygBDQ$sG!%fST5i0C>XkkS^Z0N*LCV)xjO
z+;#~eJQo8Rd`uMZSsloYupf`4-+>+!(@g~&jDV9zYKmKXPwI=H7g;0iOcm>e8ZqUc
z{VC1oOR*m_wWVfiv8~GpIQabLjizR1W{6ojpmRvM9Xn-YQsj1ic^VtrINp)%S_9D7
z3T?CI<G7Rh89<P%2INfrCGp4o$(sF8iRQtJf6Vd9%JV)f14_L0;On^PY@Kkgc2)@%
z3DC@{4u+7^#<3Z#1aHjd+DmU|$%wy1%Xs$;dE@I&{4HR25EEs=?sUCvJp_aUlMS5x
z%h(CcTlS_kYd#bzXI~o>JxtvM1lyPn&lI*C-?KEXMV@JZW|DIQZlr7hK7RR`=lapx
zNZEB6<r9K_97g%1jQE$ggYGM$UStgb;x-#5Kp`g*H}4Vpa<Rts?V#<-x2xn<S7pHe
z6#7&tJtR_#Rc6R9tIoCMu*H8kv;T(%wwGr~=>I!rkvq!^PU!zXJm+WC)ePiGujx^R
zK<2Aw*l#rVk2{qP7k&;2KGud#_nrw{8qxi%0KUICIN&%_uX2B2XK$&%L_A$fPD)ye
zC`w<k8gBq9&3Ct39B(vNBA<4_c>a?D&g;TsoaxI0_?tdYry#oY-KiQ|Q)1gZ!x4Af
zrvs;Z6`Z0d%Mz<`=Z^4u=DKYCIO~sz_si?XW_yc{MK4>})7nM*r(f~IXJdPOew%hb
zCc5FK&&e%fFT#BM#Is_<zO9}dep+RdzMw1{4dIxKwS@Fb13Ow3mXZ3iCzuHVqhLi)
z!_b)lyzS<#wfoKkFX*QYym@6}hxsGM6KB@Mhl;u}#ftY!5YaA99)lI&SUruM;jfjM
z^|8jFh6vqU<K4#<YfmqmHhV=Q;xWOg1r~?oKY+K{0=yS+#XfrU+80d)2W870Uydyb
z@stTAUlmT>yooK_DSHtT)ZP0`YxZYf&n=Vt>wq$yMT4Vd)|7sI6l^NETmFNRzkb8b
zqE2zKyuNKI(}x7q)-+v5Et)=d#W}hUw-~@BYc^R()}3AkbM1GMj2)fW5}h%TjTH1?
z=zXCb7pI<ylr;>|Jbmz!2-WB+G?7i|Sn;jrUKfQIsi0Ip`m)C77&oJqT|<?pB*NE7
zmm9mF0KlI=DzxTkIuKiab5Tm|6QiL2cCHKrb7{2)2J}9@wQiO2$D`fT3Kt;~F9CM<
z#dtm3f1y3Lhju3)%No4GUmI}SqsNBD@Xk-yN3~3dPN11Tf5}$3`E7CfRKVbEDH!XD
zypi-av#4)8fQj=dMr$W0$SXF6l?QMS`A!qIRWT(Rdd^I6+-{@P=CAeAie9~6AputN
zBOna6q6S|NOqcZtJQ6TNrXz(2Xn*2B98|M0b`ox9YKMuTS!6Xae>Pg%R;E>X*=o}1
z7g4AcX5r-&66<RLM#4e<J`aC*U8>fU6=jVvMW(CW#{@wPZ?cqH;PB1K5Pu7gFK7@4
zG!MqW76gT`n|_laj^&v(k4!swk7yk}&fcR;fY(TXyZn@lE^KT7E+PW>h${>I2ULmj
zYuBo?Zr7oLE>VqJ2bKI*rdyC2%R?XuvNcx_#NLRbt&zYfzPVmKg%sOQ@ts05Bc&$*
zGA_My={-9r{IHBrp38d6#<9%&xUy#p6;jH94SfXPBzy$}{hfNli-h3*9JAJrM^ArR
zq-~Q{c7uCakm`yk>9!pxb;=UcE*o-qN!4y%5pO8tV&WG0;pjP(f%`?Dd%iJ=SoJR0
zlNU7CK>spS`wz_LY?gFj{lo(AS0yOGZQRot>ix!fs3m21=njeXeT9jUNA{P^(UvrH
zamrH>%o2U5W0i%T(VHPEf*&&t0Yw#K?gt`MDUTXDJWTea)XQ)Kr(*(d+RKLJGoh3K
zJ;lqQL~FGf^5~(eh{A#Ggjf(Uqfk(>ALkWa<%R}(4iUmTq{5ucJs#UiLA;NDREzzT
z;wY=XQ)YLB)cRqjDo*xN26Ohd559Y^#lxkc9nJY+e8jjZU2SF*>*IVBHsF<J@Tn*h
zQG)#DVRCa{1O}piAt-zOv?uR#e*`S11kf0_4$Li_)4mA)-P<Hy#Z<wZK~rsIkN4%9
zZOg|G>EC-kTU+rPi2k48esurI8yfV|2MC*TqcTI~WZ*WVRE4LPDnNA)pc0zIdDU@%
z-gt;<jS4`B$Hd}2UzpeTsA}T*SwC{2$>?WzZvS<FVk|97fRuYM6a~;(%N`pVK9dS}
zvjs|F<cdg9ydtlK;s@63e4bjWjPR?So~a(yq;ESgCFDp)TV}}-yn?kHAzepKifyqy
zRW2%=h!r&4+SAReXQwR*!1^>pfYzVtw&wgCN;PY;Rr@yc20f`6SO3L&YOuGy$3(Wb
zVz1>EAOdAZg;-V5;Do*&TJ*D>IHctsf@pA{xyOR<)U)+AFzdoVh?DswQAhnKAk580
zl&IOe8{kFYGSB=0g&g2&Oqe<W@&)`ZY9}<%YASLgwSPyQT>AP4Hp>UrO+Q{^M<ENP
zF$t%Z&C<h|NE?N7ayx7m;WV0&jJxJ1+ZvauU6MW4p>?vm%)^e6=g?f3Ens=F3a#jV
z*}!^8zoZb7K~q&!PZk{bu|xq}nT5#^S1!Z{k31T)cp#O<3A18ULXPTF`HoA&uf2Lf
z{t#-Uth8t<>c>Rntj={o4ws<$NpO-E%!}N3(QNWPJBarH%BnYM>DZeZyPw#~li?QY
zCQt9`oK(%T*>&WiKv3W<e-~ymV*z!EQ%b)ZQa|+VtLS#uyaB3FxZkR8XhwC=rf)*_
zUDiYASC99Fbx7^F2prKbCN+}2tYu@VW8{4k!6q3N0O4JZFZ2M`zy|{4rQlh%a&MgU
zwhUT9d4VZ&By*21_F%R7gdFz8uX07~*K-gpmqdBOSTAO$c>P#*pPj<esCM+t-<$lP
z`}=er!LsQcZ@(vw4!Hu|7~bXAFO|yy@peGBX|zc+jH!;Hsp#-#jy(p`&hOv{K$TdZ
z4AM@G%?);SQBYcLC$K9?G>Be>t)mcT{g2*UWb9!7NcG{5!F|A5t`7X@@;%>IMX*4@
za^h>VI*J6;TvWvpdEErDo*+1z$y8H~?t*rpbSa(b<Kv&dGEn-2;Iw4Sr2m`}QZ6JT
zG{tP0_Rz2pAS5@=yCipg4~I3W>iq=R0c8(WjfMMqnvD`JjLtRO_7v!Yvt9fQI6_12
zG58z>$)1@rDNO;)n;x%f;E@VKdA&~7(xOe0_%{cZH*ub~R#~E8s4DY^6t*JfvnDwx
zvZ4JOU^aWqXUX{Uh^ugiNRhU<rum-BlK|HNHs7e3(#`FEm%a8&NT4kPxDd9ObY@s5
z1R$^-m}FQO31>T*Ps0O4%QJfS$2xOEmZWW{TA?LhXYS#C;^^_MxfpOi_c#Y<8T%pa
zHRoAriN!uEgiB~9CZ|$D$V;cs1(F#~Kx$Rygz_GYK@V%V#U|@*CB!Nh7+qXe1WwCk
zk91x`x+w9I_piU726$y}E?DqbBM!MiFAHpgOpw~Z%>W9~&s^i``M?%WU+WIg<W>!L
zfA#F70B61gq^-JQdo$#4gd<^+0c0I4YST0&SE1(@U4S?kApFggXl;2Wz0~uY>9^~I
zfr|_#>OUHm&}mk2AMC$3M{<&P^5N6OZ<zAT7D5gW#p@L1R`RwwPVsI?r$`By#5+!T
z1!3mrC+;+STG^TkpLWZSjhxfaCF>l0(FSBVOgK=3cotJHZd#=<oWhHo2Bs>`|Gs5z
zBHnP|N9@>J3Z{Qq{q|LA?Fbiqs+=Y=L>)<Q8ja>~<3^9fK!Lhl`w^~Jvw2fYrh5U{
zIENhFy@Kv6-&^6}bdNHnj`<1c-(eI0k>Q*l{dmyb2^U=oX)(1k<%(My+5_-s>R=i;
z5@)XwIYFJn&dt4v0UR68YXhOcT^b>{#_{GVeRJ`~p_1?-3e^vPXcj=lar#RgJ6aln
zAeo14m~PZ$<$6s90#2SLRD56%j%ASFcUbJOSV?Hb#58<{wqSk|<vq0OzfRsV33GcT
z|7|M1c8<-$*i^uq)4K4!#@_D&jQ8)HpMBRupBz_Od%~$STk9vlHHpRCkri?)q}eUw
zeC<#r$DdUdb<@giB?zY!&UHWx0ogJ=L6tvrmYNp8&7Ud?YA`WTbDkZk%ia@Tb6ZLa
zLhB^|RtMn1y*v$6@g7hP>@*cC7}TfRP)D%-cPNGoouHTSL3ShM8Uwg+W&T;uD^8`+
zx}~$s<vYrO^>P&m=|X{j7`B3=tA$6XX9pGL4Yn0y2jrfi!Vy)}K*XCdAGYDlYBnF<
zY$__T18CLkQmOo&`&nw)fRX+|B_?vvrA3HO380Q2J+y|20Lzz=o04;D-~bQ2v$2)+
zbsi9C;4EJj@#u2U!vVloq!}f4c)L!&*ADOjDaCpvbEfpFj;(jW1l^w0?g~!xVoAGE
z&6PQuT18h>I&UFzuu<!T1B>x;t;oldi&s<Mb<_g*c-iRNYWJw0dx%Wzr|EWsskSN$
zP*hJ9kj4l5BDePzz#FP#w~UUQWwee>V4{*d;P0j>i&s$e<3-0pA5JYZAp=yJ)Divw
zfAPkU;ZS?3YUgI-yg`c5=jauFY_JSJdsaa!cemi<U2Q|5i_UF;%%z%8qh`9tw+4mL
zm=m6hrSA(SfVmLVRDfnoB>?JXiIe@;D+|;3k6gpT_Ju6xa2_pp4XTf*sZ#?OU`pRE
zATZ`r&@RvKs#QnI^mUUEn~;y+<X0GRgMxLXaNX08le(Ag;W?XtJ!1$~LEU8yg1sG`
z3!prW)7{RNT`|>=3ZqyA%16DV3vk907xSb-{MwL#dz9HOIy*$ESo`ep6msymszelF
zY7_{O!VG2&`Ko$rixxOpm}?q#s;(XfnPKj?^W1~ZEazccA5283V>|;q`a=}YwUzZe
z1nbIjFgG|<R?taKd%rti8x0wuz3T;y%4Q(qK?Q{<9<i@0I9;&lE%5^volwY0z&ECm
zbR7ex7uK?-y(?AmJirPA-1}=DKGy{`etsD)3~;7O>jM0`1Jj=`uk4!Mm!8&tHV&$f
zeXft|Hdl-R?l5>=IHh&+Doev!2L!O(`@#=uBGgze*55|3Cv|^2$B;EVkOiV|j+>6#
zZ@#(lW-~z9d%^n!K-pU!Wb09-95et_9UZ(ozsdYOGwDM=DJA6T3j1zkS+@O>=HGvG
zmF_*3h#;VkSPqt`X(q91G{GM=EeAu~NsMX0+K{DCH!L|({9S7iXcbNUO64st1z7GE
zzJ1X+8nDCTh1sk_$%AceT>kH4J@Z|aoM7_UPCY^}bh{rAoIUwVo#2|_RMUfU*Z^P-
z&IG95Tx2iCqd@icqxx~$Q=gAxmmsy1AA`LBhroIGHC7|C57`aaFMru;Rho2^&RR$t
z-%)J`hjZ6tk9w%0=HQJQFX-B$A7U#I@X==^f*ZIW|HhWuyS+{>)d9Nb>)#Npsd|@8
zwJ=TX(ctW@GZW7{G+-IM=0R&e!2Bzv#^rSl=zF84?&w_*mRmj{Za$jM2JVkBK((Xj
zk$tLnffXcQrwdT_He<H+?{-JYM|E_N4JPXLK4=BbFZ~s2mZ*i6ueDP}3lHqo=KJj0
zJ}+lk(_I&NrY-0>EElhss%gRw4svR?&YQH~=bn2B?7=P+Jiuu(^aFM(aE40l{bhY1
zci5lOu_g(KwBBr-Zw@c=v}b(zd<ZpIvA0ZLO_tQbk#Ai_Z|@8#`S|Q>bddcZA4)zm
zehIqIrv|O@3(0A}+O?3djtJ7+d@$WlWy#RhUey81lQQcn5coFnhA%x~Tmth9$<r0O
z_L$j?_SQ2z3<j(#=|s9X_qqDJUm!r>i+&+qqotYMbQZ`k|IgKR|F16PyDtDk5i`5-
zFlwBDul`BHS?cF+y$vn9D<A(*F?gV25TWvGMfA<e`J3X0lw_PLH<(E$%`|N5cm&pp
zcJDm<XbS1BxR0|BA*=f};Bey=$&m|c*9X4Hg&jL_YDWmq<*mB9>pAc9`|rl;lSgRx
zr2|~XQnmdYzMf%FLI#ZwWxcq1Vdd9Dk;4bO?)>G0)vru4e!UuFx%y|TpB~Eke||v4
zVg(P}HriX_`pC*{i_YN#ikDC2oJ-1CvL-@8PoD3dkf^%q*M8;D-Uk~mXjGamTUDXU
zFFG$HdJ>bvV=st_UAz3@+wJAkw$T0G-bO6c6?Dq_U%Rl>tv_2ipwX!im^GG>dG)mL
z_7`scuQx8Yl5sHS4gcd24N<v7Ny+ya*D_nKtu%l3{ANa^eUA3e_b=iN*JbUtf43Ya
z5;}On?LBz%0``>g@)&=f*uMOT;gzdKyj%bGL&;vi2v`Co1!B&>8*G)^I+4D_P<Yk9
zYxUabT45+q5c$sogd?(qg{LmmpfUfZ0Wc)zrXeD$tmWTz9(82<hMt0}1k``8r2ac6
z9U2eT*VmtulpIPl`O8||ws*~8<*vW1frldRQ~$b*ev!=Zy&Ci9N<3?F|LghxB`-Ms
z*cRn=8?mf7_dkn?UDjSy@S9t+&5^O2q*dy_f4|CH4pItDnym=tx(&FXk)-P4xL`@&
zv?te_pQzu|UV2)1suIo%WgcA8wabqg=kktrIgTxu9)%<I%akeKf-85HLOav~co*y{
zif>4P-Z{Ah8tg!77UQQ*1)8$FFEx~UAgl^Z>toDa9GNW**1b(Cq87ET>>oFdBJ^d{
zk^PUO+FUDkSfHkx!i`&sDqq%kNp$mFwb8HBqQ+&U(wwS&VTa#Q^&dYP?dqYcKziBo
z1}l?)eU}Dbo5+oG<g|RWEzR!Tw6ikIB;&dQeEybR*!*$mj-^iTj9^f$9E&s)WHwUw
zE6&s<yy7T=-VeSj7+@<+s1|z$7BS;0UZtB*vu*K?W0cT5SAuv$JGdoO59$vNf}?Vx
z#)D^AT^=d+)xkS9u?n(S3As_`^7XU#tHN%RtM;50zGem(!w)BJ<bq@S(xx!Zk3^ny
zC`jJH1BF2+cOQ7N%vxXjrMcjRd7B|~r(|%EWNGst%q}H0mIG;RvJ+gN#wO2?q`Y@c
z@p9$eJ@(F*;9pQK@Ktf4nl{Cx-(sFIMbr@FMX%Mm-mtGsvDMB11ns!c=FtjcXHcK;
zEbXIvM+XW7keCK@POOvDqwtDXf?>F+Ig>HB;BizC?^zjR6^P!XsAxF)7O<$hLwc_s
z4&tDHS^%~`^_pRZ{bErS&VX|idZ4cTJw9_VukpTEThj2^8uq3MXHzo`O55qDS5G=C
z;+D?gwO%@?Z6-K<O?TD34;a=t1Cw65KA;`xo^rk*{*7ot#VPvKBRLgs2qdZxJMFG)
zVE_176}RXYbNRux%|!2}-|3STup>hyVVIrjQ#=j#iNcm|^J>rn&#4<n;RanH*u*GN
znsX!u>U9+B4}so3JoXaBpJMlwG`zw-l2bwl7xTIVg_B{@7gyG<#@7kRD0OWKR7`A}
z^7bu13oj<AG<^Mfp==!oBt_mL_rwUR6Qm4%Z-R3j=GsO&O}Z*OatV_~1`Gmu_-D_U
znrC;#*p=8PNnz-vB@^MCRTj{#=0o!8M2BweNNZwlm7M?K(@*sPZ7)T5Vz%=7h^|%Y
zuc9h~i?J8+@<bdHpVw^Di5eKSIf@HZoll*oYzdz^wQc6c1SHT%R0?S@_`|V0w3QsO
zfKu?3*1Ll)P;W;*t|Cs~3$ce8pmb`PM!dx-Qd*s&+c^TQ>1(Hj8_quxzRFn4>BzT5
z73cRLll11$eg*1W1cx*_`8m~FwTU2ui);9X5FBjGR&wHk8#7<(^CrZWPpzY!Y2kL!
zgc4%?tiU<xDa(##066L74@_vB6D5EZQnPk??rGL9(LsHM6_P9kYIRfh9<B?Ar|jkv
z-?A}MaKrMq?i)TsF0It!(+J#0ew(At<(W_2o0a=hiKCXH?zDLaW}gf8XclvRqAlp#
zsuj*WDhJ%-=#K9|$#0cBXm)t(M+^J%dwXBLd|8~|o03%Xyw)RZuI-!FLav`0+n#CN
zHc2J+i2qWpj&4!aDtxjW!nth5D4ag$dH29>UebN)5yg-4=IEborY`)lx?;}jg}UkL
z(J`wi%o+!Tq(m_p6zhby?`EBROmtX-9A%&O3W%r}5S@r=@<GuTdacb`DOS?&O2{c1
zD9&RvZE65GNGGAzD>(GBRnp^y5)%OD#O(zmQlwx2zBL-x?O00Ys|-jkc7DTg6MGC?
zpRhJuChHB$_o*P1Zv8x0{He%tf?0+mbGzgh$R%)3>)TRO1c%Z5CV+o|aU|R>XV69%
z!&^9_#$nkPJS29{fG^_cgnaS#{Rr`#HDOfOONHX?$_N9CBNElcbbGxOv(|Oc{gH^0
zmuVBP&r$ELYEegLNxf;oLy;#58->JeR;?RBs!Aq5zIGI{GSk)9I2*<C&Uh9Q^zv^P
zdUv-vdQ{xg7jO;38H1Y)MQ^Ox<2jGaaxJ?i<9v%hm?a>W<<%27TtoE_>1ZWO^<co1
z1^;2nBqf8{SL*~d-Ag7coU1&Q?e*A*N@l#r+m;gP=q7+#kvFlHH(Ttgx;u>*1pY3M
z!#ApV`T568)2M>f&Bjh=Vl#~kFbs09)1pe%&P*vQenfiC8jaT{QuOJsVB-0rGL6cB
zhA;vCzrFYKkE|OB7w3qHdGPE2)y9bYYhR26yiUMlC6(X|T9_C5%#5gy$`?}h%i!d8
z>&ho7+?HT@(+5mb1zkU|_)<zKij;|y!dwl5Y7Y(uV+lTNK-v5GSGyE;z85fjQ$us9
zyhevOl0vJ8y~<s1I(~R|+{s4^BQSFEnG-y1`aOCIhi~njEOm9&Wctw3rq#wEReFpo
zsgq#7OAK!^ILu#FAhugAL<d?>iaS<`!_Yo^Fs&VQ{nff*0yLc%+@qI-*-7XzCTqcc
z4i^N5)`S<THh$|zm;Ka3>}y25bx4^`jV|h@V=DXgzL$k|0<871f;`qMn}GRz3$=7`
z0kx4--{@C?OFGWsfHYaOw)uN-gGEVZxH(vwWVojMcqkHD76K$z+Z(jNBgXZu#wGSE
zwwGY<ofbCICaOMIS<@e$J`{04ch(B_1dM;rvc9ovKww>CL)cs+qvls8v$cHwxPWph
zC`CO<rHNLAUBk+eXlr^M6xTq(BD}mogJ3-afWhwvPMP$jk>o^Mz@$A@$FlSg3&CXl
zCOAxcM=^f7g+5kP-3zDI`HZAl`0&HkeP)Rc#GQBIYM>`%OsU%>%XcSaLnHfbBj>O%
z!C_s#gxP4su<<aL-0Pk)fyGtP=5e^DmpcuePPBc(1x&CPy0j{aa29I4UXW=nVvbf3
z;}-&ChLf}yZO^=p4`cn*swmzMOMLj`ltM6@oKIps0?QNn@tQbPB+FP3Pw7eXOCdb1
zkv9gl!RXv-#EICNmvZ=?=JGx<*-j^J8n&rp?~|8lalb4I^p2$DR^Vr6`9qQ51B)(W
z;<NbLmZaTt3ZE9QcPtN}STeWxX8|193JS#Puh7XmbQ&MBxsvcSMlP0wc+cR^&fHPg
zY7d3CY@cq9HzcT_WabwfWeGh@lT`LaVlAsG*T>Cir(WRwTle$}&?%cKL_Mhlm=y*j
z?=TrwVCg9%8BDR3h0>j$BGPn^s1^%&fqG^<WXlzT@8eBrI0Gi3d73>bz92<DQY9&f
zL82n2j7h`zUefDEh?%E6uF9&)sy(PpU|kYqi*w4Xz+tyGzR{v(58#}j?U(=)B%>Ip
zIrzj3(qX;UJX+rXDH|lmq+dSMudU#XIKHc8ir&QxQ)+Xr<2xZHBR1}~a4kl5xL_6#
zrnUxP44Y5Tmsk|k2C>DT$v6K+PO}oZDt6^tFJ8RZtNU!DSf{{ONoJeEc@YsCUn@TA
z=yqc1Ok`#XPB808P2_R5h@EttMWw9e&V2{?jrko>SXQe<1M4@b(^_S>hX%WMS9^<R
zrIZ3!iMGoSQc-CE5exdc!v47Pi1}N7hKQ4GX#;ki8G=+S--{l`@W5*eu=L<SCE7&@
zYsY$9QI6P@?fSVw=k_h2R0C<UO&zbu4~tAr9i?fd_j!q?A#=hIy19p*HoVTsH`etF
zBqMh%4lU!ni6QWFb7=LPasAwBVO5A*IXF*J@+Tg+2GikudbtfIvkFTSn4T3$?rUjO
zn2^~*1u3?_dTJp<@T_q}XZlHS(Ja}^t{ZAD&D=5~YXt(*x-~P<^?0FFJ%zxML%8<x
z)8D`X55OrkedJHdvft2GF;qRWrKDDnDl`u_+vM$yCYq7o<g34>PF>WUY2SFzW6K28
zO8gEO!}AngsJ?S%N@ARp%-ds&-GdRXix|U=Hx_v+EG&*eDEXnt;?H`Oy=xqz7SBSr
z6%}V@vm!X6$51joW<f)Uv8x=-HBLTUQrh$+QRb-0?m&}_8jmoF10SQe+t`8;jW5aN
z0@0;{jw$%Sk)KU)RL5fk2RA4tHW|7OuD|1*N|P}a`4pAX1bYj^@XQ!#C4uvM5gKr2
z&)D|aDZ|>F>E{bmbZ)nUrEkxt?0CcJ>HugreDcgw1fH9HbnCr2iGlsS;#=Mp^5dji
z)HvDEKK!|;7&JP)s=;WsysRS2kZdWk7Pe(^IP+ps|E!p^3U5{HE4LjvlF3oXwaV3<
zlb0dbCG*;GxcXKgUh&pyE5L>B9p48RLf52u1;m4ZiOtR>DcHv(0Mwc8UGB!iKUImb
z2~=bpo$7LAy~ho)Tbga&{c#wm27hc1BF|4e1ttHJ@5jTXp!R%`7z2p|HDs{D=?m+z
zex17~Q7CU>XhsQ!Np60$`$P<2yNc`%-kfyJ7y1&V#rVwDqEoQj*OtlSPuqp%G8V#q
zW<a4&q!4{xXT=xCb+fKzquA0O-38cc&aOu9E48k@93Krwnpsc^pk0CNf7$`V`G4I3
zZM=TM^4VmM`}#~u^c8~U!I_Pxg;~W=qJqvY60tkefmpGgefymis|F?B4V6_?JT5~n
zp+iSNNlwhF$sc7?Z8thxHe|p&808CI^Z<6^x@1CG8%_p8v|D#;jMohhO`0cOHZ;kQ
z622qBD>=fc()U0T5`0QE3K1{ljBr!GB%|G@m2I1cd#FlI6@^i{a`>0FCq8*^`i81p
z9HmhFN95iR!S#vU-a6f8iEh^&PAy7R`p9-a29LPDci-(LXXGRbC?2yD5EZBkVW-U@
z<T8<64g-HWWUHGOqEN%YKt3j7d}w+hVh~3^^`NEZVpmZ0ukp3B!Lr1j0@fg_?qvIv
zuuc;uF5Xc2=SRlIh0VGTMf&#MH^RurjWUJw5UEp5@ZQQA!@w*)4hIr+W<QQM%=UKj
znRclET_ILe&`-$5Hf?Fh!JO1NA~6T~f#D*XK*S)Dx<@I=2L_T?w!z&dm+|CNP)i<-
zP~fM+_*%E4{e6Viw}3y{Mk#DnX|iDkH_;!x@&Y5vbr8Ad^!T5a{l8;S+4dQ5%K}*5
zAF1&3FfX)IsTdEb*hRP9VPYSr4EcmK&p#PIQ)pxQDelKu81<~}vBd+9`Rd_(*&O`@
z-+|5lh@{Q@sPizNR8gBj_-n_yUPLRvUG>yH!qx$l-_K_whfM-YxgP^wVsX=1$l{qu
z=CL)*UnK7!6Feh^N)()1#<Ncg$24n|d`qxU<7Z=`T-5q$``!uReELFCw9h9t*Lp&0
zqnMa~;Rtvkl2$(7GFHYzs@dA#oSbyF(@%p7+Bix^yBRwX*`))4O<gd4C*);`-xa%z
zCG`C8Z3sX*`Gzs~`?OVJVW*N#A%@-c@>zgilKJo{CO7d&N6|-;r;n{=;I2H05BG83
z)$b#=ns=8pp*lLk7yWeNvY*aqNEo|9&2vR3ZoK_h@A&#IF@P!{=Lg`>@D>;z65=ZQ
zGv58$(#FM2aj9wJ7k8qc#Mh2(MbIyB_xp_5xP{rr^))Fpbe&Qr@N~lG=caZ{AhbKQ
zm2+?g^B07hhBMK!7kPoKFl8g*Z;zW}ek{oNOy*m@+GAS@&h~r6*HzZ!qd;fkwE<m#
za~xAc1Yu9XrjslR_U7+zP_Yd~V}deoXnSJ5*9^w2KB`-w^s8iAqDZ5eC);CFQmiYv
z;0EELG_a*=Lg)l_pLYz5cUIlbMFlZL!XAn2>*gB}(o*^zdTv)NgtihrFirKE>Mu*J
z(>*a`7y99!T0Wu;6hH(jSnW&V3_*tPIu673gM;M93-!^+nz9>&e7$yh2suzpKCgiU
z1stkGhmfJJH0p1wsLxjulcCXz>x2S7tRO#S8qgA6Ee7B}<82@AdMILpgYdA=JTQ^N
zGbzf$ouXt?SV>zo^eEuM&3J>k@@M$!3cI2Du}H5efoW<O<sQALNQ+M!#jf!$VaiJ0
z+5EM{R@<KiGce;(@lKQN&$NQdYM!4utk}-w<(7OuRkA=O6TfSt^83bKuIB*P=@vFN
zGtmi0_B!vqAA_&{g$%~Yx4qWZeW_2h@!6w>78J1MZ<vtU-J6C-I2C^2K)tpaOv^^m
zig@GP1#j0y(Fu7t0aVy4C-Pq;_(i<V%2GXq>o`YleXD-mEC3&joMqPU$j_oyS<%fq
zw`dis43%_Zs(ZE3a^a#B#`{aaM$pYfNq!NPD5#&3v(&Q|SRH!@fL5Iye?5jD-oN)j
zlInI5lP<s$lty_Cd@XZC1j^5=hV`R=g&vLN!8JE+to{&hc+tJ|zJ<?8khmAV2N&Cr
z6X$LlNrqm-zzhcA`QCy;4HhrE`zDlwLqkn6#;cmOP`_w>aUgua5603_DaJJe+U_lz
zCpyTkxBPMh-R@WS{`wMDsCYR#uK0dkkN0;@Iu90D^{{A;2I3coq&%EDEpjX-Qkqnc
ztGxAV42#o{Q_lUU*6z7g2j#I-H(|^9B`MiX0q)5)f!haN-Ph%EbU|OG8QdvE1@v)n
z>Ink?Bi4^e2{aTKe0zS+<5A4mAnWUt+{>?!n9)TPu-m^itf-o_agCASDEMfThieG`
zh#G@UvgWNIH6sz!s{F0(Hna?>%jGAwm&Bpag1(YR<tb2N6lU7r{nuMa>bE^Bc*cq#
zH>^T4ca*yhUfn;z7=1tRI;zjNhE4uq{!m7SgN%6;s*if|`@NIN9J~4IS1{?Zvy@a!
zDrrEAg~U5n+($2=ny@WfB_B&4(YN}Hn&=zx?-w{4jA6<ZIzYZ_XIw6YRhGA0<^Mu4
zr!RhQV>(c&Ac0Pe6J@<4o%0eJteg-`e`iAN1Y|p}_wWPL@TxkZ_KukwmPx+3TRyoB
zpED)Cj;T;3J2Km@*|mU!ZXX7&7p{&p>~K3eX0#5QT&^akUgD<)2Fa)=;6u}^PTYu2
z^t2M~IoV3rKfH>7Ec;%k*G%aEbeLhxUf13uOQDzhYqd>GGH&O~6Z^+y%#<hkY90lb
zwbWL6PC<zEt&BN%Wx%m%aA;ZG_1P4wK{Bcren?=t26>x73x74e>yxsZqs|z=C}8y0
zw+dDy8p50vAt9+jYqry4?<KfM*Xa;5PyfrFaagwf1C2(zbJFg{lJ_BG<kuc=_>ShU
zHs<OYT2Kz0XkY+dJ58c85gzqlzlXIBHbbxmzkm#0NA;>oJKc`AFTj&h1VK(t8Sp74
zx4^0kwV79vm)|&>De5Dzgfs3>KH&ktt<Ft@J>FAn-=m)_wZ0U}-}@_Q8|i~Ke)um3
zQvSW##R?Az1%uY~<@nP5lYgMPiw2&3zm`G4ml)sjo{&&>@^y<(j=8;QIY?x9V8b7X
zcK*HEN1LP_@7_HG6=8afO)g#;7ayvtf=KaPj^(~cUTJsZdLK%6{-pUmAsppOp@ij*
z<s!rTZd?BsM+%nAgpknL|IN{o|H$Ca;Ol?Z!GClC(1-s)_Tcj#uYt-$-f}cUXua}^
zKk@6SR0EjKD(<v@S%*LBu5KYAc};z8`ssQOkJ+kZsM`-(!u!Gk*TT#V`SFHT1rSWS
z{&FJ1)70$Ja^KOfgA;9zCyTMD5<c(3QMk5@I_ll9jVr-jEx=vL1!MO#aUk$`{JEop
z&2o4HcbB_K;deVpG<*)u7c8%56Wdif8L-wix@+`N@I7yxcLgje_54YuT4Sg-nKQb&
zy~I?Xqi^~c2=X%hKkU6{R8wjDH)_X@BZ^8DM`a#S5S3;K5pfh1#nF-86d_2JUIGb<
ziVQ*+8JaXZB25SoYNApiB1A<>NJ0>V0BJ%<LP&e>fcVVwf7Ur4&N}Oy_pCD?V}YG}
z?|t9b^=nrpUR+(4V9h2N$@YFUx}5OUVoAv?n5sJbr750-nfofQ-p=^znw%h&hpk>E
zu2DhD5Qlf4C-XiZqabTGBJ#=vzkuj1WnLLP$U@);N&*d2myT*u%fxWRx3*(ZFqStv
zfUUVSFDoMo=emA@Rv2?Y&l)X_J{eG$km--qa<Fx)nYCELQqj~zAGg>)AGvt^SNE&C
z^<ku^)gU>gb*La_={Lp&{TR66C@y+~08tu5D=yT_Q**o4sH}sIvi(b=*mF3HzSSRa
zdcNV%9`~n)-ND7SyZxEp>~{~bKk1rZ?{29fhd*RgwT)-ux?e~oXMSak`zsY^#Qgor
z4%?k)^54Xc>#n%BdSxX8j8~i_!Sg-Ju54A|MSafhj`$P0f3?f!$6;VHO~L(8_$AS?
zaUH{Z^f{ntaMue|DCVFZDU>8pFF7H(<o|r2P6b(8&O6${>y8jdB4^Bg!k|#K?hGEd
z-C2#YW@b>3S01XP_1A*nIqI=OPCZVq9mlRwtNiSatdP`sp6P5IrpLews_=;3y0WYq
zRWzQMM}A~VN0cqpc#nPRQ;slPbUIMu0n7^D_OvWJ3jcI^<EdLqXBnTKo-VJbxaap;
z+R<DKEV1V>ozymsJ)gK)#<HF-TScUVtujMIv9*L>Ykasm(keT&H1e;<W4rR}f7^dV
zCVvWI^V8ED!d=cI?n7%diCcnf>9o7GoUppp$g1$tV8lb=+x7}&G$#}tjLU;k-1nu!
z>j_xr6t@eUZ>UG<c`?P3m}<o7*o6A>8kPbZYGjlxI-S?{DTmXj^u6RsDs{3PoVEI#
zp4W`goW-q0IVH|M7Lf<;x%GQ5u}EFL{K-V4U?5ShCU7i==|z5sp*z86WPCz9Db9rE
zl8iOV(@cCUlm2$s`b_H#>h6hsh^R_2m?Tusn?sCe653pEnlv_D{8sPioXPHVW<v*J
zPKpYuC^6G;ccF(fB~P<70J7IL!GYelJ8YeG@qAN@JuhOeQ<rzYRxivnFR&zj!TzU$
zaRT!NzPHXn_EWv`L02<U-RINj9*KKh`{?xN0`!TnlhYL^moNEyv&*fp-I$q#9xzck
zfeO{#jc?az&yq+~vn|HLTYuh$Q~MXJC2d(4d(MgQ?E*$+4YnV1FRr+b0$G+XBrn?H
zS-Owli#X2xWE1Uz)<&^P&0=JehN)An!!`98?g$44llv~s!Poh;+pf_m5t%oANEB2L
zP@l;`O-a$Rzeb`t=mmZ;q@-nHBKFUM*=V-Lf!aC~*BcK#O`hCX6!%$P>q7U}{H7A3
zN0=#V4@sM~{%))TWMNCRcMbIU@x3xZh!xg<F|}BT@=)abA{)@V>LqC!uAw1`*?>l&
zu9h&YhhDoR@6+6v`P?tpC<gHBxsr8Zc?{Q3y{JavMVZFwAe`XjLW8gh>e^g;D&{Ii
zuW-?;2Rsld+zJLEV4&soA6Kt(lBUhmut%<bEz`BuO?7?HO=<8P+P?VK>ux-A{Xn?O
z6kc#j>SwF=c6zTw){@$H588La@&UlYas{=vWrNY2jD?ziW3}VS!OlnIb!$(0&)|FK
zx11AUM$P5|Zq#Q+Nk1o&27Jd{ByUJcZErjg)81HFZ4xdrRZQZG4i!!AKwQ&PQ6S+y
zlEN5t(^yGP;?kJpA@v$-`FnD1m+%WOw<UrfZK8|=O<-#8VQ#8VYFD5v@s=IZjFR0)
z=M2Lpd$rcS4#!A$j_$4?MvWj%rCNw+RUcG7YoG~Lp{zre#0*Vbrm&a-&(C?g&DzY4
zE`b&)t-W-c_lYQ2ywGvHc%a>pR+P~H3;fx;T?W%M>s%<p&`iX{_?A6kZ33xrMR=2n
zNy@|W;XvVU{A)5pT1T_cRed1Zx!wf<F`G4=)FEIqhFJjkP|v`3XtcBRE=sOnpLNwE
z6k=xw0+>8w>Ff6?He7)8ThoVTh^<ky$M*3wpt50JT$&$`fT(qIM@^)I5e;VPWY5Rz
z3FYCiKeE2Y$V3BJGgVDF1hW?}Sx<K#_2u8k9PB{7ATUwWc$r;;6i+OmK}h)YrDChf
zs_K(5$+FC(*~hGb@S<BVH7bQ~oAL8zgK<+?=Zqm8j^RLI;XH@C`P66D1q633S-XTz
zA5Kc5haK*Ta*7EL)(nW=pADV!eee2=v63DO|0S<DAu|uL(IR1WaoI7JCjx&#U$!wl
z$|WYGo@>8C^<Eh`P?U20O07Cu-*k1`$@CFXG5hlN5ee^ke0t5s3sq^;NUn}yW%{h<
zmA2cLZZC6rEi_xXaHfwpMr>3(d=(53X{NwVm>j&QTX<b&J~6k??r8qA$B4r6YJN}h
zKQcJfBra2=`gJ<hsSyC83ji6ycUm)4p|FG1DvE!i0ES44g;&%_5z%{HWZ2ZeXTm(X
zb$C419gkC6L-G-5Z@KH$^TkL`6ByBkC8?AA?2?Cq+R=~I>moGPt0yM9!n40Z>r}J>
zWH)+EEt39wj&)9xzwzUJy%?Vk7?wZT|EF<aU1W}C&L2vugj%8a3(bjoU?u1FvQked
z8z(3XPj3F=OSUM>y;~$PzrOhB5`;9%E3&e-W7Nk5?as*?7w$bV;Fl*$l5jLOj;$Zh
zCg4Z5bl<9(vEiGw(c!~lq9akAHEu7;0gSX)Se_BgR^*h_m<=PcVwf*?w1y7{UUnxN
z$!XR(!1~-#hAbi_GmGg&UjIg+T0^I*uv5DGcKKh@(g}UqwtKYXTi0!`I8$o`vd%9F
zKvR@e#(*sQB`U!6CAVa~!+p9}DZ=Mw+^uCUEkd)c3wNcKw+lCAV3OakBTaI0-K?Sm
z$c=HK)P?FG6zYPO(wzW#JUpCj8-iB%GaHZ<S%hbomp2896{`kZb%U1D_Sv2cfL!q>
zjf5?Ab-nAZO4dj?h#D6evFA9`{SaksS%VCv3P;h_A6HA{CTw0bfhrj(J<EYA5|A*m
z2$}*>bXT=y(L>|A-DP2Bo+8>r<UU>LVaK*Xz!Q^$Z*Vyf<rB!R@Lh<mH&2UE9o#_A
z7g$to%8_7lxsi2i1Zt%gih|E&vRI3>u0v&3mA5DC?JhHWw><6<w1-Oc;1zC`6EPS1
zEoRGcgQqWqi37yqq$El1>}gKzRmoj-zeXZ48``Gfoh2M(8N;f0@QOf@m9pj7x*L1N
zlYOI;xKRCV4u!Ju=47J0#yVnuDBZITa}TJ>UjB}*5-$@Hcy=*$8!)N0S?|&kG9@Bu
zuhN)(D{9E!RBDEJbfjeeF*=k39UyzJ^x@gfOdn@+14_tM+8@2j_P<OLpITwN+tWQV
zo~&N1OJ%Lwt+svVw>_N#^B?%HkAVh_F#s#TKKmqyzeGE4A-&YSfSG7dHBtFTsvv(s
z@2y-}8G<E+o)oz9*O?2{H+nUH4KRuyCGR5+Y-`{<bvG*Hx+8w>q-cEYeJaV1pRG!@
zU-@%j6_U(tGfrFAgNReC6~th|GpXICt_7GBbDD7M=Cb<|Dtiwfu+$z}akwIu0w6?%
zPGaHL(P2nV;@JrI2cUjB=n2!obD$8G;BY9P(s~UKYbXcQODfqfKucE|#0?fqKUeZP
zS+e7!!~`~35F>dahX7xBMSi>kmqn{tht<_cT26hQGQXZJs-FpJG}FXFbj+-q!RVX2
zLM!PWX=BOWM!xgDXI!1%UGx7KhhX#tBXKn-pFO2jngQKkefM%A)+21|^JcE-ri)~I
zM1<p}!k}m2_>z7Q<=maaiCZX_(mpvrM`b?Qo0r`W{pJxkmZ8E{EV{XDmIrcO!#hKl
zEO9LD;f(nQi(war2cL~19f$&5BCV2^0+bDB{d{}t*G8lH7Y8LAlLsSc+CT*7bs2kr
z6X~_?n-P{=_vT0$|E1`|xOlUKDuB+$v(S|WW3*__K`8yP(#A2xYhOb<1rLpbA&6p+
z*pwq$^haIf*MP|>JeaK&pGWLNmf3857}-#==s59@#Vop#&IaFts-Y&Cw_lg+pd)%R
z+)xw7C*Uv?vv_y-<scY7ZT@lWrVo3<aY4ZxL6xiX-TGQB;puGRS2RNL%WWR2`|^uM
zYFc5Yb5wQc@*A3}<)#_+>aUxrb`-5&UP=$)i8g2El$~Lvu#o*fVFRqgU5&74#{>8m
zT<8x8Pldgx*GNCOQMA0zwM4I3qIah$?0l0XE}gv2w*W%%qn*YX?1V}u&$>^s4aw^-
z#H(){qrj!t)Py4+jqks3ZOC($AWd7rh|K^!qhcxbf$g6hfvImWFY>5%&4=Jn=^_q!
z>X(>Oh}L!WrK{Hjniz;Rw(opN3oiGodEi0F&%Sx~t(IVx@6Nevgh~hyKr|0uMc(cq
z%EFpNBc!deVWBbatQ212D1l+f)bkHwbZ$_W?3gPpphe}^iXiT_B9=z!qjARsw+ZgZ
z=zz=Zs)*<^5l*0~czE**LNq->8#Fs-PmsfjGSY}T+}cN$$#BknfZzShovrCpe!qTm
zb8{1P_W__b(^%-oE=CsrDk&m23Vg;3eF9}459+MkwA0`#%+k3o<X<*6{6FuBPL8b)
zcNAc0TfL=XtM}8?Ex2B!!kaUIL<!*=S*|m-WS4i9tgpt<xcgs)Wi7q6tlFd(Y|zyv
znie4DS6-B8*wUz!0PQ;;qh&pAcu8bGAhx4%VH=*pyKko-e=n8V>2BJuTqYaxQi}mc
zm~7BWO(q;2c0~HCqy~Ea9FmG3c{N94_o<QxCgu?A7rq{)5T9CMcWfL3cb(T<kf;~n
zCW+SahjS1fmv!^j8gF#~(0X|5p|=xD8P=Ce56<K#-%^()<F5J{q~r6Fvf9`s6=i2S
zo0+}s<@a~31$Kj?E1Ek~CeMQ?(S<_m)E7dxT|=+rJoS7pd?*`b4|`x)Xf8ueEcAwm
zUHgolH1>$O+1V%!5wGRBpXl`4@2f*0t9`3Z|3gz&t-+h^tP)(LWVnlavpJ1j8>NkW
zgodY4nO%SxazYQHIez`<lHB017xZ^C;AJAKxr00_bz)1N#S-d<BZx5=P+vu112|l2
zkt+Uyo>RYRucQc4$;Sk05RzMJ3rBGICAn{QC1r|Pp5LBv4|e1H-n_zA%AcKBZ~MZL
z(S23sx@<=v%&l)?C&=!jXA_x)Z%PQM_%NDiG-K&)mHQK}RHm;{AKfW&gv8`^^mm<u
z70LyfB653)y32@ALt~+Vf9tb+S&%DI63QAlYMQF}i<9@@_gZ@Rf$S&Azs6MAW&zS*
z?O(1%W}A<GUyHqH!ekO_s=Z0Em&vI}+hmyjShE>Pv=HnO7@746dsZBT8=(usug<bX
z?2!5!XJ0*_<HRK~{{Cc*WbM$|&*8XuP=4^E3D=GXlRI8D4FZz6JUKU!)U#P!LROj1
zN#uj-K<ke%>$A->;iN9rur+Arrj1_SWoJ_wIiBafoXeT%8kE83c$*hHfd@Kf*Y&E1
zsb2O0vF&c3@~Hy~JuuT`5leszLI{dP!BhFco>yPm|BB2eqt{2Uq=H2KR2JjiF>V8B
zdYGH@Ia(VR+Sh8EQ$F>vvE(%kR)r5`|IxuTUlV*BG0t=T6*-eTnYWnx45&i9)`b|Q
zUxjTtfy8s&V(@aElX(a-aweB4GYw(e!ov3K#)?{|gjaI<$C~VNvfjxyl45AArRy+(
zW@``)AH#8nwXW}5I$27hX_BHlfQmry*TLzOBjML@BaMgUa=<m$n7ej<5N4UEY>`z)
zE=S28pty~MA&P5hYhjNRVdLvHF16Zidw9m*^sKL1uc6j;8%;z16CYMPR>~;#6>}KO
zU)^$*0R0zN(`|0I@8RoWutg&XP8Z4}$S~b?>b{_84Hm)y?TopZt1WU6HhC>2@pjK}
z+ZL+dign7q1=5QG%DU<$Z;D{FrXPZDkE8%j7)LR+EMfe43|ttS@h=0h<`X#(#=8Fc
zX?%SWgg#`+pZODWFeYxy_NR?+_lb~WmN~WQDsOup^CwRP8t^_JQ9jH;jddJslkBzb
z_<B@H!B{bgG~fuCmL>BaMefCS+Wgd4@yJ4V=QXdr$+Cw%*kwqlXQ!uyv>eDr^j3aY
zx9UlN&Tn`sThiB)wtDr-Rh8+3uyjAfS8Qqw%qQu&<V!@4;IqguU)BXAG}4yLlgl)e
zrPo5->)iv?eWV=KKq;4*P|Z9g!_rL3C-IK9YsN}<?hmUA7`f$yc1Q!tGV~X8iNN1X
zMn0%Y_b(Q1^O)m+p5M$X>Q<<6MG@145WVa8y1%kXR+OZc4CqTB9S&BCQ};OqP%m<|
zJ$B2AngSGbG@zQsIb}LSkm@-)qZ|?o`3S|DC2w?KG>aelY<s=D=D1FPVm*7r|2(@`
z&vDP9|Ij$I;_E?G&vxT}XV}jeiv2-9Vcjh6M1-EXkpjC0Twu4vrSy<>M2Nvz&n8h%
zikOFjY=b*+CZ%o0d`!;~V_D?-YZ?wgI)-X;2EGyDIgH7H#h8lrc0^%nsp+nRbsWSL
z(-Yl_42$MEg(PEm;y;bER$d4>x&&{6-S)c|PB7V3!j0H{R!U~%UKjsKbMniZH2W1)
zX3A#l6^p_D+j|yAJDhhRFnSoHmJqCo`U^?p2T4l~=WfN(^Q}vN|6t1I*A9%@C{wSU
z;=<V_6k?N4NlHttF&A%`PuAU;qG-M|ZVpX7=Kv1NddP(=A^B|T2+uC7omzmJ%8p2f
zb_MXrqQDYK9l)Z#!SqmPG5i^%z)nGBN)U<LfCX{#wcCPeEb+Fe+taq7n9c7kEtU9~
z`O`<8FDNqVOJn!1rX%`(z-}+JKc*GEPItT1ciueqcYshK<kX(84j)ec;t#yX%AhsK
zUUx}x{|2*jU`$1o!>dB${%y>#A_FHgUSJ95)O5`AdC6~0fGmE!da`=~H;x;(uGJ{o
zNJ&#7`bbHevaebpfD^DzBlto=3<G6?)qmL;KwG^ID1T#igKYrQ@$;<zfzHOz8aO#g
zAVrg=BlrHY^JV#tBag~*iRDyL50E+dX8$bvtzF~2kKQDWV$0zAf2>P0=OZB$_5(ga
zi8|PNJgMcQEV!1G{~GB@ptV&a7?)`;<~-l${28>4mX1y*4{lF~<0u(sIMmieuGN}3
z_Ff8wF6T6^2KjhFrL9w24(iAF9QMT7X<X~e$G<>b&bQ@Z{WVkSTX@YO@Ann-Y;{hA
zpT^V@qjx>S^)qNeWrTj&UV@RJ0Ace!Zzd$Lmk1Gap(;OC7Yv><7T&M<DBgk-Y9_y)
z$c|IgCZ()iHr@O5bjD>5k6q*&`FY1bGunTvKd4kS62&Dh-#aLM#M4-#k0?-X=S3ZI
zxq{yE@Z;>-IGnBbcA9C=h{OA6+)GC7*s4`mR8;9fhRP0{TqxiBD>0I7d01!3&Lt~U
z>Fix<p$z)N$Yi*t?unS#ky`l~I#}?K<H9ewU-MI!K%-fl!JQl&2%(HQwne_EzK5%m
zhzNm3vVLp!iIQ4$4{}Ax2o7KG2#OElpqP!VmpNZ18XD0BFi|9M+*k*hU$=42pD;1H
z&>KuN@JV@%JS}1P8uHqb@`$~qJ_6+-l0T}z`IS&(^Md&N<7e@`&X_<p-n#>;zqat)
zk^0U&sA_%;-=2SJ=3gp<FC^+ujxKrZFC4}ip@p^?*nuT;HBv-N&oJpR<sOgRQ<W*K
z!-YP*UR_YtY=|i8@A7rcs=^cbrZ7yUqvZpE@@lLmOL%-~ql;-`i;Ug)2Ij1!4;5Sy
z1$V)pruUz~loFf8Q{IPfek_aqo4(-ZaS_-dqsv*1H<#VswRyc)_ncI}jDHv&!9!lw
zx=!yPibhBEHARnGX64u9cBSMyXW#R??*420p7~SJ3TO>N>6#nL`jswafQ<;?dW(Fp
zbaxMI5re_4C2LdA1lSd{=7(8<PWjA;@B3*dAwE91t3YX^SBce!enM~J<X2SBiZu$A
zKp_e`p?Upu(oU`Ge|BlEpZURQe7j&<l_Ix%aDWgi-Z3Eb!%WzLP6=7w9Y^YVdfb%X
z>qULH7I4y?t5)63i9Rwo%VUrpzck{-gUO!;mL07@g|~?p*Q^SG5kbi0@LN4}>qk%Z
z$P$Si#s%`zFuEdR?f`wYe&Jc(f5*mHo=93=S@{=q38Ht~ZH_|#VI%;Pbn2nE*7f6i
zokYK!l}=cyv8S(K*5f00oRtI~NDp^hGB1@l@cl8^v$a`nzJa}7Y+81r(ag@i1$nc<
zYfNG8sznP&!d!u~Xwfggy)x$}`C)Zk_$hP$9Jpl{O5FL+i~iX0ozwc~2mIUY`_Cf&
z^B{ru<DZx5pEZzglmFMBgB#{@Fj}NEzuB%`a62!LGme=Jar*8_JTd>bZs7~(eubSY
z=+Q@7W=7kEnZ(}S#BwLsB%UT_%lSLNeC6C*=1SC<m#rP|>~e6b4e|&Gh3l)3M#=Uv
z<Su1=PEwZRg}@RFt17e3Kndfj+xpHQ(uclbb{ehOJ4AwQ8P3*Y6GG^ij^-f&iM!N6
zls_`#8MgabfK>>3WcGI}8e+qm_M&oNhIUJ0Mx%m$-d!O%L&;3eA_*g$Xxsspe=UFl
zmW~0<sb%nCK989gQUEHYa3AzADbd&Q_8#Ct?*1PtyW_3l!Q=Mt&Q^7~NeF-&r*-9;
z^Y{ioVKqycF<Wt|mr&^%peN@^fFYXc4(z>Kh?l<%R5hDL`3$8hlsDY>!#u8fj)JDH
zx%`9yw2PnKw=5jCKeOg4l5v(@FNZL6#PucfR}@$)FmUpt*QTG=&CBulkIN6G<~?No
zPF~+*>ps~<Nt-P3b_ULv?*apx&3$r#!P<Sl?57a#aph+1C!Aov-+er80{sAQd%A)O
z$`=aDwiOXuK<LknHr%&YJd&MMi+t!R)W-WlBiV}LrD4s;yr1U!6xgKmbq&#O(+gPm
z@qXmdB_;R!SZe@D53(IX`ApIb6GN=9J$efYj`o3KIteycL)?8ZYik5Kn40x8&S?y*
z!Hcg2NEaxljqaoQ_O3~Ue7XWz3HUm{G%vd?un%$p#><MB;%ovWMwV?S&95rS_uJB_
zb-OT$+ZDhfU4B105HOUeO~w&{aX47V(5jhLW^=iOlTJ<=sm<$Fe5TPOh^FXeFBCve
zcsG~r9%|K=bC|-+d<LG@(Nib%fRa}=rL0!?cO|~$owru2Ov2lWVA<v1dqwS(_auwJ
zF`zZ*yn?=4ddo+%r3Hdhkk@~M1zt~kax*^Xc`CAjMWKe8)ku-c2~>Usv|KJ*<Wk$X
z|1TX%Jm3ECY^gZEMs*F5uj=)jAnxE#o@u5Ags;^9Om}}F$+Pt*9G$vcIVughbfd8U
zU3HU;#~B^+grg8O25)p0iW!xwz?{m<dcLpZ*w4IuHM_FXi?X>LW#>Vyy83Tw>Khwp
z`9TfT)k}81S@d+}s?R=wLZcIg$y^pX12cEovH8iz{q10l9s8s97p>`{*0whi>S;#@
zukH)qEcV^28KHmuOH6uoo(@l&t-)085fbYePTu=8XzecK;;%1O@`UU`6hI}JHKwoR
z-t4CJMtBugVdjsOE-CY~KkAXlOM4!A-a)9+ii-9+{T~8?hMjsU<=5yzIWkFipDZF;
zt27S@NtRAsagx>@(^(a5n^>gTpdL?NtT68fo=I>>y5-2SZ!<65AF%FWeQ-t=e%<~v
zu8zs1&K%FJwpc|5rgEh0Gi+wxPf<rV1=p8VQfB!60v@K_@#vN!R6DiKEml$@3ocbX
zUt^Gyruw5md~qYKmbvC-0lRfr-6r%>0O)?rC*(D<#^A1+VE^DF(xP@ivG@iY0^_@C
zW&+9xb`Z61{lI3;?SQSBlTf{~{&?#1Vaz=TZa%;0r*;`sy$fsTBXtEPop*S_evsob
zyEjXmHmiVBKk9mCN7aENc(F*^M&m8b;nD*M@8*Hyq9OYQ?r-x~{Gob-n=-o(fBvU4
zdDECS$ni=DE^(K`7IKTBUi$D9(9OvA*+Fd`m@$%Mn^C6QY9(sazP-O-+Z4PI6$*F^
z=LCTH)nUaY8%mF{uJXT~b-<E3T~H(MDCt}b7<BddvEj!7Ph+h6j|1U_c<OY=*FEJ{
zYeX3;QchJXf$;d_%Z4Twg9+YqxLi17{G49G3NxYWsZb?wMV=RPQW6Yg8jXU>DH68M
z&J<R7tOYc}a=T#L3W<l`<9LW4X2U_${iKo3NwgRE<KRP|^Vi5P{DCcULkwI$59*qH
z3+&$0U#mLTj?AmiwbJty!|zo9RD%xPEuGyN*6>ui{r0TAjeFp3a&KC;KCSMqj8OF)
zmWL)mx&j{qGu3=c`v#_3uZBXV#Yt{1t1hj43G7g2H&&P3`TlsmVA{ySun2tGS*kDc
zCk>F||Mkgjsi+VGC$_@#W71h^e%(I$NWNSRu<j`&)Hpd;1AVLcB@P}dX;9umGog^!
zl-!Qqy(lq%)LqqLIts7|P0|lO0fZBRisi{JdkNhbLR7vc0(w?lp1xj54uCY01+zAd
zVNYs_OaqbrD<mWzYA^4aj9j<KH9x`VZea0Dt|kU*+od>X(Te$zKIDJxeEE%@8hFLM
zUhWM!%J+H}sjCg^>nAn^<o?mF*o7bK4nS?vrcLl)Vmh)lTYxyBG6v9Qse#_Z(AMf`
z_PSg;?G85|jvlEZvifRAw=o^^PN9;^^VK=hV!!8v83-vDa%;aCOf$fjISdSi8S~nM
z5NJs#eV39_!nRYm-It@-ZP#?`oaDI^d}M)P{5J)I&%}z2!MVL=cL*|vJEC4LIiO??
zhp&cEfLH~dT>@rvol>wJU11+@4#Go@E@{{(4@&|31JBOUkzq4zUkqe~3gLAp%cIY9
z+pNz!`}l<SAS$IUWc++_agA1USbx(PxXdJ_0wO^ucTZCuL^G*Hj_iw{)!kf^+Pd7q
z1U_#c%fZJr(R@p_D6msU<7)xPxtL#GGjmN#S5{YrZVF^l4+cslsew#5D<-+64JK$5
zVTS`v$v6WswA*x`M?XJ`{<c+LJH9h+>oq*^`in;{tF`0VfVFPT4W`Dv(wsfIi3nr?
zcn><e>Sucx@#_luiTj)`Rrwf$Vf}O#hbN2i9zK{a8~i;d0#kK{vjJHoDcyMvUBvPC
z7qoJ)tsCT!xce!XtBl*0L|>nq$3#jyg=T6ekf8@UujCI$(Bb2j)`JhE1xndy!SZ!e
zp0Egau0aB7t`&~G!$X-`cAgD;5l6f@Cj}(YnzvZk{rceQ^ZcjuJ<^2CvAtb1(*e~k
z09j$4&BiTFt+G2=S^*KKyqbm4>&)-S12xDe-_r&j+8<pqZa?<RLI~8JThZrz33Wj7
z)l2n!um~_g=8Oet8s~@$W3`NX!BL?hi43nOdX8>sDZ?ln%w5cPBn<UsYrBgXltB~G
z@@D|I=+35J-i?oriP_!RSSzIT`NX^XG9#Rje?hSi)z!Qj2`!_r(2g8xbTqR!1EE3t
zV3#}+<1*Pum$1X?uXlc!^-@DHuBnM6R-eguN+`08>~1;uab)k?{mLg$MqSu<M;|No
zZI$y;Jc{mvpQ>bf3_><NxM=Ai8jMU!-@NL+3az>)wy#YZVDw?zw00-WQ*Ypo5pp0}
zn$aksiF2|!B&F@b9|rJO9r;h0TM5YQHJcCZ9=76>h*rG3wlkPO98XF%n^W1z^#sAG
z3wnZ=KlB7b;X42p#>l$T0Nylun+yaYv~LJJ*p?=^ff(-n;@xof!1nWteI|Xc8a@ju
z%Z{?MUZ!|kbK@PJwtmJMJ*$|iCfW+T7rTfayk^zkL#|$I3R|}d<Wt-lu}fS3i<S3V
zSaBVgE5CQ&J`YZFr}EH{o>EiaGj{iMfN$-A#uC+ymU`Rr6@S{P@IJiL!!|YlZc#3l
zxbCT*pSm}PX2yqHvviB^1<c7e>Xn?f8=FtuNBfyDT(Pz~eg0R){11>`wAfn#s+DBU
zf&n8gCmQ8gF)SXbHO7|r9|1WZ>oE2&AEN462GSt!;jM}iQ_U2)dI59pS2NQOJyIcq
zG$&K)8~XL<JP9|@)V@P%@)Y$pIvD9%6kgg(9}a`RgvxDX<Pi?yP%-fn%%?x>R_qyY
z7BAmaItPMThin4;k%My$l>}!D^@-4VeznG7pt3Waxw1o0*RgS<CGqG9VBQ6BZDR*6
z{%dXJy;q?2w@qv*DAnC}blynb7e@!jGTx!7uVm?9=ORS>VsGtbFR4EL7W{n{gI35`
zY9tQQN43j!gW-SO&2q;hz+qnNag-1X;5=&DmSl0*0bx5AxK;geX82_&G<)+*E;6k2
z8EvfwQxXC=?+>dVHdNKJ3h`f)XBcuXryav^DJ%bB9ybGukNNh<_2-G+PG2Uxouvev
z{k0CQeOZ}I$W>O_u4GAI=B2rz_A{#(wRKXE_`=iMX}e|EU;0j98wnYsnw=&VkB9A@
zE*9TEj}~ULdR-IhX?)qf0DZ0NqbKFjLrWI*7F?bkh|A2DM>o~dcLxwyGFMS(hlI81
zNyLvv;pzTwtQCenawgsOm!`i|*qW6Tf442|7=0<BM<M{Ao@>KEt!8vR&>;IlQaS;l
ztvVV8$XSY#*|v>dv$K~Qa@SQm3i44R=13-KYE73JGZeoRjC1y{b++gQ8=^oj+`loU
zVe^~YY^aog>|uV<>k9o^#WcV(S`C$@GhtX-S))4LEH7c?x;5SW=n|@1lN~#KbvD|g
zNWF6D;@W&LCHrDjge&@1Vf*Aqk@hf)LTA~?i2a7xO=CB;t5?007X*t$FM(RLj+<5z
z_f37;xluL&KUg1~o@akypG`%4_<7lcXLz=QO!?Pa%kBeo_ZFh}u;^@e6=Ka~ad%v&
zd-(9AM!%q9p*dMNaeg00*$lM}--`Gq^AF(!cG;F4DzPyy)LPKG9e9hnf%-#5UnFv`
zJ@*Re`Vt=pPJa*Hh<1;yv#{I_q;R%&z-1PBpi?$+ty<!#F72+}ii&teJH@Y{UAI^Q
zS$7{4ig%MdB#}VUGr;DC#`Bh5bfu*ibP~wWr92zU%ckD-&m0w%H;y^A(d_+pkZE=z
z_zu;faA~djZOdhP8o*6Y@?N9L12VVScI@z15q(%(PpBYZrN#Tif!td0J_T@^7!rk$
zVUeBl94g+|`OYc4Iw-gRF-YevU$)!KSuG*aC|tZ(v~txUTqBt}+Ig9_JPLf0Cqw1>
zf&7FkD%M*bhXow=ue9~a1SOO3VwEa-*dKvXs`^t}e_^2#9WOCUCW&J_4;%axEZd~C
zHU0Ag6zoTL*Ld$dIM6ySjrElq8&y#RrwRO}gSKGA>mgijLOYz6+@uO(0f28cpx%6T
za78}{81!=*`^DTpJ78#abYk1~=DE}#-U%d3i5C?^Keb+_S{@Brg7)~G1EyOrki5>N
zx0fBZoq}7)KzIp>VvF>+-r*s`09!qsVa2Garo4Gno6l!$MFyLA0tK=^c^u9HFR1Tu
z?FG_4g?_BZnB#n$tMSV;2mk7d5pUsN10&zGr9Ng85DHo5(LafqrX;()OgUox((Sb6
zgE@sj{l;JkO5y$|rHz@Y%&lRcOmUqpm6-PObg1bc3vGGYHz3Uj$Ne76Yf&m%1@yNI
znt`U+066va)JNEpbbn2dZhN8;L6ll>T-04|1Qtkel|Y)sJQQ_fNrt8$Zg&7bJR7^S
zZ(Mmp+zXl+Fk25F7SeM&N6mqOV8g42#?pMCqi-)v0^_Vs3zRA~$urwJY&0T<;|A^=
zp+GW9sy}E<3rLgj5{+SwFY<PhpyS4B$GEu+x>x2S3XJ+P;AgOw9`4R<vg*0)acs$w
zeGp`N)kvnFrXcn-w+u+xF?gMPGqwmZz>Apr6@jw7?0Gd-U8-aB8E=9b#2dL5hBm1z
zZItOOklrtD4~X7}O!U05+#7IV`Tx)dVA!!nu)kPyJp5Xl(_nA!=Slci$&zZcEMv>h
zo6EO?DfS+qHaom#NkiC<R?&?mxx1zaXIs0?&p&~vgv_A6fe54;YY>*vx>+0(C*^;@
zMM|}0*MNP#FFm-*QE#p^68a%~CQygVh2rrxJ1HotU<Gjp<vYg_vYh0H?_BC^NkSNi
z(?v4^J9L_Z+4lXa?aS9+x-GMhf}aE@l1PIA<RJ~np)WmuN+;fg(1qs7<>iOEVkG8e
zV%a4VPcl}k2DngUDNwUV3%9)cgQ$g8vN$90lI`Do^3Uq;z3Lms6@7L6y5duZN^Z)l
zAS)^T%3keHk1+W56|Y4OC*3qfET=}~j@Ann!iVw8?&r~M3?nd+15xXr*Mr$LWqQZ=
zg>FpO9u_~P@FqJ@8u_O}=q<iE<eGJ_QfxNP`0v)mp>}u53adlY;##ldc*o@BYbLsv
z2q>TMKr@WxX=o2Mw{DgtbT2vFXsoQs^SC)xw8K|~=`p>1(7l9CY|f(jvjLPfS5Cg-
zPqmY*-?yYvpUnwGa<krbWYWF>NmB+ptn3CrC=!2leQFOf@nS+C2%Nn@gLB|XnH;Z>
zmNqkKN@bn2H*US|PKg9Toe0vGCX2yF&oIWyiuZlzTVCAcX-{q!?O+yW{i*}rjQ?ct
z6KC_SJ)62M0%uB614rYy^Tl~p&jX=Tju1@4O*(Js=Kc$mAEky6Smgw$2Vms>cTCY3
zy7h#rpp;e=!^9r6^X~zzR^cdvC>9lN4bb%YN$YyL^ua4GbMjd$U9+W}ug{B4zYsXo
z(c~z|P#>0i$J+!(uwX_N0ad+i!!}XNiTb^@Fkl&mehujU*!&GDi$46<2cf<9#i_8>
z)YxN;U`<#qhXyvUu;9>GKUB|sM2^_cXjAZ5?j-KP9HsQK8>CdQJ5Pe(w64FdKy60B
zh+@|)@$Nhi;<Td##BcfF{fGSbiiz!8Pj2~UA9Z7yQegO1Q4Ig~j_NgwrdO=;*Ym~W
z=e2-%orC2wG5ls132-IBh$Ukmn1o>oo3ZPMCa1VHXs@o~y+;#{1G{aN3>Yidp5C)E
z<pm(=_2f=2zZU7CM}YjEj6%F@_)Q+S<&@mdWhuNW(*>PK8-_6St#!CDbjFr`S^Ndr
zl_DYsJQi+C=I7K8>kmj@f}_+y!#R1FBubW$g_#}8mmAuwnYpOEa;#7uKgHG03OF7E
zX!$fGiwq}Z!<Ld5)dqd1mQ=}%D{5&yB^x$&32f{<xG`80TOc@oOe*)QuXr7vK%D9)
z2TH@Zyn0D-RW#CUpf92WD1Z)IzL*P<L>}(JXER)=$Pot$YTB2vnLE$NQ?+K&5~kCp
z&H%4@6RYq!6zX-v4T#(_6UH68^tmll$y*TXMVQKB=|1dN6qoznj19X1<i4iaqW2I*
zS+CSA8^QT3y<Zkty&)WeUC~ERlj+&GdlV`k?5!7VI}3AJPz!dyeV6`c*34Jrd%cq8
ziT)nFT3&aW-3(`(k`aLOPajPzUwi&~taQFBUFo^DBk}m>3?F^bHIs?@^|wzPVMHtt
z_OsuTBVi(H-!oyqP0}Y{VI(&~qoegJeT^!i$C7gD;bUIErwq>id^r-^%f2wYD>uJK
zv^UyJbxjHJ&-tEv)2_h4k>;dKoM8JXe!*AoMtxD(e=5pLA{oy6!cY4TPQJ@o&dEE`
z*;t6A<jV%wX@a*YF1!Suod8<FgAi;vzA#TKnp<lf?pRe9J}yi0tB`aMJ6hjg;UJu|
zid)`oN72Z3$qLwKMsGGr2$jDnb?FPxtfmc*k1sfaDy9VY+b%K^L+fN~41NQFi-vbq
z9{v8y-=i(%`|hG|-X#RgiCV{HzH@1xvtVL4%Hgyx2Kxfq!B4yD9GQ<o3J_eyam=(`
zqINLZ`DTA<LZ+0HYkPRii@L;(oaj*h!*6dye&1$@_R*S%W5bUv(7MNas$G#wKR@`_
z-WBPq6)L@>qfJ%T1<o_dTTOh5%Ee>m6ud#2YK3cJp`&2<^<zSP*&&A`mfL$6lO3}9
za{Y7oFa+{jWT!do^qYR}h}{gx@}!NGF1zLzbeN4Q-(ZcSiM+dWB_n!ajb>HlQ?K7l
zO~fv~Z3RDT2z>h_r!2QGfl!;%hh_hMDM8eT!+2p!#P03;LnL2f6=CB28fMX%&^cgh
z(W0vh#K*@f<)U_4GZW?atnJgo*y7#(%`8&DWnp9_yYal?4^zJ0#BbU6@3xsw3{;SB
z&8_Q^Ul+drTpjCUnwW+%OVEa1M0ZzJ^@(oiF)!+<UHkIpmYsvcJB{m&_Y;ES*S9=0
zpRmnvHzCh+<-Ztxuje+e9qK7@B;~uw<n}*>J&gIV+9MYie#hLe1X<M!X!AeR`9IXT
zJlXc2=ir|;_~$v8PkH`-{#|g}AWuk=&NID>e%NMSZ$F!FrY;}m`}FBk=3M3`5PRh=
zP#n`pCv;U+70!W}cg4;@>w1_xY!@*HMJ?KDV(k++@-D5hJGaH_N_GHTB1*Fp2N`<$
zgIw{>>fXx}Jy!3LwOe)SS#BWp4y4+63!b@8L<usrL4NLgX7pY&>PV6x*sS_@0Lf5U
zXP~3n!6Fjd6a@eVC<mK=KcR(#ktF@w&L)@UpY@e&R2zg{VV*JD`QgR?I4keo<&0JX
zkh>_?Q%9bdF3U?!_#6hEqtKfjs!8X>-*%3goqT^|cA`Hw;E{9(`58?IjJedMu&L;j
zS06xTwe1c$Mpy`Sy+bb4GY8`*8H%eaedg)}xa!=|Cb!{xGMHDe{bAJ=DvA00b>n5$
zEz`NAOwp7%g=2rw4f6~aaUkcxtHiYDBV}&_dVs&aM%EJjzN52wI%#0|dz?5w0;k4x
z?c~&snl+NB0@wWJyx#J&9p%*!qK{5_it%k{AHyw2YZ^Z=dq528d}4lDFW^c2Q`M9<
zZms*zZ1A~D`?jYWKd~MQPhWpP-yS{NRLB;4fJ~xvRmb=dM$zO9m%;#P?@XGaijuse
z6$Cp0@BA8KH5)TYbfDyU;W=qFjr)uBBY#%rR0<%Qxoy+0&)Ni&>E6!|7K=?(T21tl
z7aon`p8TF6PuUiqdLVMt{dcgpl>K&&rC+p`aH<4yun^jzK|$x~d<kf!ghQN9(~LI3
z#xAntu7X@a-XHz=Y7_@R;9&u`_Umu4=0~@oj>h01@pXa-qR!97(vtf7O`osm9?^fl
z%|@z5&9N?fw13A!FFo=*T8JC*h&pBIHg^8u-_^!*t1&q8&UG<NeGL&4FSrzXsSbkJ
zcIBM3vPZqgH7jXksZ-xTaQjVXTD=ArWVk^BU%EBlo`Ww87zvpoy2b^CqQpRKW8LeH
zT08qv50Q7z2Dn|(8mi+u3i8kkBbB}dFk{c-SzF;QpN+eA?hh$E^6G5p_e!eTlcVw*
zD#AcF52uD9{an@z!Uu&j(K-0)ocY0Eue==PY;;!JTw;F%yo78P4%a(~QgvI|SgnR(
z*>_wYs5f{*BO`h>!vcPb;TA1*xsdN_eu-#1W;6scWMf~)p^5`;YWEWiE4TH`re-oI
zo?qtO)5m^5mj&>v*?x0N^n;S_OO;2H%WFIfz_=jBE@qmL58t+!<P(n0?@S{vH&%7N
zf6^l^ka#ywp{xnhB^~!=w)}6&Q=FKTlp{!w*w_A9AL>Sqh<LW18G>j;Y{P5@OC)wp
z$a>m%CT9w3mN_$Y(zn<g<W#z`CMBI_0X;T`H2BSh%X;U*@LS=@{+gbBH3PhPPnRr;
z>sipl@SBg!)6t9i(Bf_}R(a^D?ooUl2&5O}8CxE?q<JY$ydCKoD9eYS@?~;fDq~!v
z*bz*_)uApMrju<2icp|^|BYhyVc#wn<pm%zvKZt~(EL{HrazWCvlFYFK^pVFdFc;X
z*O>-8zdia1fQXnCrY%W)MtGm+*P}~_!33upu2FN3luI2aIBph_ILQQITG4^9XWC_<
zqrdd-E*O?>SyL_+Z|@}sqybyVP#NjswCGQnJTzO)ZTYS46I79yCJxdwQt_hhc!<cp
zPQueWSt%Av7_uBjvCXxSf#Ibk_rFyv6w&W4Hwu>@?v}nGKQDYPDdL#vvNnQu{_}^w
zL#Lkt<l5%jQ{?Vi2x$#@qN*l()=rHbNu4v46W)iDe4;=@Ul1H9hP&7H!pRK?1H;FQ
z4hELeYjLyPBh1iM@;+O&3?UEYf~NOIN{`(wNal3bZY57?@cfd6=hTpl-|xcSx%5m?
zBXas8{Cyd<@8%L3L=o_4wBX*{<FYkSSliZoEIc!3&SZWdM~gcPlnl05>lU%ObSf04
zbv@6Izis1KEm1a5`oUF3h-3S$FpHr;3RJu;-3*ldvFN}Zqf(QJzrR(twt!Gd`sqjv
z5a3`32v6s4*UUn|tjN{t`3wH0jdFtqu2wiR56{SH-m{_c<L-pvAY|--7<-u?exf1<
z*a$dzS8=%74!{@K`DAyxNdXm+nZ$+BqN#1L0Li**U(kfZqLWaX#(<`z=_uc_Yayqa
zX3`BE?X4Kz2H9o{xgHmF7moqIjl}A4>l1FB!_MCo8x#*5Fd{vDwm#Zg1cokz+QzXd
zi733wZ-B+#b+a-0_7)xe-u0GpXNwe}z7|Yx&PQjnVyMVAY^w31?+<AAnB0p-ZpZAz
zO@m<YKR!PIwyA~b0^tt`rC3$FJBa~)>+8_sunLa%Oq3I}(Zp91A@CN|cABA^Yax!Z
zb%iKL+7zb)>2CyJ-2nAZdV~>I1A%~RG?dg6n3Kmu;78%J9t_EVoY7aTTF9B!t3df9
z%#`32mt^IzFpK4K>uS+YcXJckK5uzVryJ2w_4K(fd#V^foVgiRR)n~xQN<$#R9Got
zqU%;>?)#35vl#^1WBG1wyEhYrvM6ku;CxNHxopaIj7DiXwUSYOAp9Vied@h<pMJHb
zgXMPA={TK0S>QJRLoXMCFAVylQsJuI`L96C{lEUx4{gW<dCzO#ppa&sKo=2`PK9l=
zCF9>#mc50AcxDg2wgE3&i2t?^C|_PtU<GexJUFn_fTGC|`L+YHzKXhD?pWd;9BJAB
z5>^lPL^D4C6sdpq%>Ll8;HS5Y%(hO?!~Ca4zjKkK@dMl~I={6mEcfpeIHwq9g1}%3
zyLg41j|_Ziu*wjG+#VsuNCnq!A&IOC&faHORU6Ae;Zce|2QGJ8e)%20ccFX#ty)TT
z3UB)>>C{fr+}*u~vfH&Y<t2#L_$jr(k;vQEPJ0}45y@6wKIgU<sU;3)RFg}4alizr
zCHYu06Uw){FsxT#$nSO-LIt0r)n7g$60!o6J`@N~`?C~DtZlRY*Y}IetM9)K5ZUSg
zjgKlJteD<6w{s!yO`Y!Lj86#PgCLC#s_yI{cHi5E>6BLvrnugLFh$$3vfg0xL~{8z
zQ$zzT<>(R~7#yff5eLEkOtwAo1Id|{Hhp^W73DPl4P*~MHUOkgf9HqB0_hhcIXn1_
z!ArU{BPShLlaW8~Mj;}G%aaz*>YZ&q(d+#i$F&s(2N|?ZcOKR|i^`SsrOrkhQWQ;m
zS15e4l@St0G9_It#;Dn2L^$JjS4+2g$JHS=Zeysq^YWGTCjgdCblQSc$oTP7v{TD2
z<m)L$^CYD%VV2u9)BBqKSj{V|E4pM2$Z?ydm8-<xjM-R0;C!A)!_I}W;pg#qZT;S0
zy&N6=$?-*}D=SaS>jw*I+~;`{4JUlrY#aDe`<3YZO_PgXguikHF|`FlUdXp*VTaC&
z306}Xc@n@Ia;E61*&;S&#rFCecxaA1dHP^+fP&^(-ddicN`5}ty>?h#WUsR<aL3u0
zYi6C|sNj~^5(}9;CxXEfelXYBSsqz*T?w_v_RU=4;e%gvf~mrH9l0y0kui&}`@xMx
zRr%l%Yw(f}bCLTpW26Bvm2at^>GjxcnUr02`){Vans`jkjj<B*+OFC7MkM_BmIX2m
z<C6Qn$gF_yDoo0&(|@W=a|#wLGS#-#RqdyiWS<_@{3BIXt2W1+N!q_X(x?Ku!~|F0
z^&LjHId~0PP^biHu_RaSx6OAZ(Wt~AC+S6o5ZDHJ*<FvB+_0m|vpDxfq?uy(KxTMq
zIYnTf4m(Tpf8YT&RKt8oE;Z<Z{<Z=K8RdKS^eb!9seOPET6e%fJ`<<jf(Il&w7`~r
ziG6~DC=)ap=A(RJ<j40KN7o2NV#VlNQgGM(1A<=y6fmqd73d#&?PCy{*jrl{|9H2$
z?Cy^XgVau~TBm0BgG95?UQyw0>1NW3qDGa5*GeT0H6hW#tE3hI%|as3pn$ylkfV#_
zA=6x_6p%yo$AOa!=J>P!PMX$$O}8ihIr{{1YuQl!r18fn-8FkEX~Rj1o|yUJw<(IN
zn7Di3T@w0)CrH%wrc6@?$lKQ*undSsU=dE3_Ku&`t)ynMIM;{yiu!8BD}=s2Xihj5
z=sGDERb4hNoB+>%+PL-1f6`(4DQ7L+7*nEsxkX=TrkV<xk>LTZ*Vt$I=bou=?6cnb
zFy`(F*?x^JC=$h5_5ijJ1x=00#|}?nNx@lXS3YP|If9m-TanRV1_E1n0cD?D#Hp52
zROdrGpWg%Gps~*5xUE+Psdri+)W~;0Q+oedG2zrc%?+7JBX6E{GbX<#vIEcPF{Q_K
zs@M}N;%J6nPMVH6Jw!p2^WbQXH$c{N&_Z*cYD7`2juEHTiCsJ5Ogo78etqn{1mpX+
zlU>CC*s}g+a)sJ!podGogbz(f*X4T5D7O4bb4eOGb{-?`1q6Q;WlUUVJ3Qs8gx_jF
z^Wpv>D&ZK2>X~87ICr%<1t7*(zq0{l)~dN%x=sW}JNWsDddiJ^#55x$O%pCZ;seZN
z-{YTc){GScSE@I+dYK7z^M?CrYgZk@TR-dc#f@U&LskqF?NpiN+;(|xl@KAEBe+J}
z<{Z$yoQk`WiBHtbF{X&^hFxSF3(h%DLU*xqS)QEOVpiGiUg7u#|1VIH>IgIx8)jeS
zjRf3!Nyq}0VR{{1G8#R_nP@+<u#<Y<Tz34zf3jNj@_oT@ktRe%djO_Mx5lf}gKcN^
zTp`;|^jl2zCO0Q^b@WDmC>B_^mgoDe7a;tVuF)y2?(sC!p9&nxeDM6Lju<s*NB2Iw
zQ^;**2^|&sqP8Q;coNlWq%Z}}!~+Q37)b2iWiKc1hxA<<^;JIVWfN3&!)fb~Jka?H
z!xWfwzdFWe(?=q55U6<ax=OjM2_f?2(K_c=<$*&KCk@8M<{oE&v%J^YTlVg&(d1sl
zGui}JFzg@-RSI|if^vD&&4ez#oe=R2&7xsKi>~{}K2+8r4MP~~ls84)lHx3u$Y&FY
zww5)sPAu0xxO$<HQZZrplvh{BJLEBt<<&l(T6w&N(wx&}cklhaVf)2}@EY<TOCt|y
zXjd$jdio8}U}bU(scVP=D_5>(&BduziLJzp_CV7$iqkm0SI!JN%kNWsQl^mlk-b?|
zG8?MA@s30E^ysL&dqS*vI~Z>Cc3{}){T1{GkSOgTa}0HMA2QHzxvLaIfbtVB1PU)b
zMnG<`jxITa(FFPLR#QFe=GNk31vt+s%=cEMEs_(&Y8Ne;`_Fl%4OI1F*Q`4vd_mb`
zSGhg8lL1V~(+>`2blXWU0}sI&!uu`lkkSq3WP8F9aS3F7Sy>68&n~ZBoi-RZoe`BU
zliXf5!KJ#KHNB&qaPe0AkI3LCp<Yjzy$Lv~esD=^(~0iqlAiZ6va6b65>z_q8JeBV
z@0&)FdfC%lT9lvUqpuYbxI29yd2qr9+3c2Vg5ZEO#^-a!V}9hmbcU2QFTNUxD*hbQ
z#Ird=xigMXG-O|w%~?4q@hHqQZ=H^kjX0EAI3W7JB0mQSx-_56a%yRqXe4GXo>_>q
zOX$DCmSl&{Q6L{Ff-M5~+16vkV4(0{Z_MO42Y!(bzX3JgRdDjponrKQp+~LlyUh!z
zMxOk5TD|;l;ezuxl}iaPg8xf+Vyqtn*i@ZqQWtUIw)YxE{6l10PZkFQB@*|t4t%sP
zA{d32@_bWY%o#;vZTkhwHt!xCI{9(Ud5-o@xyS;Gtqp=Ght;IKT)S{UJ_Ubs{>hug
zC3A^qP4ham*grAVD(z^EW{Km4+E;R!SSW~*j8d!DrTm=*dIDG=Gn*A^Ort#ARx#6x
zII;N0cs;hq*^h{PSw_$;Daa|~=>>#gYXuRa?I%tyG<T=0EeD^DDJn#NpBKQipL?Pf
zS^fyKnOhLWl}`Ug+ZKf_co}b)SH4;N--unDj$#hTTRVO7nECJDO^)AFmFIu>e&YoK
zXa4h%-xd~P{`3D%!J>1G#-c@={{J)>@}Il#@2KOeic^Wj|6iL~`~T)7*8iKw@BZ^u
z{&yUT{Lfwa-{!8w%}t)up5L>#I{nw{w{~cHyen`)Q89m<#ZsUC&r#e-4ZJElN3wMZ
z9OAb1|D0Y^WD=?(s>Nq0vMW2htuchv<;q1er@Q@)z<R0w(PVU9t)rP}Er|Z=SYmW}
zPNoFEJSk$9o6ECig~2B0x-ah{M>~@Asd$6Uy@$#3>2UJ7zuVprkoNyIsrxu}g60|I
znu&NW5fkqEjr<=(VM8ILLjD2UxuL-^01>YvZ&BW87o(<VHojtMb&*dvE@9T8x<jKF
zh2NN1l||Wr)YQ7({Uv@`j})LV$ga?Rp19Hz+jO(f{Lks)Dxo@6pECv)0b=EM2_PSn
z(C8D7zgW!&Nj&%3%Nkzte(~rY9~nlCM#)c3Xy)I#xy%O0;%N`W3zJ7Y;ZeZ1`n=7d
zxZ!Oz^<&hbJl*50&Z+^HkvF&uNipT6g{50#Ef6{Ny39*{GJ^FH@WcMoIjKNfRF~&x
zQup600|c60mj7W=J+L+@3Odo@y>`0&{pqPk1L9j(=i{(n<6Y+-lb>HmyE{4?u6T~e
zqO!B=pB^EqJu3r-4zK2+TnfSouL*cK-OV5D@)+BmTSre`5$P}>*iCO);ip;9oiF(U
z5}RM;9I)dh2Dl-qObudl!J6_BcUj*ZZN&ik8x^ekmq<S4&Z+-eL%*a5@l`?MHEa1Q
z7G67onvbwsKcfk|?>R(yXc73_p3c6axOrS?m4jkjyDO28gv*BZ6d*-TYU*UuXB~%o
zP2cTtyfcYHL0erQ%5+XrtI^5fPN6ZsA21D^;qn})%?j45(~US$B(ot5n_;GTEk7x0
zFD!?&w%YeI$gq}V$lrn3iFVi6$PZs)f-c@*VBPtQei^<|6q&@p2?!jRg`<Vhb=~{U
z4aE-=x9UclUEf=(mFULaky&4pdE}4r)=c6KW9PrVz24My>$SbpW8FQzDC%}4(yuuG
zobgcauQjWJSeY-9)7F>rYA<a1QxJaqF!w^)wxlP)%;2+{iV^4G#}D2=$SG&e3J-Hq
z{S_(cqLd?C<ZO*V-1|wgO1nM$)GYFn8SX3ktW|D(l+HfM_*sWBYPCq~`fGs~-eMaA
z9U#an!W=vr%xrFoa=1FJU^FT<zGVDp*`xC?&M~N${bkDsw;$aqJ^#+{$|>K2*e+X|
z=}GIbcQl8zniZ!n9<x~d8$k0LL%?V(=5Nf7p+xnKHIa3;QvBp^=U}K9t;X_5nbl!&
zfaWuj5BDm63+C#->g&^^2$askpt}}(^WM1-uN!^3rQf9HQ5oUw$q1E=S$SW<-g4lM
zALHlYXSpX`NvFWnJWdNF<NS<ljm>CRJV))M0*0>I@m>-A>|h4#w){36ab6duvqogr
z)ZiLRd|D){`bz^zX1V^a_P#wH>h%4)m1Wz+uG&sQrIO@O&c{)<D5CA85K72dj2VoX
zS=rdKqU3B)PT4{X2Ez<l<Tx2I4aSh;G$)5KW6bIK(0+g4UHd%$J<mVSYv<1o^O?`*
zzOVbbuj~DOU)OzYfM|tG0g4>ohrE5P9%ncpib1p4LW7`Rfll=41BvZUzSC^7#;RmV
zR||qCsl-Xk#4EJJ*)%!cZH#I#)#TQ@@%?S1E8V-Eox1XQ7WGnA%T8{~hl|zkub!_j
zD~eALsm8fz@D%DB9?3`@nFe|V()tT*$OUB7F7re4jZZA21tF%w8JfOC9~)>jIa?8w
z`YM>v{vbe0tHh3tdm=BHifEYkGWmUKNY&E50}*Q6bk7fWiT-|-`S5iP^;X>k@ZA(R
zOF=m03j4a<S0a7~srfa|ISm<RuKd~3(M;CeEC=Y*bp#bD`B5M;)7Li+nP+bGt+!9-
ztgX<WU0)YX(<u&K3M7(he^j)(0nYUtLHvaB{xSowl(FXNZIIEo$U0Y5s3CI65kn*v
z41U<lT;LfTdh;YVdQ37Vh*CAoEi{X*@YKS>Uwb=e(GjAK?NFBp@;@zPMIocNM@zK*
zht%osw@KNjXgZbOAvAN`=98O$)b!<+m5~%#CpVe-x}EPp6P6aJze#A6k$WC`+hx8L
zha72Y;2Xpb)a#%}PHN&Uhm@o;_~`?_V*0q`n`q3_oyZgO;lWqwi|a_m8+1!h(Yx(B
zfVo|FahtQ?1;tATosk$CcjAC4#PclUPsV9i>h=zYYn!|mttON%q$@*IA|C$ZnYDH+
ze!OS5=h1iDHrZz@Sg!1+V1c7Ck3g=KABfo+9MrVaT8eQ&?`D(_T@O4?;hzBEm1sRl
zrtV=!m7@ab{vF$PypHCy4}~SV;l^o2VDph%9y4WIw?htoBnp=o0%q@LK5SjrJYF=R
zYnJviuaxVQC_A2|qHo7b8{+D-XBr5S1yQGZvOFpVA8&V@;cx3Pe@PZfOcA*Tj*k>E
zB8M}p6qFi2w0z1-aAArbofkE3U(WY>cpzr)qn!=sHEM<?!LJ0ZZ<MU1bk1B6vklQ`
zbm?GKlv(#Z+wn*}gKQLI8BU!}wqgqmwywa3KU+Xg5b&y-D*|`vydE^mm3r*0<4^r;
zSu7II)NMmne9SYUh{3Wr^K^MAOVXbS*2Xrr^_0$O3O36Y@QlU?+6gDZ%x`|Hon3iT
zvlpMubkj8rnU`7yZRX=<*Gx{hUFmpZu%$gAx1w;G5M1(r-`!L`y(m20AmWEQN8<AI
z2*9j;JWid(5<uxe7JHq34lW!l@^2zlmZ<o;TT8W?8JYW*700{l6F;dt!K~%OoOTg!
zPa>jLm{U$eQPmk1j%F}m+9Eq1q^loWUUK*ln|3%(+V80I+_9n$afqWn^)6$oA>lhy
z#jwuO%O%9M!etB1%5&CI-83RzwX5rW0qt!pW2V2u{HDFnn0eiDI7K*~$eB>R4;<mV
z=Ck+)u`Z3gJe{t&W~-&7UBrhZ8yvMqlQt(vNGUNZS%}u?%{9$(>+DlhjTTSh$x+JP
zc=N{ak?dV>%O+twyoC}jmvxONn%s+*X?hD=g4Ga-HtNKQ=On`vc}Yjn7N4qMi<x%{
zGOA(wm7_&JW(8Kvekx_uK2eNb5y%gB3`4_JmMSXl_zd~bUQ>1Umh$(nMl&u!zZq|S
zMe9+m8j%4fm0cEM5U|^HflV9S-xBpPI(IYek6_uP5|PRs|M|}X>`MVGT{~lsS#pZR
z=T(`>j2?X4!*r1q;H<K5=Uh|!IW`g*aoee5t)$b+Vq0#ar4b3F1nO~ht`i($242aZ
zI}r2CaJVBhaJ*)$4)ZiuRDk?l+&=8*EM`9tpXj4xeKHlth~&1&SwcE^g4+#3T4<Ev
z%rDQIfJ+wb$yUAq3%I^@oCi+na-I`=&}X`8p8I0@ID>C{&o&71mt?KE#-Y1@5DPl_
zOS1!?e~(G}Sa-i6b*JO9taKkmN#h*4BgY(hKpAC{?2U-fM@0Zz3BT_NVh<TH%DCHK
zM>r5OP2J}hZy20+u0q(o0Im{Q5E@5xtV+nM@YmPWP<^#|p!rxe+6rNqU=d>9MC#YP
zlvH9jk~KUHW-T*J-U_fT6a}|Q#OPkRv}M)et=aT@y2_EWcLd=m+YUE+`ck~uQEg@G
zX!G9Mu=yI#R)2lK?;>Gz9PpBiD5jZ8fQ2{R-;E489zSM+lN^W{gGKLG7EjNWJa-QX
z-YMy|qpi&Flw6Z~>cUwR6@>%8kr3=EPpY`#WIeKWm};OToynh=05d3JpVjf1rRreh
zEU~%yl%z>XxsMiUn*BJ?5`y_MN6LZ1<-U-LlV{4c%8DuMr|!aJtB$ivuc{-wR+i6Z
z3Z@*Q8c7r`Pw&uo;Wy0guy_is-B!Rlaetvwef-3si`M<8oivIJjhTj5XNYf+AP0Y{
zPm;RPPyKjmBTy|tTWjn9&DYU>nGP~-2ruUKOmLY6NP{_$xA%;P>1ub;1m0Y3N6Sc`
zzI?oN1Eg-WIenJ+q_;!mygJg1m~7ef%po8x8=&H06}yp4u=BpAxDU|Y>cwi<Gs-~&
zV>NR#oD;ITMp-n!XD)EF3x!9`)QFSz_9piDs}Bg@4{>9yH+2vh?`nCyRB%PCYNI<L
zomh{lVzSN1t;x**X}pB<vfiy1jIy&+W8YiCsH4A1cHrtAdM|b$dr8YT8kh-JVc5+%
zwoBPMjhWu)QxE$IqjhNDngS=+(^*8|%?T*!JfBxhx7qk7eXb)wV}tO?JBG<2ckaIQ
z;5*sWx=qq&-^OImR3u$!TsxoRXQZ}u(ZcBIhNYL8L$d0da8z4=s&>iQU`<IGqG%zV
z!cw2V4HV)zYeHoqJHA<z7!X_W>`CGl!5+okpR2S>+H`UYYo4Q|42i2$CnAwOBZFhz
zZs3+MB|$;99>|AL@2mbuw%T@P>&>EP!{zbPwI_L4K`Cvxul~__abqaZE`~GLU%)W?
z{)Z2uD(*$y{qOscJ&#9TNbVHcjxHrzY4%FMuRf??+9E3&_nGQNMKzW2kcQ>WOnMMq
z{FxB%Zda1Q9z}fmK<kx)aGbfCl1TlC(H+)OvTli_+l;gok=|Q{#K!jAl62w%>xO8}
zuQG?B)a-Mlg<xT$8RrfDKn!>ZY`Fh~>lCIOHQE3l*CUz@Jty>@&w{^NT4DN#c%&&X
zP~MR!)|Q!!mtGVHT&T+F6WyUN%WWIt#NCUFm#yU>lYo}VZiw}W`Jx@QhKLSUDN#@9
zdo2kv(Ap;dZFDp;UGi8U6(@~zw^INy+1ivn-gT>;Ynz4$jH%4nSuVem9Y$-<%YuFU
z{&^tCT1r`o?pww;E)xb4^G^$|BS0%rO1@rG-P^V>bbeBBPTQ^c^7-B8ub!Rshr?e<
zVgu7N1;WlT@&5fgQU#H1$;~e(?l*)lfrG6)M#!@kqK<tc!pkE;{vgL$ap0NHZt>Dl
zUJ-A!qMg{z9IRF_hSQgyYm%jH{w`==fs>)a4-;kjlAu`u^!z**_CaGjzV9<@3>1~N
z0*tL*NqnC?TWKkGc`V_a7mT-DjgCJMLn(txV2NoX*2Phj%X4tII4XTUOVq?R(Y>fU
zv2+b+>kG7Y-eQV9CuH||o<-u9p-#q&uO_~FIL4-CreH_UtY_TAS-gRcrl9FbFw*I=
z)EPj>Nt}$i1ZP3*Xy@jGJ?MnA0cK41_Y4|fgUX5-Lm-usktTN7y%E;7IB9PBU*p%z
z8gZyk*j|Qsm&*-=@Q?+6=AJW4$eYF39QorcdIZ#yap)M2r<AU=yTpAJQgMQ+u{Q~!
ze@R|`kGmgl3OJz}dri;+o>Pw>m^cbT95#aq?`A0vxI1#9Em6XjAs)L?HeSMP1=Ged
zs2`gYBgc&!FijTOSxrR9n>&47l)$g)E4;Ai8`K~0K22H+9rC*ZXw2L<e-72_<;vEt
zmA8XbvHdzz0-qr$)M>YtI95wM((#6WV=uV@Uw~12YpHFdyM#RQ`h0m_S<iX3Uh!vp
z(*^E0I|TuZU?R5A3YzTwDSy4Z@3)$?v<e<J)y17Xqx$g?5I9Qg)Oy2#@*9mZhv8FE
z%v%ixB)Ap{LRDybc_5~GE&LT(?iv_<hleU{@S;~qtKJ%vInlUkUq{Qb>&NVuQ&h*k
zJW$>>r*1*~VE3R7+{e4h|AS<1itE3qK=%J>w3Y;H4oeqO;5&MHi@*~gd4<%MdLYX*
z%f=I{0<8OKUacSD(KO4Soz~gpNs~8iGy^TBmkHygA&AkmT}Fp=+a^szl@xa$ZNU0W
zL7{TrJd*TliIqUq1{U&q98cz&+XWwSGQ}{p4IH|3&rI(8)GFDpCTaRz^K9girp3ut
z+Wd$*F_Y(5TtGgoAp<hY22!l4ndbN7hh@_S>Pco8Cbw?Q#PWleS#Ge@g8-I6Tuc;)
zQ4!|8N)j!9g&+=2{3!^UjkpEW;F9>WF_-X_Pi2w<bgZRb*^x<5AXBXV&o&eOW`V<M
zh8S<oT$DX(mhed2H1(nMG7U|z?9NU7XcY;o7tu+z2#ckm`f7d1MURRz_eOvvXUTN)
z6h~x<8ZxtbJIOqDTZtWwsPnj}HT#UOy57hm%GjRi8z-<=k1OnR;$27kQ*`pE$HwBz
zB{*N9pq>pVo3-c|*dA*sb%fci@X&pVyW!=m6KC3x!Ltu~d9Nh7?G-if<K)>-gFJP{
zH@d$Yxqf$(j=b)cSJt-_AjV~yJauq)*~5pQnBchxG-r_w2FCYR(#h&C&F-A2!UBV{
zX%nM;%x*>s7#ycd1-#MAY24U!hYXitA+({isrf+6!`Mw!?Pa!0bjDCsw)!4dd>hDd
z>DP3%jd7|<7ERnCQoS!aUL*PSpu$fR_>rEzzT#{Cr}v!mF66++#O$&B<_9Od#@!M=
z)i%j*dEHZ7kexJwNL)D$$K+@?4pMI2nv&zNDg%OOd4VOMD_u~`M|OZ6)xu18Jqw;i
zq3DT4qx57ei72o7FhWw<rm%&Bcd7tCu2#hw$USWZG6E6%*=%<cda3mjK2k%OD#UdR
z6zRw!fPU26L>>mR*H|}9zt>^eRCbKsYbV3lpH3KB^ZH?eI$rc!Hs&$wqBR$e#cJ*o
zESHPf>u00M?Kz%lhfZm}=nL{9>2Wf!v`!wGIS_HfT1w4qOW|y{?{l7Mu?g#1lu*DT
zx!Wh=8I6-~yA@McBh^2j>Q=OtGT5O7PQ9d1#+mn)#Cw3^q!$%a#RD_33YNA!7NtA)
zA?oWQcI;N#T}Uyf1>+<8sIxp>_t1NdGe#IwP_q1T`3cGW-^|j+{7r#fD2jfZ-hUGE
z>N@IN2@>W@&sw}ic5hC$s^H{a{EUdyR_??VuN;U$bnXk};fun`L;?HN(htOlc{zMq
zC&N@S`A8xbL@fHKW87Ll@*0`G0#Pp_$$l)npdd935vi|)FxH>GCW?%%UG3N*f7Tpz
zp~KL9Amcy`_ZW)YxbKvYZi#??&UzhG6{5*2bQHqo<Y1bhRn%2h0bNSEpyOvdel5zp
zw|h9e#nPj~S}K;hw7U4rM@;ad&fd<#8Z9OjC^-^QtfiZ-w%yw$D0D%SOp{$GGKZDE
z68+-k2ZGsOH@m3ob_%<&AZgi?(+YPWJ5wbCQ(dW=WIf%cI!++P93}6CQ8leG4xL%*
z-mSR%qp7I*Ie0Pjil5;C(MGk{&NG|RnWpV-YuDoLcTKE^t%Y9<(vd9A&%kZmKtwTh
z+?h!xe++Jf2P`2c^%}PBR-D_xB10BXXbOR%#CKs1@WCfXPTbbh@v!v>IXdy)an{E2
za_pHXb-vsduv_;`n0Rim4E;%chlEnY@c#5zHZT@%<Bf<ucypzdtKdkdw}Dw8d7lo0
zl*b7Dl^mw0csUci=rc?FR7*L$xDv1oRm_Nl(yOE`V$JS=bJc+3!0~bJ;W(&3oR>Q)
zgqsbx8R@O-cR6xtlrZQ(3aVIIykJaWEg!>bWo`vL!ZG|TkrE71fnG;OV|mUHiUKoI
zJ*I^e>G|jqGQx+!^buE-q@I<Cl@uVxkDk~af5!iVym6eJcduFp?7w`W^F-p^O&*s3
z1zNU>MbtBkEXp4D)yMMRa=Kg6fxxt0vX*b=@~li(hAY9JM0zxPGj*!TQ#Z0X8pfVp
z99rfp6k7-9eT?p7EvbwO5wZgd-7VDXFj8i)y16w@<mTm*lZ3x=`g?1sz|HB8cbfz5
zEL>at-h>ZXqysqIKz@OEe=`7t6;6WajRbXKzbfQgx;2esqw25{&1f4J;H?)xb!k6o
zsNNNG@0gm;3uV3C&c!1`BlO}Cx*;&LU?H_=;y!#{(4_Q?Z35VFtO1j%Q=#gpZk9*S
zQ+Bi&t_Ykd8=y@|#u|c00U#}C`(@l0d)B*Q*KM?F(K+S+0m$I7sywq)egzTD$ffNt
zS%Z&+mB@DZ?67D%eUUv};f0epL!jheYAqDD$l?2v*yRq4q9L()NNG&{S?4X~^1AvA
zU8uSRUmNS0-B%<d137&C7;kL&r_wy=Uo)8L!SlVrB_^Oyi?V3T`S_1kUD3?c-yh0+
z_2PfBn1YYlokhX;52$lCh_QE%df+?S1bY?d<`_FeZr5e>5;ahc(bf8)ajTp(f@-hK
zwzIjsjt*kHQM-jdREf(aCGKFln7YjQ5DXyMq6PLWfGL^gY8UGTtBgiHc9SnwWA_zL
zbKSYMGHSz-Ku3yfAA&61XsNY;*6KLIUigF1;U?E8F-sfWTqLmP40qhsE+SuE@bNiX
zq3pOz1`3{YRXhmmdG3DP%SgU!5<Q5(Bf|rxYI+M>CHD?!S>&;JBa9L9q#J0LYt))(
z<Yy#A$&t+^XyYSHD2Vxhl?I+N6+=vc84+FC^cn8^QbxF~hgMlM4v3R=3fl-y?dcK%
z0B^?3;@7hBolSoXrBrjUF){ZY_~Il6&X*Yp*iPVlWti^->%1R4k3vG{=34j_`ZrZz
zGs+W{@7ygGY~HL8hR5b*6?9PC@Tnj-I=mN*-QDQ=j|r(+j_;mUO5_YqdON#NEnzic
zUMg5JGaaLPe=|Qqqip!>exrV?=DU9eNEW4MpE#G2UC-GW?HVASM~mpbanZ&7kz?M@
z;?`(C^LI;;?p3qB)uF-n=owM*AMDe|$fbw0&(Ff$91^qbe7)u^SSSm8!c*Lj&P;*>
zaJdmv#W(r9@5WNnEU|K|TuOGJ(tVp;RkhoD9|o2yXsUX)7e%_dh;LkDSuPqV`G)(x
zu?q5G`8ls}92z8TBJk|M18@!IjQCo))&<MWmbI7zF^Rz}c{z1gdkjuR!oKJ4PIX9^
z#noPZ>n^%;!hkjA$YUF>;ZyOs^iOU6Oj2ch=v^Qy3G`12N^E@GO&bOvju8Z&ZMbuw
zMl+Uo(>@HJ*Qocl9b;#``Ri-~R0L_*UQTswSiT$It@zApBe)#7O)^339UXj79TM(w
z@OV!LjdgjuROF*LsRonq&sSF0nuu0NbIV~f!k(3zU%9<OK^<PcoVjGmgCK3<V}lVH
zdtsEtbx(NSN6iAJ>-@&1K4&LBAc*SgaSW!k7X5mT3*7A!l8)q6qs1JA^F9r7t7|<f
zO@iwPx|;_iSaf7l_Rh0$I<-F9T0-7rb!}it40A}r9o*5;RCD>qANE#mbc>6RfOr)+
z2G3w&uNmWv{=WB}3`T%oja%~#%)DbhRO4Z0iZ4zMUzV#=JWvssI%k8p+L6d~@#W7%
zXW`SP+b=oqfx&gw!mOA1%Qe-D;|>jQ(QtFK73A7#Y{zD74y$b}PMUaD55DZ~Vz$W@
zcMnwNuoN6httj!)?9TS-SIA8T&guYH-UC?(y#teC<umuiNgKkc=U*kUO4@r{&>}QN
z0T+>xYfhbhH9En+rg??%V-aHPZ;ALC4KTBli0Y7$exz%<Z@xuVwGhT*dGVwMuJe~^
z;MIaQCMj;UJaezAG5|gi{PND(NS4l^&{SJh@kl(4*i;s{B3pyYbu5v1dDHC$B!`l?
z8~fK@YZLeA#z?Sn=rUSQc^TVJ@06|&Ra*n-{RMlsdp>RXDrOBp$Gm0fv$y5cSe!_#
zzSMfD0TYfi(*v33$V&=Qs`#oJn3duFkQK@CAWcRBI>Uj7<F$ZaE=3KSe89RP%|OO!
z-l})paiy7QZw<CC(z>>quQ!<3)OBp*gD3*%2EEM3cR5>~)_F0@H?ZkjC+AXJUq~|<
z?K4ndlT_v6!Q1J$?HOVF(%FXjaBjY7<go272W7i>WCrRvwHE+L-<kOk8LIA_GYi0^
zF2>VVvg3&@Eu?|I7SiLQ?f5K(TB_>4;~x(MyV>gaIernub!l#RlCO*Jt<gzNZxjgG
zrQ{rKiJVV9rVElxttpP8y0k>KRVJ;E<)%34Ke?DcTi;v#v=QI`@_(BiZHzC!RD^B1
z_-um(f2jfmdggC5ia&3(<M763{ql$WqZ_^ApI<oha06h!p2Y(Vrq6F!@&CTUe=(bX
zXW=i1?Z1ZfUlRJyvG{j619<VDWAU#?0$9+0o$7y~lYh6i|I)!<j!f9}e_?mFpIoEI
zD!wc_kEo5iy>23u3O8Cio7*Caa}@jM?*ahK23YH=65YSFeo+^`3?v=MqusOxquy%n
zP40uK7XVz)o2uzl<+5(fKNI;7jhl@T_n`HB-I}U+N2Xwz6EOV>EJ&%Z<^)1q#zH}9
z*P<fyB6{RABcj*zD>6x$IYrf37SX0krGwEqX>0k(@&j;q5Iyo+7&Yu4&tgC)s1%zS
zX(tKyCsvK9mfN2+KJ>m*Ry8`AMZu<1zI1AO);@qe6<lQv(K|Gs<l{TtJ>q*n^LI0i
z*RT6T*N2F$ZR01*?tEG(`(Zufw{Sb3;Phfv_mFPJUzrB4XndNAgm*e?D`Hdoi9^XH
z{Rd)#ynTEwzp9~=Pp^+@{{AwXgtZ?k-VLNCg7d4NA)|vku3no(9)IPl5w-r3Ff!94
zONEa0@#PQ9{Xwn=FQ)+M5&Qsr8N=|m)7}u76)CP!Nl4Hx;deOzDHHifM*1(Q(;<a2
zZ#{z%-&9)F9;(3rv6rupEJI~?4>&$!UDlwkzx>aN{$kpsn|syD-8@N_S;)DV2^DYS
zGuyUIMIBdOQ**#5eDFCs(wE~l*;dpzT*{<;oX)nRO<??eCj|_<V}~|AlF*QtLQogB
zgiSj;*O6_Z__{n+bVKsaatMAX*(&$N#LxXXAf#nkc@&8kd;(<|vS0b`ET@-Ie9*$(
z-p37;0np&bdl4v#;fu&gu&FtEIjc6bL3GUh7kJ~Y%ln+P-9GRlzt2hpPl)V%O$3f>
zLz|y<Eq&FZ3(+l@w+FgcUYhJK0>j2@UjV?WdD;buUWo6{6xZ9g?zixcrk{$d?zd$5
zH_bLiEI2n{7hgC~G|z4bRL_&ci!0vR$+_OJ=i&D(ZLSQ{%F$6N_U7g>-1T>SHw|>j
zwp%~1q;0Hm{6I{032a)Tq*`{zNPcDYbd=|kI1{`=wX{c*W(F8K-UF8)&`R!t8w2bb
z8Tk{n#6uHEmejIi_v`p=T2Bv-p4GHyBr`AXBrYy7W-LDM8V{p=Muzv@2`)<Pg~ef*
zI^MSXsh(v02;ZLxd3QRhK(VoMP;pkPo*}!$pF#p&M9|bbc@_5$+@h#<;SdEymv}I}
z01Bww$GvqcaB7Hs*MG7d8?>2DlzPnB)p;gBo@AE{2nQIoUnS<@8?FMCu>wVzU@ic3
z7f$*W%{1B3MzOmtsbReckO)avS?aZ*Tsw7yIl87j`^2Gj10Spf=qIRIY8z)BS(VLu
zxxC`w7hMmiD%7ygXV1uOnSvcxRzfxnz<T+vd-Kt15T3!zg5_%fiA2UT{gJ<SXjE==
zI$pKh@w03IBWHr7hpKeO9{?a&ap#&3@$rb#iLmu}k*Y5k8uE`c@#);fapOR2R%W&S
z$WC`Gwg!ES3O0g1Tv>R|AjPeZbN;YQx8R^oukavX7@#zv&Nkh;kyG-qa*~Kq{BNzJ
zD9#j?!F`mlA)V2vXraA9Sc>1Ck#>s%pwQuHAYG=7HNnVwmFUoITU;Z6){+1fo1U69
z_A%OB0fzvfUQfZSAG>PHpFd1oqf$m*xw<>Fp`d(#0{+%GZ3gJP+BD;;y)Ai-C!z2r
z9HOcSB@EX%tYCFUoI#lWu516s#Z{fFQ}TcT42RKT{g@mU0Kc)l>+r*3M%OFszJ!I@
z-VXtgDiRN#ZY|ZsCfV~}b250h@g2)20WFrhV1-2K8pHQ8rsr|~%PQ0?-CM)LRsfeU
z(gk1b0yu%n02N1w=GtWHwn)tF6DV%eGp!s3|GcwU2yxE9kJ4SV2|9%%c{(Mjvx55C
zB>)~jI)43Hx{TYBQCaos9KvtZ)z%4{j?cwr?F=#`ifmq$F2lL`596R_5k%(;3i!K)
zZI0!|^$c$PtF6B4AO>U>Iqljs0xw^au=Sk?EF3cGruCI<r46UD;OPT3eg4s-_Vmk5
z?dduj$K$oLJN2{ZJBAk^+$>l=UWQmYWI^uni{d6V^juDLyt2wZT;hN_zkayj>pzZc
z+j6|bPLdnA6%?j_Ga{K{zM4^zK<mktMH7v>6~+K)<G{*R+X`Kz+RYVHY~VQm#c%R*
z`jUtPs0jF!ptteqV;nw%i{W)V>sa=kzj=T1bFKwQ8@9LW8!BEap`>covL=JLR0L)*
z#r~>dx84scgAc)l7!lG)kCuwGKmig@<h^}SVs6-RoaC&jF=r)~mgufutV!Y-A9^F~
zZgok5EuDVUZX30xuR`qH>DU`E2I6EoXF96l!tP96AEreY4|nKs%#p?^Zj?r(KtLs}
zVTVzVov6WgFl*bY)de{A1N-XCOw^h1ocihz-nrqCm(9RDy%SxR(b@JMfn#|z*L?xO
zta<vz@7_%MH)d!f8**|JEn57wEL<4~=IPbX07Al253^39rt!~4xyE#ZZ8l;^gX4u)
zbfB1W!!>BFA>2Ux+?dyi;>?o~N2tI4_&|aP6$|;*4bG#f$-f7V0zwY0+|LT%<R~3`
zEJ!km6rVOV4cp%<oWE<j0Qk#Alo~CSi-f8gnK>7-!fZYGMjZpsdq;eTnvzg+Q~;8o
z$5vs&37Ht}!fA#-G`MThG~^IUa1v(gR2nC(UBriCy5<9^^hh-nKb-}SWlvOsUmEF$
zyc8{uyb_NsNf@Ua=C27%Ob@SOnH^$Agy~|dI7~%E(165zq&rA8JrzeZR0n?RVS<8L
zz!E1!Oi#&Av$)pHbP~1c)3|>yKtK4aItoO_3l$ZGnZ2_n$?~G;H*;*uj$bI+xqZc+
z*-3B%6yF-M-frr{hjw@NFQU#uR~KhWYb_U`CZ~|c7mCW&y#Zp-xbznVXnSr>Jr%Ss
zjP^?l4}9m>iLM%8r@y6HpiFkCsQjlm**E0rtGs&HD_>@RnR3Y+(~;Ms;sz<t%Reao
zAQJQf0_F6zvxnibtDgYLSkmL5)Iz=Z*td0(jLghf+MG^$96<&7=IR?LiwZfi{L52V
zNB6d9fE^@9GXQ=Jqqc2+z>fBZccfX$W62TI_~^Z@9pO{6YCcaC_K|vK$ehhbl%3eP
zeYHb*-nQe*YP5I$9RrN?&9ACVG=@JbD1cdPrd<D^{St_&5civW1Sc2ieMC8$?&`d^
zVPE&+(74sCSQT~7EEj~I?cN5qTg_xz?<B*PJ}0?RqFLMV%KXt<5GWqggaaJ9syguj
zC`i(CJdY4(?(I;_k125$4?ZayaR=>^FVsiE13rn?oYX~iAfgiC?gSm6M1HWR$CmMV
zO1{jI=?@vIjH@$MWlU%Rz<dgmyjBd{agiX48%in~A6Gii)uHV6kX;5y(5OcEz<7}C
zE}eg<nBo#d{>X20@4*in$l$#7!i)@c5_C;h-mi%wcJDrzArMVYxaHs32*!2!uEc|l
zw&L-r<zHhu>W9ReeTA=gW4EUYBmYM^4L!J5agC{h)FPtK^2a?Pqg;`?+8J7IytK2g
zxWwf!r%r1WHq9_gZ<l=S_u~FzI-r5s7h&Lr0U<uRkQRLEc!YLj_?+Bo%WTq6d6~;w
z1ym`h9r(Iwg_>aBZ;iN_;K#vZ>xw9f%vo8JYvQH5D|MBC7~+o^|9#}|^Nn6BGLD{@
z-($(8H^IMM^BVb~IR7Fas=0&X5tJjJ;^(~gI0=+{@t62Vz~SQ01eF1W%QfCnjl`S?
z-~0V3*VN^I3V2;oGc#Geh<QlJP!S_SF6KYEb`~E!Py8Cicw^lR#MHFOG15ARn`c$9
zQE9KCqPV+fbTGg!mES!f%Ffg$x0U}mjc^0Hclbrv<^!Uo3fnPYfE*<!JTYmiPEzb}
z+kB^w4oCB0hnL;DSH29&Zzd_o5;`Bctev8J@|%8`7~|bxj#xqK_gltEJChuOn4u>)
ztESjP;9Pb@1E*SKYHDUF*Q7uZ#43N&e%C;!C}m<ne}FOKf>GrUzH|GEvb={3U$d-D
zcm2nxHl4!zani*#6`l`YmvrQDli0UCy*^!Rxw6i-Hnn@qd)&44(5CR|4(hG)`1X3(
zbYR;*E#hsmRc%>U^3eGPv+7iPqc9rF2(ta1^%WM|2$1$T;6F0m;*m=Yi>isw@~n%G
z#x4P8!{1C0dzf-kT4^EARG}4WNI;|D*V1upuFef;{eoR;F?6Injeev@>)7YX=G*Ah
zykEK<C%w8^Z8Q56uNLET84Z0MyZJDAyg6eL$DfXFm`_LnfNCb8-(MC@8Y0Xs{00bz
zc$cI{v~AXNvFdH^1(G2W)?}dgj<uA)H0>kczBsw;PotOQ)r*4))$@4nleL$S!26!C
z5auzsTZSaWISXoAwn8j2K|6i}#{4XFFCgccFg>}((2bw)%FX)y>3{V@PG}#W>-dkT
zOq$ze@g3)V4P(NWYx)Kl&R5Zbr|)<2(04mOq-am;zF+_{9Q<B)K$<StC%M4Y!-%3$
zUp-+}hU-|oHydIlch-LOXc|nTDtJ{CrIa0bW;AA!33Qb($G)}-F)!nTL&NgD=bvt-
z*ctrhn!7%)b2pks*jFqcm#2WPkpa0_*m2L-+tRo#(_e8tn1qBTQe7bQzJ=VDLcG>C
z*PXXYn~qKI*qGruc*aQ|akKW0IQ=?Z(ec97b5}Tji!)>%QM#G#_}XQ9CZ?~bs4`Yc
z=a%jGK$&748YN_qfDp~0Zw#S21he%33I+rUl2D@d_kl&KigOBIf$b#i1ouD*uNg*!
z=Z)1@^TVR~!lA=EqO8BpY$u?<iXlBN*^1mPdI+LLth)<qgC-stR-f>MvYwLvj2kS9
zaq#jr`XOvfyfioZxF@5intF=f-wf0~#n<4T?!^L~VJiBQt#15Byxdiq$wH2bVhod&
zzm>FXsq|HC1;2QU#AKpS7s}EGKay>~Uu_gon@+`S_3Y}q-^BrB=9*~Q+{~m5CYj)J
zs^jV-AlZE~hC_zgi5IR}4j=oV&=~o==^)$-pro4=^hz!lgvm%8hw<W050g9!Wc<gh
z;Bcr4z}w_`LxcSSZr;xqnX4m|M6W+iL^NQx-=SgzVn9t<2wiW}BCg-Ilb%XF=RxQ=
z*?#e>Jc5IiwY5}%*%rBF&oEzFFu6BHpIwrKC8T#B{{GST|Cp{a&I*qL5Vd4qK(TQ;
zpsPa{^^%iks2V+gssRd|g>BC`e)P1etE~-skuyGmZpm-915_W+aM>>Ps<!cvhGKE<
z4}ca><28DO>!%6?)|X&+6NEpBc>H%2aZb1^z!g%YF-ovJv`1KAb|@f~t4HGm>lAHA
zu?<dg-XohlWgMf}q`FDpLOsxDOPqkdZ)o?LH6FV?w|6xUbWVxPoU9?XW{VuWpri#I
zpevHZbI<6Lr~_J#jei8sjKq`)3#7zr3iB}}yk4XXscq0tZ;G3HN>-wE&XSjE&~F--
z^vWeg{esm5JkoX8;rKry<t*fCoy58<`yt4KG;rQ?if#s=6bUbdc}JYeT|0AQXl-pl
z^cfoXC|2~owCxH0x5nQN#FS~xV6;s_j-nbHMdH2>lj{na{+pLI=IWxb`=>(6RsnH*
zHI+)n)YiTO%@22u!~JUW3IGceteh1SS3j+7X*7}#u`>^4mtbwSDd2{9dZKsMcyaX!
zS0cj>ytas6;8ylZD2U7lVh1T3G5Zq4rI>7)7GRTjt^+N6)yE4$CIPw^*YwQ1pqt|L
zH7wSU%jH`_jV#EeHzXaCU@WOLI)}yntk1a*T}ww~wypxg7VYtDcRBFYk-k;-UA$}<
zg$<}*T4E<q)k;XR_%|i$C{y8xeWM(-BPvShE8&NdEuQb+n0^)+7^4Qe3sv95f_@c{
zEg!VG{kVBk%KvDV`x$*Jx1r#R`R)i@&ViUdQMwo?!Wc2xsDwElT`-Hf_EX0o$V}4Q
z14!Y0m?R+!sIqZ56LrUZUH6h=90HUsaJUb9dF&p-;}SK{yP&*vF)j|#yj4|kw{eQ3
z8GDLv86ipPD`X1ns@)w8rFDZaKerRGc_o>gE<BK3U~P;*cJU4<hYs|tD$LwH*UqM@
zqSNb}_4N0cmzEZF4tgj7nS14zH8soT!z)53+6@Q+{0KK{CGI%s+DyIhis}XZaY#nT
zKsSBwM2!r9=76}e3YIvm>AjBF8jymkpGVN>^wv1pE#+E?$iBqjbYcx_>4HJw{NhGJ
z@fm4ep(SXhJwQW`6I+vBf5~-oiTsOGKObFd3DbZOMRAXEy+bRp!v1~+LzD@gnoWHD
zQtNkns;_N+0otYcA-K*-%+*_JRipi3BdC|T1dJfp!%&%?HlLk&t_0M}8c7rZ&7jQP
z&!bDmY-M2;McJ_cKdTz&!-O`@14k$U)k-e2*7)5f1IWwJ1POOIzM?dBhDvDwsT`cU
zTg<;~b9O8N$iaAKSReZ^^Im-cgj+`<e4ReCW@tZ0jYLFYkcq1a=>tqzLT%nx61b*C
zJ=AmjPH+AEYi=jk^w68MYnJZzTS>c!pJp|;W9A5w>qBwc=X80UlxoF*GYw+KEx#Tf
zp7lxii{7eD+?JiJ^1>k=8E!$Rkqg?bo+D4(%nlin%-<svDKRUQEnX}KvrCjVs}mm6
z30(&_uHv2{mC-p-2)#tqIzN^Yqxwdvv?dffC+n}1g1C#;{I3GV32ie^C`rxgO|1rp
zozN=NAHkQuJ;FwjMkC9Z&u+b4#Ij%?ZM#vVEA5y!%lC(Bm*6V;mnKi0qpk~ykCr7?
zH-H1{bTWZ7-m)o$g}es>fWQMcmC2crMDpiJW+2deEkFzjf?Lp3H*GaetBI!#z4HXX
zf#P<GfF`D#b<V+?gQ;!xF0=`Grz6*4VE;Q%#(y*;iBP2kAiZx9Xcq%1P>%7w%E(dP
z&bYj#uA1ISmO1t9pqTY_cBXq)qvq)?|1hiJvwB+F9Q+ce76)IcfxxnWrcw4%rcd$j
zhxdLvH}fA6E^fUw+op(mLn2xhGcf%Iw5MmfdS#l6{!oGj#4pQueHomjt$O8d3HlcR
zk3pKBgM#*G49E7sL)9(DZdv8~iz=W4r$cTqCVGm1bT((GmZ|?;CaX?|Dm-RjXSx6V
zKeif(5)un)QyA>;DRN0F7MAv20q{<Tbr^VB|4TjbQPh7`A6DdOlbcus(70<pA`ctE
zS_s{&)XNz}2BM^+`0XMZ;b)!k>&^g@$g}C|$-@FDsuFLYHFyXDGGBmemRUCB?KP5x
z8fVx!fo|9*0tlr#fv5Ype+_2YJvNaq*sE&9GaOquxp4%qw44oFX*iY=re#p9Ouy|{
z(E8PXHbcm|G-cDKKQ_j($2>(hq8h>eBQ;*D^*KTMxbu!VAKZTZtAeAqgzrC4_~e>f
z7kk><`CE6#WJjl`sz&QR`we<yh}gwPsMUyZS-H_mh3SKM3~}`=1oU7|PB!O%C<aLI
z!iN~}c(Z=w8XyCEzqKaAA-+QWUG^^GdG68m`+78PxbKdP_GLN=?cQQ(9$f4l*3kGG
z&2ee*f9R>Y9_Vh$*a-Iit4jD+l>nI1{{;eV-5@vptIz>TlKz#q{3~w(mczem;s4cI
z2ojSdtL~T8qoR%eQXKJ@WheG{5%Y8Zk4Z}Qf2ke+i&y}Zbyqh(dya1N*HZg0TjaWO
z&xTC@*-HKYaPK;XmsNhsU-<98Y@*0Z5VQU7&;9Rz_3t|OFAj6l|9=8P=O6#}`BML?
zb?|SWFI`7=e*d@6m*TR7U;p;`Qs#N)Z@2vMi!AhScRaXA$d#mvk+un0H%floSG_j8
O=gv6%Sbo~${{I4(OB+xC


From 0ce4077254d79884b27f6e93c73e78511f15cf2e Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Mon, 28 Mar 2022 10:43:49 +0800
Subject: [PATCH 079/126] test=doc

---
 docs/source/released_model.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/source/released_model.md b/docs/source/released_model.md
index 826279e6..9a423e03 100644
--- a/docs/source/released_model.md
+++ b/docs/source/released_model.md
@@ -8,7 +8,8 @@ Acoustic Model | Training Data | Token-based | Size | Descriptions | CER | WER |
 :-------------:| :------------:| :-----: | -----: | :-----: |:-----:| :-----:  | :-----:  | :-----: 
 [Ds2 Online Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_online_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 345 MB  | 2 Conv + 5 LSTM layers with only forward direction | 0.080 |-| 151 h | [D2 Online Aishell ASR0](../../examples/aishell/asr0) 
 [Ds2 Offline Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_aishell_ckpt_0.1.1.model.tar.gz)| Aishell Dataset | Char-based | 306 MB | 2 Conv + 3 bidirectional GRU layers| 0.064 |-| 151 h | [Ds2 Offline Aishell ASR0](../../examples/aishell/asr0) 
-[Conformer Offline Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_conformer_aishell_ckpt_0.1.2.model.tar.gz) | Aishell Dataset | Char-based | 177 MB  | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0483 |-| 151 h | [Conformer Offline Aishell ASR1](../../examples/aishell/asr1) 
+[Conformer Online Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_chunk_conformer_aishell_ckpt_0.1.2.model.tar.gz) | Aishell Dataset | Char-based | 189 MB  | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0565 |-| 151 h | [Conformer Online Aishell ASR1](../../examples/aishell/asr1) 
+[Conformer Offline Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_conformer_aishell_ckpt_0.1.2.model.tar.gz) | Aishell Dataset | Char-based | 189 MB  | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0483 |-| 151 h | [Conformer Offline Aishell ASR1](../../examples/aishell/asr1) 
 [Transformer Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_transformer_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 128 MB | Encoder:Transformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0523 || 151 h | [Transformer  Aishell ASR1](../../examples/aishell/asr1) 
 [Ds2 Offline Librispeech ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr0/asr0_deepspeech2_librispeech_ckpt_0.1.1.model.tar.gz)| Librispeech Dataset | Char-based | 518 MB | 2 Conv + 3 bidirectional LSTM layers| - |0.0725| 960 h | [Ds2 Offline Librispeech ASR0](../../examples/librispeech/asr0) 
 [Conformer Librispeech ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr1/asr1_conformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 191 MB | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring |-| 0.0337 | 960 h | [Conformer Librispeech ASR1](../../examples/librispeech/asr1) 

From 1e350079257cc680d78bf007be5d0a45d0be47c4 Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Mon, 28 Mar 2022 10:47:46 +0800
Subject: [PATCH 080/126] test=doc

---
 examples/aishell/asr1/RESULTS.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/examples/aishell/asr1/RESULTS.md b/examples/aishell/asr1/RESULTS.md
index 5ebcfe50..420d60be 100644
--- a/examples/aishell/asr1/RESULTS.md
+++ b/examples/aishell/asr1/RESULTS.md
@@ -12,14 +12,16 @@ paddlespeech version: 0.1.2
 
 
 ## Chunk Conformer
+paddle version: 2.2.2  
+paddlespeech version: 0.1.2
 Need set `decoding.decoding_chunk_size=16` when decoding.
 
 | Model | Params | Config | Augmentation| Test set | Decode method | Chunk Size & Left Chunks | Loss | CER |  
 | --- | --- | --- | --- | --- | --- | --- | --- | --- |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | attention | 16, -1 | - | 0.061939 |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | ctc_greedy_search | 16, -1 | - | 0.070806 |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | ctc_prefix_beam_search | 16, -1 | - | 0.070739 |  
-| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug + shift | test | attention_rescoring | 16, -1 |  - | 0.059400 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug | test | attention | 16, -1 | - | 0.0573884 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug | test | ctc_greedy_search | 16, -1 | - | 0.06599091 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug | test | ctc_prefix_beam_search | 16, -1 | - | 0.065991 |  
+| conformer | 47.06M | conf/chunk_conformer.yaml | spec_aug | test | attention_rescoring | 16, -1 |  - | 0.056502 |  
 
 
 ## Transformer 

From 64e12e949a022e4fdaa302438201c5ca161cb361 Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Mon, 28 Mar 2022 10:48:01 +0800
Subject: [PATCH 081/126] Update RESULTS.md

---
 examples/aishell/asr1/RESULTS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/aishell/asr1/RESULTS.md b/examples/aishell/asr1/RESULTS.md
index 420d60be..73cd57bd 100644
--- a/examples/aishell/asr1/RESULTS.md
+++ b/examples/aishell/asr1/RESULTS.md
@@ -13,7 +13,7 @@ paddlespeech version: 0.1.2
 
 ## Chunk Conformer
 paddle version: 2.2.2  
-paddlespeech version: 0.1.2
+paddlespeech version: 0.1.2  
 Need set `decoding.decoding_chunk_size=16` when decoding.
 
 | Model | Params | Config | Augmentation| Test set | Decode method | Chunk Size & Left Chunks | Loss | CER |  

From 0ffe1f91143b0489fd38be90747afcbb5e61fedc Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Mon, 28 Mar 2022 03:35:55 +0000
Subject: [PATCH 082/126] replace kaidi_fbank with paddleaudio

---
 examples/aishell/asr1/conf/preprocess.yaml   |  9 ++--
 paddlespeech/s2t/transform/spectrogram.py    | 45 ++++++++++++++++++++
 paddlespeech/s2t/transform/transformation.py |  1 +
 3 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/examples/aishell/asr1/conf/preprocess.yaml b/examples/aishell/asr1/conf/preprocess.yaml
index f7f4c58d..a20ff2ab 100644
--- a/examples/aishell/asr1/conf/preprocess.yaml
+++ b/examples/aishell/asr1/conf/preprocess.yaml
@@ -3,8 +3,9 @@ process:
   - type: fbank_kaldi
     fs: 16000
     n_mels: 80
-    n_shift: 160
-    win_length: 400
+    n_frame_length: 25
+    n_frame_shift: 10
+    energy_floor: 0.0
     dither: 0.1
   - type: cmvn_json
     cmvn_path: data/mean_std.json
@@ -23,7 +24,3 @@ process:
     n_mask: 2
     inplace: true
     replace_with_zero: false
-
-
-
-
diff --git a/paddlespeech/s2t/transform/spectrogram.py b/paddlespeech/s2t/transform/spectrogram.py
index 889cd349..f779b07d 100644
--- a/paddlespeech/s2t/transform/spectrogram.py
+++ b/paddlespeech/s2t/transform/spectrogram.py
@@ -14,8 +14,11 @@
 # Modified from espnet(https://github.com/espnet/espnet)
 import librosa
 import numpy as np
+import paddle
 from python_speech_features import logfbank
 
+import paddleaudio.compliance.kaldi as kaldi
+
 
 def stft(x,
          n_fft,
@@ -309,6 +312,48 @@ class IStft():
 
 
 class LogMelSpectrogramKaldi():
+    def __init__(self,
+                 fs=16000,
+                 n_mels=80,
+                 n_frame_length=25,
+                 n_frame_shift=10,
+                 energy_floor=0.0,
+                 dither=0.1):
+        self.fs = fs
+        self.n_mels = n_mels
+        self.n_frame_length = n_frame_length
+        self.n_frame_shift = n_frame_shift
+        self.energy_floor = energy_floor
+        self.dither = dither
+
+    def __repr__(self):
+        return (
+            "{name}(fs={fs}, n_mels={n_mels}, "
+            "n_frame_shift={n_frame_shift}, n_frame_length={n_frame_length}, "
+            "dither={dither}))".format(
+                name=self.__class__.__name__,
+                fs=self.fs,
+                n_mels=self.n_mels,
+                n_frame_shift=self.n_frame_shift,
+                n_frame_length=self.n_frame_length,
+                dither=self.dither, ))
+
+    def __call__(self, x, train):
+        dither = self.dither if train else 0.0
+        waveform = paddle.to_tensor(np.expand_dims(x, 0), dtype=paddle.float32)
+        mat = kaldi.fbank(
+            waveform,
+            n_mels=self.n_mels,
+            frame_length=self.n_frame_length,
+            frame_shift=self.n_frame_shift,
+            dither=dither,
+            energy_floor=self.energy_floor,
+            sr=self.fs)
+        mat = np.squeeze(mat.numpy())
+        return mat
+
+
+class LogMelSpectrogramKaldi_decay():
     def __init__(
             self,
             fs=16000,
diff --git a/paddlespeech/s2t/transform/transformation.py b/paddlespeech/s2t/transform/transformation.py
index 381b0cdc..3b433cb0 100644
--- a/paddlespeech/s2t/transform/transformation.py
+++ b/paddlespeech/s2t/transform/transformation.py
@@ -31,6 +31,7 @@ import_alias = dict(
     freq_mask="paddlespeech.s2t.transform.spec_augment:FreqMask",
     spec_augment="paddlespeech.s2t.transform.spec_augment:SpecAugment",
     speed_perturbation="paddlespeech.s2t.transform.perturb:SpeedPerturbation",
+    speed_perturbation_sox="paddlespeech.s2t.transform.perturb:SpeedPerturbationSox",
     volume_perturbation="paddlespeech.s2t.transform.perturb:VolumePerturbation",
     noise_injection="paddlespeech.s2t.transform.perturb:NoiseInjection",
     bandpass_perturbation="paddlespeech.s2t.transform.perturb:BandpassPerturbation",

From bc5ae43d3a5ecdb3fbcea92d18a054ddc260816a Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Mon, 28 Mar 2022 04:49:10 +0000
Subject: [PATCH 083/126] restructure expand in length_regulator.py for
 paddle2onnx, test=tts

---
 examples/csmsc/tts3/README.md                 |  4 +++-
 .../t2s/modules/predictor/length_regulator.py | 24 ++++++++++++-------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/examples/csmsc/tts3/README.md b/examples/csmsc/tts3/README.md
index 7b803526..a31203f8 100644
--- a/examples/csmsc/tts3/README.md
+++ b/examples/csmsc/tts3/README.md
@@ -227,7 +227,9 @@ Pretrained FastSpeech2 model with no silence in the edge of audios:
 - [fastspeech2_nosil_baker_ckpt_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_ckpt_0.4.zip)
 - [fastspeech2_conformer_baker_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_conformer_baker_ckpt_0.5.zip)
 
-The static model can be downloaded here [fastspeech2_nosil_baker_static_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_static_0.4.zip).
+The static model can be downloaded here:
+[fastspeech2_nosil_baker_static_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_static_0.4.zip).
+[fastspeech2_csmsc_static_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_csmsc_static_0.2.0.zip)
 
 Model | Step | eval/loss | eval/l1_loss | eval/duration_loss | eval/pitch_loss| eval/energy_loss 
 :-------------:| :------------:| :-----: | :-----: | :--------: |:--------:|:---------:
diff --git a/paddlespeech/t2s/modules/predictor/length_regulator.py b/paddlespeech/t2s/modules/predictor/length_regulator.py
index 2472c413..be788e6e 100644
--- a/paddlespeech/t2s/modules/predictor/length_regulator.py
+++ b/paddlespeech/t2s/modules/predictor/length_regulator.py
@@ -73,15 +73,21 @@ class LengthRegulator(nn.Layer):
         batch_size, t_enc = paddle.shape(durations)
         slens = paddle.sum(durations, -1)
         t_dec = paddle.max(slens)
-        M = paddle.zeros([batch_size, t_dec, t_enc])
-        for i in range(batch_size):
-            k = 0
-            for j in range(t_enc):
-                d = durations[i, j]
-                # If the d == 0, slice action is meaningless and not supported in paddle
-                if d >= 1:
-                    M[i, k:k + d, j] = 1
-                k += d
+        t_dec_1 = t_dec + 1
+        flatten_duration = paddle.cumsum(
+            paddle.reshape(durations, [batch_size * t_enc])) + 1
+        init = paddle.zeros(t_dec_1)
+        m_batch = batch_size * t_enc
+        M = paddle.zeros([t_dec_1, m_batch])
+        for i in range(m_batch):
+            d = flatten_duration[i]
+            m = paddle.concat(
+                [paddle.ones(d), paddle.zeros(t_dec_1 - d)], axis=0)
+            M[:, i] = m - init
+            init = m
+        M = paddle.reshape(M, shape=[t_dec_1, batch_size, t_enc])
+        M = M[1:, :, :]
+        M = paddle.transpose(M, (1, 0, 2))
         encodings = paddle.matmul(M, encodings)
         return encodings
 

From e52fc08c586e1da29bb5ccf1ebaf6b639e21412d Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Mon, 28 Mar 2022 05:53:31 +0000
Subject: [PATCH 084/126] update readme, test=doc

---
 examples/csmsc/tts3/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/csmsc/tts3/README.md b/examples/csmsc/tts3/README.md
index a31203f8..08bf2ee9 100644
--- a/examples/csmsc/tts3/README.md
+++ b/examples/csmsc/tts3/README.md
@@ -228,8 +228,8 @@ Pretrained FastSpeech2 model with no silence in the edge of audios:
 - [fastspeech2_conformer_baker_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_conformer_baker_ckpt_0.5.zip)
 
 The static model can be downloaded here:
-[fastspeech2_nosil_baker_static_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_static_0.4.zip).
-[fastspeech2_csmsc_static_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_csmsc_static_0.2.0.zip)
+- [fastspeech2_nosil_baker_static_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_static_0.4.zip)
+- [fastspeech2_csmsc_static_0.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_csmsc_static_0.2.0.zip)
 
 Model | Step | eval/loss | eval/l1_loss | eval/duration_loss | eval/pitch_loss| eval/energy_loss 
 :-------------:| :------------:| :-----: | :-----: | :--------: |:--------:|:---------:

From 5db7e6382a50d8b9e22bce3ff186ff037febed82 Mon Sep 17 00:00:00 2001
From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com>
Date: Mon, 28 Mar 2022 14:57:39 +0800
Subject: [PATCH 085/126] test=doc

---
 examples/aishell/asr1/conf/chunk_conformer.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/aishell/asr1/conf/chunk_conformer.yaml b/examples/aishell/asr1/conf/chunk_conformer.yaml
index 1ad77f97..9f70e4c5 100644
--- a/examples/aishell/asr1/conf/chunk_conformer.yaml
+++ b/examples/aishell/asr1/conf/chunk_conformer.yaml
@@ -39,6 +39,7 @@ model_conf:
     ctc_weight: 0.3
     lsm_weight: 0.1     # label smoothing option
     length_normalized_loss: false
+    init_type: 'kaiming_uniform' 
 
 ###########################################
 #                   Data                  #
@@ -61,12 +62,12 @@ feat_dim: 80
 stride_ms: 10.0
 window_ms: 25.0
 sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs 
-batch_size: 64
+batch_size: 32
 maxlen_in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
 maxlen_out: 150  # if output length > maxlen-out, batchsize is automatically reduced
 minibatches: 0 # for debug
 batch_count: auto
-batch_bins: 0 
+batch_bins: 0
 batch_frames_in: 0
 batch_frames_out: 0
 batch_frames_inout: 0
@@ -77,13 +78,13 @@ num_encs: 1
 ###########################################
 #                 Training                #
 ###########################################
-n_epoch: 240 
-accum_grad: 2
+n_epoch: 180 
+accum_grad: 1
 global_grad_clip: 5.0
 dist_sampler: True
 optim: adam
 optim_conf:
-  lr: 0.002
+  lr: 0.001
   weight_decay: 1.0e-6
 scheduler: warmuplr
 scheduler_conf:
@@ -93,4 +94,3 @@ log_interval: 100
 checkpoint:
   kbest_n: 50
   latest_n: 5
-

From ed7113f320e3f72203c76fff40d6093968a02da1 Mon Sep 17 00:00:00 2001
From: xiongxinlei <xiongxinlei@baidu.com>
Date: Mon, 28 Mar 2022 15:14:59 +0800
Subject: [PATCH 086/126] change the vector output to numpy.array

---
 dataset/voxceleb/voxceleb1.py                 |  24 ++-
 dataset/voxceleb/voxceleb2.py                 |  29 +--
 demos/audio_searching/src/operations/load.py  |   5 +-
 demos/speaker_verification/README.md          | 178 ++++++++----------
 demos/speaker_verification/README_cn.md       | 176 ++++++++---------
 examples/voxceleb/README.md                   |   6 -
 examples/voxceleb/sv0/local/data_prepare.py   |   1 -
 paddleaudio/paddleaudio/datasets/__init__.py  |   2 +-
 .../paddleaudio/datasets/rirs_noises.py       |   4 -
 paddleaudio/paddleaudio/datasets/voxceleb.py  |   4 +-
 paddlespeech/cli/vector/infer.py              |   8 +-
 paddlespeech/vector/exps/ecapa_tdnn/test.py   |   2 -
 paddlespeech/vector/io/augment.py             |   2 -
 paddlespeech/vector/io/signal_processing.py   |   2 -
 tests/unit/vector/conftest.py                 |  16 +-
 tests/unit/vector/test_augment.py             |   8 -
 16 files changed, 211 insertions(+), 256 deletions(-)

diff --git a/dataset/voxceleb/voxceleb1.py b/dataset/voxceleb/voxceleb1.py
index d0978d9d..90586200 100644
--- a/dataset/voxceleb/voxceleb1.py
+++ b/dataset/voxceleb/voxceleb1.py
@@ -63,13 +63,15 @@ TEST_TARGET_DATA = "vox1_test_wav.zip vox1_test_wav.zip 185fdc63c3c739954633d503
 
 TRIAL_BASE_URL = "https://www.robots.ox.ac.uk/~vgg/data/voxceleb/meta/"
 TRIAL_LIST = {
-    "veri_test.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7",             # voxceleb1
-    "veri_test2.txt": "b73110731c9223c1461fe49cb48dddfc",            # voxceleb1(cleaned)
-    "list_test_hard.txt": "21c341b6b2168eea2634df0fb4b8fff1",        # voxceleb1-H
-    "list_test_hard2.txt": "857790e09d579a68eb2e339a090343c8",       # voxceleb1-H(cleaned)
-    "list_test_all.txt": "b9ecf7aa49d4b656aa927a8092844e4a",         # voxceleb1-E
-    "list_test_all2.txt": "a53e059deb562ffcfc092bf5d90d9f3a"         # voxceleb1-E(cleaned)
-    }
+    "veri_test.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7",  # voxceleb1
+    "veri_test2.txt": "b73110731c9223c1461fe49cb48dddfc",  # voxceleb1(cleaned)
+    "list_test_hard.txt": "21c341b6b2168eea2634df0fb4b8fff1",  # voxceleb1-H
+    "list_test_hard2.txt":
+    "857790e09d579a68eb2e339a090343c8",  # voxceleb1-H(cleaned)
+    "list_test_all.txt": "b9ecf7aa49d4b656aa927a8092844e4a",  # voxceleb1-E
+    "list_test_all2.txt":
+    "a53e059deb562ffcfc092bf5d90d9f3a"  # voxceleb1-E(cleaned)
+}
 
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(
@@ -176,6 +178,7 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path,
     # create the manifest file
     create_manifest(data_dir=target_dir, manifest_path_prefix=manifest_path)
 
+
 def prepare_trial(base_url, data_list, target_dir):
     if not os.path.exists(target_dir):
         os.makedirs(target_dir)
@@ -185,10 +188,12 @@ def prepare_trial(base_url, data_list, target_dir):
         if not os.path.exists(os.path.join(target_dir, trial)):
             download_url = " --no-check-certificate " + base_url + "/" + trial
             download(url=download_url, md5sum=md5sum, target_dir=target_dir)
+
+
 def main():
     if args.target_dir.startswith('~'):
         args.target_dir = os.path.expanduser(args.target_dir)
-    
+
     # prepare the vox1 dev data
     prepare_dataset(
         base_url=BASE_URL,
@@ -209,8 +214,7 @@ def main():
     prepare_trial(
         base_url=TRIAL_BASE_URL,
         data_list=TRIAL_LIST,
-        target_dir=os.path.dirname(args.manifest_prefix)
-    )
+        target_dir=os.path.dirname(args.manifest_prefix))
 
     print("Manifest prepare done!")
 
diff --git a/dataset/voxceleb/voxceleb2.py b/dataset/voxceleb/voxceleb2.py
index ef7bb230..22a2e2ff 100644
--- a/dataset/voxceleb/voxceleb2.py
+++ b/dataset/voxceleb/voxceleb2.py
@@ -22,12 +22,10 @@ import codecs
 import glob
 import json
 import os
-import subprocess
 from pathlib import Path
 
 import soundfile
 
-from utils.utility import check_md5sum
 from utils.utility import download
 from utils.utility import unzip
 
@@ -40,9 +38,8 @@ BASE_URL = "--no-check-certificate https://www.robots.ox.ac.uk/~vgg/data/voxcele
 DEV_DATA_URL = BASE_URL + '/vox2_aac.zip'
 DEV_MD5SUM = "bbc063c46078a602ca71605645c2a402"
 
-
 # test data
-TEST_DATA_URL = BASE_URL +  '/vox2_test_aac.zip'
+TEST_DATA_URL = BASE_URL + '/vox2_test_aac.zip'
 TEST_MD5SUM = "0d2b3ea430a821c33263b5ea37ede312"
 
 parser = argparse.ArgumentParser(description=__doc__)
@@ -56,14 +53,16 @@ parser.add_argument(
     default="manifest",
     type=str,
     help="Filepath prefix for output manifests. (default: %(default)s)")
-parser.add_argument("--download", 
-                    default=False, 
-                    action="store_true", 
-                    help="Download the voxceleb2 dataset. (default: %(default)s)")
-parser.add_argument("--generate", 
-                    default=False, 
-                    action="store_true", 
-                    help="Generate the manifest files. (default: %(default)s)")
+parser.add_argument(
+    "--download",
+    default=False,
+    action="store_true",
+    help="Download the voxceleb2 dataset. (default: %(default)s)")
+parser.add_argument(
+    "--generate",
+    default=False,
+    action="store_true",
+    help="Generate the manifest files. (default: %(default)s)")
 
 args = parser.parse_args()
 
@@ -138,7 +137,7 @@ def download_dataset(url, md5sum, target_dir, dataset):
 def main():
     if args.target_dir.startswith('~'):
         args.target_dir = os.path.expanduser(args.target_dir)
-    
+
     # download and unpack the vox2-dev data
     print("download: {}".format(args.download))
     if args.download:
@@ -157,7 +156,9 @@ def main():
         print("VoxCeleb2 download is done!")
 
     if args.generate:
-        create_manifest(args.target_dir, manifest_path_prefix=args.manifest_prefix)
+        create_manifest(
+            args.target_dir, manifest_path_prefix=args.manifest_prefix)
+
 
 if __name__ == '__main__':
     main()
diff --git a/demos/audio_searching/src/operations/load.py b/demos/audio_searching/src/operations/load.py
index 80b6375f..7a295bf3 100644
--- a/demos/audio_searching/src/operations/load.py
+++ b/demos/audio_searching/src/operations/load.py
@@ -26,8 +26,9 @@ def get_audios(path):
     """
     supported_formats = [".wav", ".mp3", ".ogg", ".flac", ".m4a"]
     return [
-        item for sublist in [[os.path.join(dir, file) for file in files]
-                             for dir, _, files in list(os.walk(path))]
+        item
+        for sublist in [[os.path.join(dir, file) for file in files]
+                        for dir, _, files in list(os.walk(path))]
         for item in sublist if os.path.splitext(item)[1] in supported_formats
     ]
 
diff --git a/demos/speaker_verification/README.md b/demos/speaker_verification/README.md
index c4d10ccf..8739d402 100644
--- a/demos/speaker_verification/README.md
+++ b/demos/speaker_verification/README.md
@@ -46,56 +46,46 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
 
   Output:
 
-```bash
-  demo  {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
-         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
-       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
-         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
-         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
-        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
-        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
-        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
-        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
-         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
-         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
-       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
-         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
-         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
-       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
-        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
-         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
-         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
-       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
-        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
-         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
-        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
-        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
-         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
-        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
-         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
-         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
-         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
-        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
-       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
-        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
-         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
-        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
-        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
-        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
-        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
-         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
-        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
-        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
-       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
-        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
-       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
-         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
-         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
-         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
-        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
-         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
-        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
-      dtype=float32)}
+  ```bash
+    demo [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+      -3.04878      1.611095    10.127234   -10.534177   -15.821609
+      1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+    -11.343508     2.3385992   -8.719341    14.213509    15.404744
+      -0.39327756   6.338786     2.688887     8.7104025   17.469526
+      -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+      8.013747    13.891729    -9.926753     5.655307    -5.9422326
+    -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+      3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+      1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+      -6.40137     23.63524      2.9711294  -22.708025     9.93719
+      20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+      15.999649     3.3004563   12.747926    15.429879     4.7849145
+      5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+      15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+      -9.224193    14.568347   -10.568833     4.982321    -4.342062
+      0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+      -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+    -11.54324      7.681869     0.44475392   9.708182    -8.932846
+      0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+      2.9079323    6.049952     9.275183   -18.078873     6.2983274
+      -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+      4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+      18.495346   -14.293832     7.89578      2.2714825   22.976387
+      -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+    -11.924197     2.171869     2.0423572   -6.173772    10.778437
+      25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+      -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+      -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+      -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+      -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+    -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+      -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+      8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+      10.833007    -6.717991     4.504732    13.4244375    1.1306485
+      7.3435574    1.400918    14.704036    -9.501399     7.2315617
+      -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+      6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+      -3.2701402  -11.508579  ]
   ```
 
 - Python API
@@ -118,55 +108,45 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   Output:
   ```bash
   # Vector Result:
-   {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
-         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
-       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
-         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
-         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
-        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
-        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
-        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
-        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
-         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
-         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
-       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
-         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
-         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
-       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
-        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
-         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
-         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
-       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
-        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
-         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
-        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
-        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
-         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
-        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
-         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
-         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
-         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
-        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
-       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
-        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
-         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
-        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
-        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
-        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
-        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
-         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
-        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
-        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
-       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
-        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
-       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
-         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
-         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
-         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
-        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
-         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
-        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
-      dtype=float32)}
+   [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+      -3.04878      1.611095    10.127234   -10.534177   -15.821609
+      1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+    -11.343508     2.3385992   -8.719341    14.213509    15.404744
+      -0.39327756   6.338786     2.688887     8.7104025   17.469526
+      -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+      8.013747    13.891729    -9.926753     5.655307    -5.9422326
+    -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+      3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+      1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+      -6.40137     23.63524      2.9711294  -22.708025     9.93719
+      20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+      15.999649     3.3004563   12.747926    15.429879     4.7849145
+      5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+      15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+      -9.224193    14.568347   -10.568833     4.982321    -4.342062
+      0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+      -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+    -11.54324      7.681869     0.44475392   9.708182    -8.932846
+      0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+      2.9079323    6.049952     9.275183   -18.078873     6.2983274
+      -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+      4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+      18.495346   -14.293832     7.89578      2.2714825   22.976387
+      -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+    -11.924197     2.171869     2.0423572   -6.173772    10.778437
+      25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+      -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+      -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+      -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+      -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+    -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+      -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+      8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+      10.833007    -6.717991     4.504732    13.4244375    1.1306485
+      7.3435574    1.400918    14.704036    -9.501399     7.2315617
+      -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+      6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+      -3.2701402  -11.508579  ]
   ```
 
 ### 4.Pretrained Models
diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md
index e2799b75..fe8949b3 100644
--- a/demos/speaker_verification/README_cn.md
+++ b/demos/speaker_verification/README_cn.md
@@ -45,55 +45,45 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
 
   输出：
   ```bash
-  demo  {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
-         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
-       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
-         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
-         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
-        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
-        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
-        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
-        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
-         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
-         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
-       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
-         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
-         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
-       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
-        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
-         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
-         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
-       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
-        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
-         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
-        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
-        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
-         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
-        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
-         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
-         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
-         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
-        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
-       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
-        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
-         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
-        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
-        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
-        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
-        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
-         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
-        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
-        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
-       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
-        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
-       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
-         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
-         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
-         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
-        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
-         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
-        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
-      dtype=float32)}
+  demo  [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+      -3.04878      1.611095    10.127234   -10.534177   -15.821609
+      1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+    -11.343508     2.3385992   -8.719341    14.213509    15.404744
+      -0.39327756   6.338786     2.688887     8.7104025   17.469526
+      -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+      8.013747    13.891729    -9.926753     5.655307    -5.9422326
+    -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+      3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+      1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+      -6.40137     23.63524      2.9711294  -22.708025     9.93719
+      20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+      15.999649     3.3004563   12.747926    15.429879     4.7849145
+      5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+      15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+      -9.224193    14.568347   -10.568833     4.982321    -4.342062
+      0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+      -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+    -11.54324      7.681869     0.44475392   9.708182    -8.932846
+      0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+      2.9079323    6.049952     9.275183   -18.078873     6.2983274
+      -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+      4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+      18.495346   -14.293832     7.89578      2.2714825   22.976387
+      -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+    -11.924197     2.171869     2.0423572   -6.173772    10.778437
+      25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+      -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+      -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+      -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+      -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+    -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+      -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+      8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+      10.833007    -6.717991     4.504732    13.4244375    1.1306485
+      7.3435574    1.400918    14.704036    -9.501399     7.2315617
+      -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+      6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+      -3.2701402  -11.508579  ]
   ```
 
 - Python API
@@ -116,55 +106,45 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav
   输出：
   ```bash
   # Vector Result:
-   {'dim': 192, 'embedding': array([ -5.749211  ,   9.505463  ,  -8.200284  ,  -5.2075014 ,
-         5.3940268 ,  -3.04878   ,   1.611095  ,  10.127234  ,
-       -10.534177  , -15.821609  ,   1.2032688 ,  -0.35080156,
-         1.2629458 , -12.643498  ,  -2.5758228 , -11.343508  ,
-         2.3385992 ,  -8.719341  ,  14.213509  ,  15.404744  ,
-        -0.39327756,   6.338786  ,   2.688887  ,   8.7104025 ,
-        17.469526  ,  -8.77959   ,   7.0576906 ,   4.648855  ,
-        -1.3089896 , -23.294737  ,   8.013747  ,  13.891729  ,
-        -9.926753  ,   5.655307  ,  -5.9422326 , -22.842539  ,
-         0.6293588 , -18.46266   , -10.811862  ,   9.8192625 ,
-         3.0070958 ,   3.8072643 ,  -2.3861165 ,   3.0821571 ,
-       -14.739942  ,   1.7594414 ,  -0.6485091 ,   4.485623  ,
-         2.0207152 ,   7.264915  ,  -6.40137   ,  23.63524   ,
-         2.9711294 , -22.708025  ,   9.93719   ,  20.354511  ,
-       -10.324688  ,  -0.700492  ,  -8.783211  ,  -5.27593   ,
-        15.999649  ,   3.3004563 ,  12.747926  ,  15.429879  ,
-         4.7849145 ,   5.6699696 ,  -2.3826702 ,  10.605882  ,
-         3.9112158 ,   3.1500628 ,  15.859915  ,  -2.1832209 ,
-       -23.908653  ,  -6.4799504 ,  -4.5365124 ,  -9.224193  ,
-        14.568347  , -10.568833  ,   4.982321  ,  -4.342062  ,
-         0.0914714 ,  12.645902  ,  -5.74285   ,  -3.2141201 ,
-        -2.7173362 ,  -6.680575  ,   0.4757669 ,  -5.035051  ,
-        -6.7964664 ,  16.865469  , -11.54324   ,   7.681869  ,
-         0.44475392,   9.708182  ,  -8.932846  ,   0.4123232 ,
-        -4.361452  ,   1.3948607 ,   9.511665  ,   0.11667654,
-         2.9079323 ,   6.049952  ,   9.275183  , -18.078873  ,
-         6.2983274 ,  -0.7500531 ,  -2.725033  ,  -7.6027865 ,
-         3.3404543 ,   2.990815  ,   4.010979  ,  11.000591  ,
-        -2.8873312 ,   7.1352735 , -16.79663   ,  18.495346  ,
-       -14.293832  ,   7.89578   ,   2.2714825 ,  22.976387  ,
-        -4.875734  ,  -3.0836344 ,  -2.9999814 ,  13.751918  ,
-         6.448228  , -11.924197  ,   2.171869  ,   2.0423572 ,
-        -6.173772  ,  10.778437  ,  25.77281   ,  -4.9495463 ,
-        14.57806   ,   0.3044315 ,   2.6132357 ,  -7.591999  ,
-        -2.076944  ,   9.025118  ,   1.7834753 ,  -3.1799617 ,
-        -4.9401326 ,  23.465864  ,   5.1685796 ,  -9.018578  ,
-         9.037825  ,  -4.4150195 ,   6.859591  , -12.274467  ,
-        -0.88911164,   5.186309  ,  -3.9988663 , -13.638606  ,
-        -9.925445  ,  -0.06329413,  -3.6709652 , -12.397416  ,
-       -12.719869  ,  -1.395601  ,   2.1150916 ,   5.7381287 ,
-        -4.4691963 ,  -3.82819   ,  -0.84233856,  -1.1604277 ,
-       -13.490127  ,   8.731719  , -20.778936  , -11.495662  ,
-         5.8033476 ,  -4.752041  ,  10.833007  ,  -6.717991  ,
-         4.504732  ,  13.4244375 ,   1.1306485 ,   7.3435574 ,
-         1.400918  ,  14.704036  ,  -9.501399  ,   7.2315617 ,
-        -6.417456  ,   1.3333273 ,  11.872697  ,  -0.30664724,
-         8.8845    ,   6.5569253 ,   4.7948146 ,   0.03662816,
-        -8.704245  ,   6.224871  ,  -3.2701402 , -11.508579  ],
-      dtype=float32)}
+   [ -5.749211     9.505463    -8.200284    -5.2075014    5.3940268
+      -3.04878      1.611095    10.127234   -10.534177   -15.821609
+      1.2032688   -0.35080156   1.2629458  -12.643498    -2.5758228
+    -11.343508     2.3385992   -8.719341    14.213509    15.404744
+      -0.39327756   6.338786     2.688887     8.7104025   17.469526
+      -8.77959      7.0576906    4.648855    -1.3089896  -23.294737
+      8.013747    13.891729    -9.926753     5.655307    -5.9422326
+    -22.842539     0.6293588  -18.46266    -10.811862     9.8192625
+      3.0070958    3.8072643   -2.3861165    3.0821571  -14.739942
+      1.7594414   -0.6485091    4.485623     2.0207152    7.264915
+      -6.40137     23.63524      2.9711294  -22.708025     9.93719
+      20.354511   -10.324688    -0.700492    -8.783211    -5.27593
+      15.999649     3.3004563   12.747926    15.429879     4.7849145
+      5.6699696   -2.3826702   10.605882     3.9112158    3.1500628
+      15.859915    -2.1832209  -23.908653    -6.4799504   -4.5365124
+      -9.224193    14.568347   -10.568833     4.982321    -4.342062
+      0.0914714   12.645902    -5.74285     -3.2141201   -2.7173362
+      -6.680575     0.4757669   -5.035051    -6.7964664   16.865469
+    -11.54324      7.681869     0.44475392   9.708182    -8.932846
+      0.4123232   -4.361452     1.3948607    9.511665     0.11667654
+      2.9079323    6.049952     9.275183   -18.078873     6.2983274
+      -0.7500531   -2.725033    -7.6027865    3.3404543    2.990815
+      4.010979    11.000591    -2.8873312    7.1352735  -16.79663
+      18.495346   -14.293832     7.89578      2.2714825   22.976387
+      -4.875734    -3.0836344   -2.9999814   13.751918     6.448228
+    -11.924197     2.171869     2.0423572   -6.173772    10.778437
+      25.77281     -4.9495463   14.57806      0.3044315    2.6132357
+      -7.591999    -2.076944     9.025118     1.7834753   -3.1799617
+      -4.9401326   23.465864     5.1685796   -9.018578     9.037825
+      -4.4150195    6.859591   -12.274467    -0.88911164   5.186309
+      -3.9988663  -13.638606    -9.925445    -0.06329413  -3.6709652
+    -12.397416   -12.719869    -1.395601     2.1150916    5.7381287
+      -4.4691963   -3.82819     -0.84233856  -1.1604277  -13.490127
+      8.731719   -20.778936   -11.495662     5.8033476   -4.752041
+      10.833007    -6.717991     4.504732    13.4244375    1.1306485
+      7.3435574    1.400918    14.704036    -9.501399     7.2315617
+      -6.417456     1.3333273   11.872697    -0.30664724   8.8845
+      6.5569253    4.7948146    0.03662816  -8.704245     6.224871
+      -3.2701402  -11.508579  ]
   ```
 
 ### 4.预训练模型
diff --git a/examples/voxceleb/README.md b/examples/voxceleb/README.md
index a2e58e00..42f8903e 100644
--- a/examples/voxceleb/README.md
+++ b/examples/voxceleb/README.md
@@ -48,9 +48,3 @@ You can do the conversion using ffmpeg  https://gist.github.com/seungwonpark/4f2
 |VoxCeleb1-H(cleaned) |list_test_hard2.txt | 550894 | 275488 | 275406 |
 |VoxCeleb1-E | list_test_all.txt | 581480 | 290743 | 290737 | 
 |VoxCeleb1-E(cleaned) | list_test_all2.txt |579818 |289921 |289897 |
-
-
-
-
-
-
diff --git a/examples/voxceleb/sv0/local/data_prepare.py b/examples/voxceleb/sv0/local/data_prepare.py
index 19ba41b8..03d05400 100644
--- a/examples/voxceleb/sv0/local/data_prepare.py
+++ b/examples/voxceleb/sv0/local/data_prepare.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import argparse
-import os
 
 import paddle
 from yacs.config import CfgNode
diff --git a/paddleaudio/paddleaudio/datasets/__init__.py b/paddleaudio/paddleaudio/datasets/__init__.py
index 6f44e977..ebd4af98 100644
--- a/paddleaudio/paddleaudio/datasets/__init__.py
+++ b/paddleaudio/paddleaudio/datasets/__init__.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 from .esc50 import ESC50
 from .gtzan import GTZAN
+from .rirs_noises import OpenRIRNoise
 from .tess import TESS
 from .urban_sound import UrbanSound8K
 from .voxceleb import VoxCeleb
-from .rirs_noises import OpenRIRNoise
diff --git a/paddleaudio/paddleaudio/datasets/rirs_noises.py b/paddleaudio/paddleaudio/datasets/rirs_noises.py
index 80bb2d74..68639a60 100644
--- a/paddleaudio/paddleaudio/datasets/rirs_noises.py
+++ b/paddleaudio/paddleaudio/datasets/rirs_noises.py
@@ -13,12 +13,9 @@
 # limitations under the License.
 import collections
 import csv
-import glob
 import os
 import random
-from typing import Dict
 from typing import List
-from typing import Tuple
 
 from paddle.io import Dataset
 from tqdm import tqdm
@@ -26,7 +23,6 @@ from tqdm import tqdm
 from ..backends import load as load_audio
 from ..backends import save as save_wav
 from ..utils import DATA_HOME
-from ..utils import decompress
 from ..utils.download import download_and_decompress
 from .dataset import feat_funcs
 
diff --git a/paddleaudio/paddleaudio/datasets/voxceleb.py b/paddleaudio/paddleaudio/datasets/voxceleb.py
index b9b8c271..3f72b5f2 100644
--- a/paddleaudio/paddleaudio/datasets/voxceleb.py
+++ b/paddleaudio/paddleaudio/datasets/voxceleb.py
@@ -17,9 +17,7 @@ import glob
 import os
 import random
 from multiprocessing import cpu_count
-from typing import Dict
 from typing import List
-from typing import Tuple
 
 from paddle.io import Dataset
 from pathos.multiprocessing import Pool
@@ -135,7 +133,7 @@ class VoxCeleb(Dataset):
         # so, we check the vox1/wav dir status
         print(f"wav base path: {self.wav_path}")
         if not os.path.isdir(self.wav_path):
-            print(f"start to download the voxceleb1 dataset")
+            print("start to download the voxceleb1 dataset")
             download_and_decompress(  # multi-zip parts concatenate to vox1_dev_wav.zip
                 self.archieves_audio_dev,
                 self.base_path,
diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py
index 79d3b5db..175a9723 100644
--- a/paddlespeech/cli/vector/infer.py
+++ b/paddlespeech/cli/vector/infer.py
@@ -82,7 +82,10 @@ class VectorExecutor(BaseExecutor):
             choices=["spk"],
             help="task type in vector domain")
         self.parser.add_argument(
-            "--input", type=str, default=None, help="Audio file to extract embedding.")
+            "--input",
+            type=str,
+            default=None,
+            help="Audio file to extract embedding.")
         self.parser.add_argument(
             "--sample_rate",
             type=int,
@@ -344,8 +347,7 @@ class VectorExecutor(BaseExecutor):
             Union[str, os.PathLike]: audio embedding info
         """
         embedding = self._outputs["embedding"]
-        dim = embedding.shape[0]
-        return {"dim": dim, "embedding": embedding}
+        return embedding
 
     def preprocess(self, model_type: str, input_file: Union[str, os.PathLike]):
         """Extract the audio feat
diff --git a/paddlespeech/vector/exps/ecapa_tdnn/test.py b/paddlespeech/vector/exps/ecapa_tdnn/test.py
index 76832fd8..d0de6dc5 100644
--- a/paddlespeech/vector/exps/ecapa_tdnn/test.py
+++ b/paddlespeech/vector/exps/ecapa_tdnn/test.py
@@ -12,12 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import argparse
-import ast
 import os
 
 import numpy as np
 import paddle
-import paddle.nn.functional as F
 from paddle.io import BatchSampler
 from paddle.io import DataLoader
 from tqdm import tqdm
diff --git a/paddlespeech/vector/io/augment.py b/paddlespeech/vector/io/augment.py
index 6e508c37..3baace13 100644
--- a/paddlespeech/vector/io/augment.py
+++ b/paddlespeech/vector/io/augment.py
@@ -14,7 +14,6 @@
 # this is modified from SpeechBrain
 # https://github.com/speechbrain/speechbrain/blob/085be635c07f16d42cd1295045bc46c407f1e15b/speechbrain/lobes/augment.py
 import math
-import os
 from typing import List
 
 import numpy as np
@@ -22,7 +21,6 @@ import paddle
 import paddle.nn as nn
 import paddle.nn.functional as F
 
-from paddleaudio import load as load_audio
 from paddleaudio.datasets.rirs_noises import OpenRIRNoise
 from paddlespeech.s2t.utils.log import Log
 from paddlespeech.vector.io.signal_processing import compute_amplitude
diff --git a/paddlespeech/vector/io/signal_processing.py b/paddlespeech/vector/io/signal_processing.py
index a61bf554..ee939bdb 100644
--- a/paddlespeech/vector/io/signal_processing.py
+++ b/paddlespeech/vector/io/signal_processing.py
@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import math
-
 import numpy as np
 import paddle
 
diff --git a/tests/unit/vector/conftest.py b/tests/unit/vector/conftest.py
index 7cac519b..cc5dccd1 100644
--- a/tests/unit/vector/conftest.py
+++ b/tests/unit/vector/conftest.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
 def pytest_addoption(parser):
     parser.addoption("--device", action="store", default="cpu")
 
@@ -8,4 +23,3 @@ def pytest_generate_tests(metafunc):
     option_value = metafunc.config.option.device
     if "device" in metafunc.fixturenames and option_value is not None:
         metafunc.parametrize("device", [option_value])
-
diff --git a/tests/unit/vector/test_augment.py b/tests/unit/vector/test_augment.py
index 21d75bb3..5ae01da4 100644
--- a/tests/unit/vector/test_augment.py
+++ b/tests/unit/vector/test_augment.py
@@ -11,15 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-
-import numpy as np
 import paddle
-import paddle.nn as nn
-import paddle.nn.functional as F
-from paddle.io import BatchSampler
-from paddle.io import DataLoader
-from paddle.io import Dataset
 
 
 def test_add_noise(tmpdir, device):

From d6a97a3a9fc566a803176623e4ae0b523e9cbe39 Mon Sep 17 00:00:00 2001
From: KP <109694228@qq.com>
Date: Mon, 28 Mar 2022 17:28:03 +0800
Subject: [PATCH 087/126] Update requirements.

---
 paddleaudio/setup.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/paddleaudio/setup.py b/paddleaudio/setup.py
index 930f86e4..e08b88a3 100644
--- a/paddleaudio/setup.py
+++ b/paddleaudio/setup.py
@@ -82,13 +82,9 @@ setuptools.setup(
     ],
     python_requires='>=3.6',
     install_requires=[
-        'numpy >= 1.15.0',
-        'scipy >= 1.0.0',
-        'resampy >= 0.2.2',
-        'soundfile >= 0.9.0',
-        'colorlog',
-        'dtaidistance >= 2.3.6',
-        'mcd >= 0.4',
+        'numpy >= 1.15.0', 'scipy >= 1.0.0', 'resampy >= 0.2.2',
+        'soundfile >= 0.9.0', 'colorlog', 'dtaidistance == 2.3.1', 'mcd >= 0.4',
+        'pathos'
     ],
     extras_require={
         'test': [

From 606c46c9ced7a077175f37de9678fbc872fad900 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Mon, 28 Mar 2022 12:15:37 +0000
Subject: [PATCH 088/126] more comment of code;

---
 speechx/README.md                             |   4 +-
 speechx/build.sh                              |   5 +-
 speechx/cmake/FindGFortranLibs.cmake          | 145 ++++++++++++++++++
 speechx/cmake/external/openblas.cmake         |  21 +++
 speechx/cmake/external/openfst.cmake          |   9 +-
 speechx/speechx/frontend/feature_cache.cc     |   4 +
 speechx/speechx/frontend/feature_cache.h      |  13 +-
 .../frontend/feature_extractor_interface.h    |  21 ++-
 speechx/speechx/frontend/linear_spectrogram.h |   8 +-
 9 files changed, 211 insertions(+), 19 deletions(-)
 create mode 100644 speechx/cmake/FindGFortranLibs.cmake

diff --git a/speechx/README.md b/speechx/README.md
index 7d73b61c..07449f9f 100644
--- a/speechx/README.md
+++ b/speechx/README.md
@@ -5,7 +5,7 @@
 We develop under:
 * docker - registry.baidubce.com/paddlepaddle/paddle:2.1.1-gpu-cuda10.2-cudnn7
 * os - Ubuntu 16.04.7 LTS
-* gcc/g++ - 8.2.0
+* ** gcc/g++/gfortran - 8.2.0 **
 * cmake - 3.16.0
 
 > We make sure all things work fun under docker, and recommend using it to develop and deploy.
@@ -29,6 +29,8 @@ nvidia-docker run --privileged  --net=host --ipc=host -it --rm -v $PWD:/workspac
 
 2. Build `speechx` and `examples`.
 
+> Do not source venv.
+
 ```
 pushd /path/to/speechx
 ./build.sh
diff --git a/speechx/build.sh b/speechx/build.sh
index 3e9600d5..8e36d233 100755
--- a/speechx/build.sh
+++ b/speechx/build.sh
@@ -2,8 +2,7 @@
 
 # the build script had verified in the paddlepaddle docker image.
 # please follow the instruction below to install PaddlePaddle image.
-# https://www.paddlepaddle.org.cn/documentation/docs/zh/install/docker/linux-docker.html
-
+# https://www.paddlepaddle.org.cn/documentation/docs/zh/install/docker/linux-docker.html 
 boost_SOURCE_DIR=$PWD/fc_patch/boost-src
 if [ ! -d ${boost_SOURCE_DIR} ]; then wget -c https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz 
   tar xzfv boost_1_75_0.tar.gz
@@ -23,6 +22,6 @@ cd build
 cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR}
 #cmake .. 
 
-make -j1
+make -j10
 
 cd -
diff --git a/speechx/cmake/FindGFortranLibs.cmake b/speechx/cmake/FindGFortranLibs.cmake
new file mode 100644
index 00000000..763f7883
--- /dev/null
+++ b/speechx/cmake/FindGFortranLibs.cmake
@@ -0,0 +1,145 @@
+#.rst:
+# FindGFortranLibs
+# --------
+#  https://github.com/Argonne-National-Laboratory/PIPS/blob/master/cmake/Modules/FindGFortranLibs.cmake
+#  https://enccs.github.io/cmake-workshop/cxx-fortran/
+#
+# Find gcc Fortran compiler & library paths
+#
+# The module defines the following variables:
+#
+# ::
+#
+#
+#   GFORTRANLIBS_FOUND - true if system has gfortran
+#   LIBGFORTRAN_LIBRARIES - path to libgfortran
+#   LIBQUADMATH_LIBRARIES - path to libquadmath
+#   GFORTRAN_LIBARIES_DIR - directory containing libgfortran, libquadmath
+#   GFORTRAN_INCLUDE_DIR - directory containing gfortran/gcc headers
+#   LIBGOMP_LIBRARIES - path to libgomp
+#   LIBGOMP_INCLUDE_DIR - directory containing omp.h header
+#   GFORTRAN_VERSION_STRING - version of gfortran found
+#
+set(CMAKE_REQUIRED_QUIET ${LIBIOMP_FIND_QUIETLY})
+
+if(NOT CMAKE_REQUIRED_QUIET)
+  message(STATUS "Looking for gfortran related libraries...")
+endif()
+
+enable_language(Fortran)
+if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+
+  # Basically, call "gfortran -v" to dump compiler info to the string
+  # GFORTRAN_VERBOSE_STR, which will be used to get necessary paths
+  message(STATUS "Extracting library and header information by calling 'gfortran -v'...")
+  execute_process(COMMAND "${CMAKE_Fortran_COMPILER}" "-v" ERROR_VARIABLE
+    GFORTRAN_VERBOSE_STR RESULT_VARIABLE FLAG)
+
+  # For debugging
+  message(STATUS "'gfortran -v' returned:")
+  message(STATUS "${GFORTRAN_VERBOSE_STR}")
+
+  # Detect gfortran version
+  string(REGEX MATCH "gcc version [^\t\n ]+" GFORTRAN_VER_STR "${GFORTRAN_VERBOSE_STR}")
+  string(REGEX REPLACE "gcc version ([^\t\n ]+)" "\\1" GFORTRAN_VERSION_STRING "${GFORTRAN_VER_STR}")
+  message(STATUS "Detected gfortran version ${GFORTRAN_VERSION_STRING}")
+  unset(GFORTRAN_VER_STR)
+
+  set(MATCH_REGEX "[^\t\n ]+[\t\n ]+")
+  set(REPLACE_REGEX "([^\t\n ]+)")
+
+  # Find architecture for compiler
+  string(REGEX MATCH "Target: [^\t\n ]+"
+    GFORTRAN_ARCH_STR "${GFORTRAN_VERBOSE_STR}")
+  message(STATUS "Architecture string: ${GFORTRAN_ARCH_STR}")
+  string(REGEX REPLACE "Target: ([^\t\n ]+)" "\\1"
+    GFORTRAN_ARCH "${GFORTRAN_ARCH_STR}")
+  message(STATUS "Detected gfortran architecture: ${GFORTRAN_ARCH}")
+  unset(GFORTRAN_ARCH_STR)
+
+  # Find install prefix, if it exists; if not, use default
+  string(REGEX MATCH  "--prefix=[^\t\n ]+[\t\n ]+"
+    GFORTRAN_PREFIX_STR "${GFORTRAN_VERBOSE_STR}")
+  if(NOT GFORTRAN_PREFIX_STR)
+    message(STATUS "Detected default gfortran prefix")
+    set(GFORTRAN_PREFIX_DIR "/usr/local") # default prefix for gcc install
+  else()
+    string(REGEX REPLACE "--prefix=([^\t\n ]+)" "\\1"
+      GFORTRAN_PREFIX_DIR "${GFORTRAN_PREFIX_STR}")
+  endif()
+  message(STATUS "Detected gfortran prefix: ${GFORTRAN_PREFIX_DIR}")
+  unset(GFORTRAN_PREFIX_STR)
+
+  # Find install exec-prefix, if it exists; if not, use default
+  string(REGEX MATCH "--exec-prefix=[^\t\n ]+[\t\n ]+" "\\1"
+    GFORTRAN_EXEC_PREFIX_STR "${GFORTRAN_VERBOSE_STR}")
+  if(NOT GFORTRAN_EXEC_PREFIX_STR)
+    message(STATUS "Detected default gfortran exec-prefix")
+    set(GFORTRAN_EXEC_PREFIX_DIR "${GFORTRAN_PREFIX_DIR}")
+  else()
+    string(REGEX REPLACE "--exec-prefix=([^\t\n ]+)" "\\1"
+      GFORTRAN_EXEC_PREFIX_DIR "${GFORTRAN_EXEC_PREFIX_STR}")
+  endif()
+  message(STATUS "Detected gfortran exec-prefix: ${GFORTRAN_EXEC_PREFIX_DIR}")
+  UNSET(GFORTRAN_EXEC_PREFIX_STR)
+
+  # Find library directory and include directory, if library directory specified
+  string(REGEX MATCH "--libdir=[^\t\n ]+"
+    GFORTRAN_LIB_DIR_STR "${GFORTRAN_VERBOSE_STR}")
+  if(NOT GFORTRAN_LIB_DIR_STR)
+    message(STATUS "Found --libdir flag -- not found")
+    message(STATUS "Using default gfortran library & include directory paths")
+    set(GFORTRAN_LIBRARIES_DIR
+      "${GFORTRAN_EXEC_PREFIX_DIR}/lib/gcc/${GFORTRAN_ARCH}/${GFORTRAN_VERSION_STRING}")
+    string(CONCAT GFORTRAN_INCLUDE_DIR "${GFORTRAN_LIBRARIES_DIR}" "/include")
+  else()
+    message(STATUS "Found --libdir flag -- yes")
+    string(REGEX REPLACE "--libdir=([^\t\n ]+)" "\\1"
+      GFORTRAN_LIBRARIES_DIR "${GFORTRAN_LIB_DIR_STR}")
+    string(CONCAT GFORTRAN_INCLUDE_DIR "${GFORTRAN_LIBRARIES_DIR}" "/gcc/" "${GFORTRAN_ARCH}" "/" "${GFORTRAN_VERSION_STRING}" "/include")
+  endif()
+  message(STATUS "gfortran libraries path: ${GFORTRAN_LIBRARIES_DIR}")
+  message(STATUS "gfortran include path dir: ${GFORTRAN_INCLUDE_DIR}")
+  unset(GFORTRAN_LIB_DIR_STR)
+
+  # There are lots of other build options for gcc & gfortran. For now, the
+  # options implemented above should cover a lot of common use cases.
+
+  # Clean up be deleting the output string from "gfortran -v"
+  unset(GFORTRAN_VERBOSE_STR)
+
+  # Find paths for libgfortran, libquadmath, libgomp
+  # libgomp needed for OpenMP support without Clang
+  find_library(LIBGFORTRAN_LIBRARIES NAMES gfortran libgfortran
+    HINTS ${GFORTRAN_LIBRARIES_DIR})
+  find_library(LIBQUADMATH_LIBRARIES NAMES quadmath libquadmath
+    HINTS ${GFORTRAN_LIBRARIES_DIR})
+  find_library(LIBGOMP_LIBRARIES NAMES gomp libgomp
+    HINTS ${GFORTRAN_LIBRARIES_DIR})
+
+  # Find OpenMP headers
+  find_path(LIBGOMP_INCLUDE_DIR NAMES omp.h HINTS ${GFORTRAN_INCLUDE_DIR})
+
+else()
+  message(STATUS "CMAKE_Fortran_COMPILER_ID does not match 'GNU'!")
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+# Required: libgfortran, libquadmath, path for gfortran libraries
+# Optional: libgomp, path for OpenMP headers, path for gcc/gfortran headers
+find_package_handle_standard_args(GFortranLibs
+  REQUIRED_VARS LIBGFORTRAN_LIBRARIES LIBQUADMATH_LIBRARIES GFORTRAN_LIBRARIES_DIR
+  VERSION_VAR GFORTRAN_VERSION_STRING)
+
+if(GFORTRANLIBS_FOUND)
+  message(STATUS "Looking for gfortran libraries -- found")
+  message(STATUS "gfortran version: ${GFORTRAN_VERSION_STRING}")
+else()
+  message(STATUS "Looking for gfortran libraries -- not found")
+endif()
+
+mark_as_advanced(LIBGFORTRAN_LIBRARIES LIBQUADMATH_LIBRARIES
+  LIBGOMP_LIBRARIES LIBGOMP_INCLUDE_DIR
+  GFORTRAN_LIBRARIES_DIR GFORTRAN_INCLUDE_DIR)
+# FindGFortranLIBS.cmake ends here
\ No newline at end of file
diff --git a/speechx/cmake/external/openblas.cmake b/speechx/cmake/external/openblas.cmake
index 3c202f7f..5c196527 100644
--- a/speechx/cmake/external/openblas.cmake
+++ b/speechx/cmake/external/openblas.cmake
@@ -7,6 +7,27 @@ set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix)
 # OPENBLAS  https://github.com/lattice/quda/blob/develop/CMakeLists.txt#L575
 # ######################################################################################################################
 enable_language(Fortran)
+
+include(FortranCInterface)
+
+# # Clang doesn't have a Fortran compiler in its suite (yet),
+# # so detect libraries for gfortran; we need equivalents to
+# # libgfortran and libquadmath, which are implicitly
+# # linked by flags in CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES
+# include(FindGFortranLibs REQUIRED)
+# # Add directory containing libgfortran and libquadmath to
+# # linker. Should also contain libgomp, if not using
+# # Intel OpenMP runtime
+# link_directories(${GFORTRAN_LIBRARIES_DIR})
+# # gfortan dir in the docker.
+# link_directories(/usr/local/gcc-8.2/lib64)
+# # if you are working with C and Fortran
+# FortranCInterface_VERIFY()
+
+# # if you are working with C++ and Fortran
+# FortranCInterface_VERIFY(CXX)
+
+
 #TODO: switch to CPM
 include(GNUInstallDirs)
 ExternalProject_Add(
diff --git a/speechx/cmake/external/openfst.cmake b/speechx/cmake/external/openfst.cmake
index 07abb18e..dc9cdff6 100644
--- a/speechx/cmake/external/openfst.cmake
+++ b/speechx/cmake/external/openfst.cmake
@@ -1,13 +1,14 @@
 include(FetchContent)
+set(openfst_PREFIX_DIR ${fc_patch}/openfst)
 set(openfst_SOURCE_DIR ${fc_patch}/openfst-src)
 set(openfst_BINARY_DIR ${fc_patch}/openfst-build)
 
 ExternalProject_Add(openfst
   URL               https://github.com/mjansche/openfst/archive/refs/tags/1.7.2.zip
   URL_HASH          SHA256=ffc56931025579a8af3515741c0f3b0fc3a854c023421472c07ca0c6389c75e6
-#   #PREFIX            ${openfst_PREFIX_DIR} 
-#   SOURCE_DIR        ${openfst_SOURCE_DIR}
-#   BINARY_DIR        ${openfst_BINARY_DIR}
+  PREFIX            ${openfst_PREFIX_DIR} 
+  SOURCE_DIR        ${openfst_SOURCE_DIR}
+  BINARY_DIR        ${openfst_BINARY_DIR}
   CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure --prefix=${openfst_PREFIX_DIR}
                       "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR}"
                       "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}"
@@ -16,4 +17,4 @@ ExternalProject_Add(openfst
   BUILD_COMMAND     make -j 4
 )
 link_directories(${openfst_PREFIX_DIR}/lib)
-include_directories(${openfst_PREFIX_DIR}/include)
+include_directories(${openfst_PREFIX_DIR}/include)
\ No newline at end of file
diff --git a/speechx/speechx/frontend/feature_cache.cc b/speechx/speechx/frontend/feature_cache.cc
index d23b3a8b..dad6907c 100644
--- a/speechx/speechx/frontend/feature_cache.cc
+++ b/speechx/speechx/frontend/feature_cache.cc
@@ -41,6 +41,7 @@ void FeatureCache::Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs) {
 // pop feature chunk
 bool FeatureCache::Read(kaldi::Vector<kaldi::BaseFloat>* feats) {
     kaldi::Timer timer;
+
     std::unique_lock<std::mutex> lock(mutex_);
     while (cache_.empty() && base_extractor_->IsFinished() == false) {
         ready_read_condition_.wait(lock);
@@ -64,10 +65,13 @@ bool FeatureCache::Compute() {
     // compute and feed
     Vector<BaseFloat> feature_chunk;
     bool result = base_extractor_->Read(&feature_chunk);
+
     std::unique_lock<std::mutex> lock(mutex_);
     while (cache_.size() >= max_size_) {
         ready_feed_condition_.wait(lock);
     }
+
+    // feed cache
     if (feature_chunk.Dim() != 0) {
         cache_.push(feature_chunk);
     }
diff --git a/speechx/speechx/frontend/feature_cache.h b/speechx/speechx/frontend/feature_cache.h
index e52d8b29..b6bbdf3c 100644
--- a/speechx/speechx/frontend/feature_cache.h
+++ b/speechx/speechx/frontend/feature_cache.h
@@ -24,17 +24,24 @@ class FeatureCache : public FeatureExtractorInterface {
     explicit FeatureCache(
         int32 max_size = kint16max,
         std::unique_ptr<FeatureExtractorInterface> base_extractor = NULL);
+
+    // Feed feats or waves
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs);
+
     // feats dim = num_frames * feature_dim
     virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* feats);
+
     // feature cache only cache feature which from base extractor
     virtual size_t Dim() const { return base_extractor_->Dim(); }
+
     virtual void SetFinished() {
         base_extractor_->SetFinished();
         // read the last chunk data
         Compute();
     }
+
     virtual bool IsFinished() const { return base_extractor_->IsFinished(); }
+
     virtual void Reset() {
         base_extractor_->Reset();
         while (!cache_.empty()) {
@@ -45,12 +52,14 @@ class FeatureCache : public FeatureExtractorInterface {
   private:
     bool Compute();
 
-    std::mutex mutex_;
     size_t max_size_;
-    std::queue<kaldi::Vector<BaseFloat>> cache_;
     std::unique_ptr<FeatureExtractorInterface> base_extractor_;
+
+    std::mutex mutex_;
+    std::queue<kaldi::Vector<BaseFloat>> cache_;
     std::condition_variable ready_feed_condition_;
     std::condition_variable ready_read_condition_;
+
     // DISALLOW_COPY_AND_ASSGIN(FeatureCache);
 };
 
diff --git a/speechx/speechx/frontend/feature_extractor_interface.h b/speechx/speechx/frontend/feature_extractor_interface.h
index 3668fbda..5da2526b 100644
--- a/speechx/speechx/frontend/feature_extractor_interface.h
+++ b/speechx/speechx/frontend/feature_extractor_interface.h
@@ -21,17 +21,26 @@ namespace ppspeech {
 
 class FeatureExtractorInterface {
   public:
-    // accept input data, accept feature or raw waves which decided
-    // by the base_extractor
+    // Feed inputs: features(2D saved in 1D) or waveforms(1D).
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs) = 0;
-    // get the processed result
-    // the length of output = feature_row * feature_dim,
-    // the Matrix is squashed into Vector
+
+    // Fetch processed data: features or waveforms.
+    // For features(2D saved in 1D), the Matrix is squashed into Vector,
+    //    the length of output = feature_row * feature_dim.
+    // For waveforms(1D), samples saved in vector.
     virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* outputs) = 0;
-    // the Dim is the feature dim
+
+    // Dim is the feature dim. For waveforms(1D), Dim is zero; else is specific,
+    // e.g 80 for fbank.
     virtual size_t Dim() const = 0;
+
+    // End Flag for Streaming Data.
     virtual void SetFinished() = 0;
+
+    // whether is end of Streaming Data.
     virtual bool IsFinished() const = 0;
+
+    // Reset to start state.
     virtual void Reset() = 0;
 };
 
diff --git a/speechx/speechx/frontend/linear_spectrogram.h b/speechx/speechx/frontend/linear_spectrogram.h
index ffdfbbe9..10853904 100644
--- a/speechx/speechx/frontend/linear_spectrogram.h
+++ b/speechx/speechx/frontend/linear_spectrogram.h
@@ -23,12 +23,14 @@ namespace ppspeech {
 
 struct LinearSpectrogramOptions {
     kaldi::FrameExtractionOptions frame_opts;
-    kaldi::BaseFloat streaming_chunk;
+    kaldi::BaseFloat streaming_chunk;  // second
+
     LinearSpectrogramOptions() : streaming_chunk(0.36), frame_opts() {}
 
     void Register(kaldi::OptionsItf* opts) {
-        opts->Register(
-            "streaming-chunk", &streaming_chunk, "streaming chunk size");
+        opts->Register("streaming-chunk",
+                       &streaming_chunk,
+                       "streaming chunk size, default: 0.36 sec");
         frame_opts.Register(opts);
     }
 };

From 84d712d4935eb14695bebee19360fd30644deedf Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Mon, 28 Mar 2022 12:16:06 +0000
Subject: [PATCH 089/126] format code, test=doc

---
 dataset/librispeech/librispeech.py           | 2 +-
 demos/audio_searching/src/encode.py          | 2 +-
 demos/audio_searching/src/operations/load.py | 5 ++---
 examples/ami/sd0/local/ami_prepare.py        | 2 +-
 paddlespeech/s2t/decoders/recog_bin.py       | 2 +-
 paddlespeech/s2t/utils/cli_utils.py          | 2 +-
 paddlespeech/s2t/utils/utility.py            | 2 +-
 paddlespeech/vector/cluster/diarization.py   | 2 +-
 utils/DER.py                                 | 2 +-
 utils/addjson.py                             | 2 +-
 utils/apply-cmvn.py                          | 2 +-
 utils/copy-feats.py                          | 1 +
 utils/merge_scp2json.py                      | 3 ++-
 13 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/dataset/librispeech/librispeech.py b/dataset/librispeech/librispeech.py
index 69f0db59..65cab249 100644
--- a/dataset/librispeech/librispeech.py
+++ b/dataset/librispeech/librispeech.py
@@ -20,12 +20,12 @@ of each audio file in the data set.
 """
 import argparse
 import codecs
-import distutils.util
 import io
 import json
 import os
 from multiprocessing.pool import Pool
 
+import distutils.util
 import soundfile
 
 from utils.utility import download
diff --git a/demos/audio_searching/src/encode.py b/demos/audio_searching/src/encode.py
index 83b9e3df..cf5f29a4 100644
--- a/demos/audio_searching/src/encode.py
+++ b/demos/audio_searching/src/encode.py
@@ -16,8 +16,8 @@ import os
 import librosa
 import numpy as np
 from config import DEFAULT_TABLE
-
 from logs import LOGGER
+
 from paddlespeech.cli import VectorExecutor
 
 vector_executor = VectorExecutor()
diff --git a/demos/audio_searching/src/operations/load.py b/demos/audio_searching/src/operations/load.py
index 7a295bf3..80b6375f 100644
--- a/demos/audio_searching/src/operations/load.py
+++ b/demos/audio_searching/src/operations/load.py
@@ -26,9 +26,8 @@ def get_audios(path):
     """
     supported_formats = [".wav", ".mp3", ".ogg", ".flac", ".m4a"]
     return [
-        item
-        for sublist in [[os.path.join(dir, file) for file in files]
-                        for dir, _, files in list(os.walk(path))]
+        item for sublist in [[os.path.join(dir, file) for file in files]
+                             for dir, _, files in list(os.walk(path))]
         for item in sublist if os.path.splitext(item)[1] in supported_formats
     ]
 
diff --git a/examples/ami/sd0/local/ami_prepare.py b/examples/ami/sd0/local/ami_prepare.py
index 569c3a60..1f02afe0 100644
--- a/examples/ami/sd0/local/ami_prepare.py
+++ b/examples/ami/sd0/local/ami_prepare.py
@@ -24,11 +24,11 @@ import json
 import logging
 import os
 import xml.etree.ElementTree as et
-from distutils.util import strtobool
 
 from ami_splits import get_AMI_split
 from dataio import load_pkl
 from dataio import save_pkl
+from distutils.util import strtobool
 
 logger = logging.getLogger(__name__)
 SAMPLERATE = 16000
diff --git a/paddlespeech/s2t/decoders/recog_bin.py b/paddlespeech/s2t/decoders/recog_bin.py
index cd7a360a..0470a034 100644
--- a/paddlespeech/s2t/decoders/recog_bin.py
+++ b/paddlespeech/s2t/decoders/recog_bin.py
@@ -17,10 +17,10 @@ import logging
 import os
 import random
 import sys
-from distutils.util import strtobool
 
 import configargparse
 import numpy as np
+from distutils.util import strtobool
 
 
 def get_parser():
diff --git a/paddlespeech/s2t/utils/cli_utils.py b/paddlespeech/s2t/utils/cli_utils.py
index 4aee3f43..ccb0d3c9 100644
--- a/paddlespeech/s2t/utils/cli_utils.py
+++ b/paddlespeech/s2t/utils/cli_utils.py
@@ -14,9 +14,9 @@
 # Modified from espnet(https://github.com/espnet/espnet)
 import sys
 from collections.abc import Sequence
-from distutils.util import strtobool as dist_strtobool
 
 import numpy
+from distutils.util import strtobool as dist_strtobool
 
 
 def strtobool(x):
diff --git a/paddlespeech/s2t/utils/utility.py b/paddlespeech/s2t/utils/utility.py
index dc1be815..fdd8c029 100644
--- a/paddlespeech/s2t/utils/utility.py
+++ b/paddlespeech/s2t/utils/utility.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Contains common utility functions."""
-import distutils.util
 import math
 import os
 import random
@@ -21,6 +20,7 @@ from contextlib import contextmanager
 from pprint import pformat
 from typing import List
 
+import distutils.util
 import numpy as np
 import paddle
 import soundfile
diff --git a/paddlespeech/vector/cluster/diarization.py b/paddlespeech/vector/cluster/diarization.py
index 99ac41cd..597aa480 100644
--- a/paddlespeech/vector/cluster/diarization.py
+++ b/paddlespeech/vector/cluster/diarization.py
@@ -18,11 +18,11 @@ A few sklearn functions are modified in this script as per requirement.
 """
 import argparse
 import warnings
-from distutils.util import strtobool
 
 import numpy as np
 import scipy
 import sklearn
+from distutils.util import strtobool
 from scipy import sparse
 from scipy.sparse.csgraph import connected_components
 from scipy.sparse.csgraph import laplacian as csgraph_laplacian
diff --git a/utils/DER.py b/utils/DER.py
index d6ab695d..59bcbec4 100755
--- a/utils/DER.py
+++ b/utils/DER.py
@@ -26,9 +26,9 @@ import argparse
 import os
 import re
 import subprocess
-from distutils.util import strtobool
 
 import numpy as np
+from distutils.util import strtobool
 
 FILE_IDS = re.compile(r"(?<=Speaker Diarization for).+(?=\*\*\*)")
 SCORED_SPEAKER_TIME = re.compile(r"(?<=SCORED SPEAKER TIME =)[\d.]+")
diff --git a/utils/addjson.py b/utils/addjson.py
index 013d1472..e1be7ab3 100755
--- a/utils/addjson.py
+++ b/utils/addjson.py
@@ -10,8 +10,8 @@ import codecs
 import json
 import logging
 import sys
-from distutils.util import strtobool
 
+from distutils.util import strtobool
 from espnet.utils.cli_utils import get_commandline_args
 
 is_python2 = sys.version_info[0] == 2
diff --git a/utils/apply-cmvn.py b/utils/apply-cmvn.py
index b92e58f4..cf91bdfc 100755
--- a/utils/apply-cmvn.py
+++ b/utils/apply-cmvn.py
@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 import argparse
 import logging
-from distutils.util import strtobool
 
 import kaldiio
 import numpy
+from distutils.util import strtobool
 
 from paddlespeech.s2t.transform.cmvn import CMVN
 from paddlespeech.s2t.utils.cli_readers import file_reader_helper
diff --git a/utils/copy-feats.py b/utils/copy-feats.py
index 2e120881..dc7a70b4 100755
--- a/utils/copy-feats.py
+++ b/utils/copy-feats.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import argparse
 import logging
+
 from distutils.util import strtobool
 
 from paddlespeech.s2t.transform.transformation import Transformation
diff --git a/utils/merge_scp2json.py b/utils/merge_scp2json.py
index 650e4669..99db6bac 100755
--- a/utils/merge_scp2json.py
+++ b/utils/merge_scp2json.py
@@ -5,9 +5,10 @@ import codecs
 import json
 import logging
 import sys
-from distutils.util import strtobool
 from io import open
 
+from distutils.util import strtobool
+
 from paddlespeech.s2t.utils.cli_utils import get_commandline_args
 
 PY2 = sys.version_info[0] == 2

From ed490b66cb052c1308117e5e9703d94d8e43239a Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Tue, 29 Mar 2022 03:20:07 +0000
Subject: [PATCH 090/126] update spectrogram, test=asr

---
 examples/aishell/asr1/conf/preprocess.yaml |  5 ++--
 paddlespeech/s2t/transform/spectrogram.py  | 34 ++++++++++++++++------
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/examples/aishell/asr1/conf/preprocess.yaml b/examples/aishell/asr1/conf/preprocess.yaml
index a20ff2ab..d3992cb9 100644
--- a/examples/aishell/asr1/conf/preprocess.yaml
+++ b/examples/aishell/asr1/conf/preprocess.yaml
@@ -3,9 +3,8 @@ process:
   - type: fbank_kaldi
     fs: 16000
     n_mels: 80
-    n_frame_length: 25
-    n_frame_shift: 10
-    energy_floor: 0.0
+    n_shift: 160
+    win_length: 400
     dither: 0.1
   - type: cmvn_json
     cmvn_path: data/mean_std.json
diff --git a/paddlespeech/s2t/transform/spectrogram.py b/paddlespeech/s2t/transform/spectrogram.py
index f779b07d..75787d92 100644
--- a/paddlespeech/s2t/transform/spectrogram.py
+++ b/paddlespeech/s2t/transform/spectrogram.py
@@ -312,17 +312,33 @@ class IStft():
 
 
 class LogMelSpectrogramKaldi():
-    def __init__(self,
-                 fs=16000,
-                 n_mels=80,
-                 n_frame_length=25,
-                 n_frame_shift=10,
-                 energy_floor=0.0,
-                 dither=0.1):
+    def __init__(
+            self,
+            fs=16000,
+            n_mels=80,
+            n_shift=160,  # unit:sample, 10ms
+            win_length=400,  # unit:sample, 25ms
+            energy_floor=0.0,
+            dither=0.1):
+        """
+        The Kaldi implementation of LogMelSpectrogram 
+        Args:
+            fs (int): sample rate of the audio
+            n_mels (int): number of mel filter banks
+            n_shift (int): number of points in a frame shift
+            win_length (int): number of points in a frame windows
+            energy_floor (float): Floor on energy in Spectrogram computation (absolute)
+            dither (float): Dithering constant
+
+        Returns:
+            LogMelSpectrogramKaldi
+        """
+
         self.fs = fs
         self.n_mels = n_mels
-        self.n_frame_length = n_frame_length
-        self.n_frame_shift = n_frame_shift
+        num_point_ms = fs / 1000
+        self.n_frame_length = win_length / num_point_ms
+        self.n_frame_shift = n_shift / num_point_ms
         self.energy_floor = energy_floor
         self.dither = dither
 

From 6c163b6323ccfeac02ed9f609e5e24f770381223 Mon Sep 17 00:00:00 2001
From: Yang Zhou <goat.zhou@qq.com>
Date: Tue, 29 Mar 2022 16:06:28 +0800
Subject: [PATCH 091/126] fix path error in speechx/examples/decoder

---
 speechx/examples/decoder/path.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/speechx/examples/decoder/path.sh b/speechx/examples/decoder/path.sh
index 7b4b7545..a0e7c9ae 100644
--- a/speechx/examples/decoder/path.sh
+++ b/speechx/examples/decoder/path.sh
@@ -10,5 +10,5 @@ TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin
 
 export LC_AL=C
 
-SPEECHX_BIN=$SPEECHX_EXAMPLES/decoder
+SPEECHX_BIN=$SPEECHX_EXAMPLES/decoder:$SPEECHX_EXAMPLES/feat
 export PATH=$PATH:$SPEECHX_BIN:$TOOLS_BIN

From 284a5f26c20aa21401f0720ac594d7c5800ef629 Mon Sep 17 00:00:00 2001
From: Yang Zhou <goat.zhou@qq.com>
Date: Tue, 29 Mar 2022 17:17:32 +0800
Subject: [PATCH 092/126] fix speechx doc typo

---
 speechx/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/speechx/README.md b/speechx/README.md
index 07449f9f..bb256712 100644
--- a/speechx/README.md
+++ b/speechx/README.md
@@ -5,7 +5,7 @@
 We develop under:
 * docker - registry.baidubce.com/paddlepaddle/paddle:2.1.1-gpu-cuda10.2-cudnn7
 * os - Ubuntu 16.04.7 LTS
-* ** gcc/g++/gfortran - 8.2.0 **
+* gcc/g++/gfortran - 8.2.0
 * cmake - 3.16.0
 
 > We make sure all things work fun under docker, and recommend using it to develop and deploy.
@@ -24,7 +24,7 @@ nvidia-docker run --privileged  --net=host --ipc=host -it --rm -v $PWD:/workspac
 
 * More `Paddle` docker images you can see [here](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/docker/linux-docker.html).
 
-* If you want only work under cpu, please download corresponded [image](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/docker/linux-docker.html), and using `docker` instead `nviida-docker`.
+* If you want only work under cpu, please download corresponded [image](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/docker/linux-docker.html), and using `docker` instead `nvida-docker`.
 
 
 2. Build `speechx` and `examples`.

From 678c8369aa6e01063496d5a9992e893ad444012b Mon Sep 17 00:00:00 2001
From: Yang Zhou <goat.zhou@qq.com>
Date: Tue, 29 Mar 2022 17:38:50 +0800
Subject: [PATCH 093/126] fix typo

---
 speechx/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/speechx/README.md b/speechx/README.md
index bb256712..610b88a8 100644
--- a/speechx/README.md
+++ b/speechx/README.md
@@ -24,7 +24,7 @@ nvidia-docker run --privileged  --net=host --ipc=host -it --rm -v $PWD:/workspac
 
 * More `Paddle` docker images you can see [here](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/docker/linux-docker.html).
 
-* If you want only work under cpu, please download corresponded [image](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/docker/linux-docker.html), and using `docker` instead `nvida-docker`.
+* If you want only work under cpu, please download corresponded [image](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/install/docker/linux-docker.html), and using `docker` instead `nvidia-docker`.
 
 
 2. Build `speechx` and `examples`.

From f47146af494f510428e9d14702f4b735c88843aa Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Tue, 29 Mar 2022 12:09:54 +0000
Subject: [PATCH 094/126] add docstring, test=asr

---
 paddlespeech/s2t/transform/spectrogram.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/paddlespeech/s2t/transform/spectrogram.py b/paddlespeech/s2t/transform/spectrogram.py
index 75787d92..4a65548f 100644
--- a/paddlespeech/s2t/transform/spectrogram.py
+++ b/paddlespeech/s2t/transform/spectrogram.py
@@ -355,7 +355,20 @@ class LogMelSpectrogramKaldi():
                 dither=self.dither, ))
 
     def __call__(self, x, train):
+        """
+        Args:
+            x (np.ndarray): shape (Ti,)
+            train (bool): True, train mode.
+
+        Raises:
+            ValueError: not support (Ti, C)
+
+        Returns:
+            np.ndarray: (T, D)
+        """
         dither = self.dither if train else 0.0
+        if x.ndim != 1:
+            raise ValueError("Not support x: [Time, Channel]")
         waveform = paddle.to_tensor(np.expand_dims(x, 0), dtype=paddle.float32)
         mat = kaldi.fbank(
             waveform,

From fa160d86e326e7f2a8260f07ccbaa68c28c9defc Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Wed, 30 Mar 2022 07:57:52 +0000
Subject: [PATCH 095/126] update version of ctcdecoders, test=asr

---
 third_party/ctc_decoders/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/ctc_decoders/setup.py b/third_party/ctc_decoders/setup.py
index 4a11b890..ce2787e3 100644
--- a/third_party/ctc_decoders/setup.py
+++ b/third_party/ctc_decoders/setup.py
@@ -127,7 +127,7 @@ decoders_module = [
 
 setup(
     name='paddlespeech_ctcdecoders',
-    version='0.1.1',
+    version='0.2.0',
     description="CTC decoders in paddlespeech",
     author="PaddlePaddle Speech and Language Team",
     author_email="paddlesl@baidu.com",

From d847fe29cfc83afbe5e4fc6f3717240516600eb7 Mon Sep 17 00:00:00 2001
From: WilliamZhang06 <xd_0614@163.com>
Date: Wed, 30 Mar 2022 16:44:04 +0800
Subject: [PATCH 096/126] added online asr engine , test=doc

---
 paddlespeech/s2t/frontend/audio.py            |  12 +
 paddlespeech/s2t/frontend/speech.py           |  16 +
 paddlespeech/server/bin/main.py               |  10 +-
 paddlespeech/server/conf/application.yaml     |  27 +-
 .../server/engine/asr/online/__init__.py      |  13 +
 .../server/engine/asr/online/asr_engine.py    | 355 ++++++++++++++++++
 paddlespeech/server/engine/engine_factory.py  |   3 +
 .../tests/asr/online/microphone_client.py     | 154 ++++++++
 .../tests/asr/online/websocket_client.py      | 115 ++++++
 paddlespeech/server/utils/buffer.py           |  59 +++
 paddlespeech/server/utils/vad.py              |  79 ++++
 paddlespeech/server/ws/__init__.py            |  13 +
 paddlespeech/server/ws/api.py                 |  38 ++
 paddlespeech/server/ws/asr_socket.py          | 106 ++++++
 14 files changed, 996 insertions(+), 4 deletions(-)
 create mode 100644 paddlespeech/server/engine/asr/online/__init__.py
 create mode 100644 paddlespeech/server/engine/asr/online/asr_engine.py
 create mode 100644 paddlespeech/server/tests/asr/online/microphone_client.py
 create mode 100644 paddlespeech/server/tests/asr/online/websocket_client.py
 create mode 100644 paddlespeech/server/utils/buffer.py
 create mode 100644 paddlespeech/server/utils/vad.py
 create mode 100644 paddlespeech/server/ws/__init__.py
 create mode 100644 paddlespeech/server/ws/api.py
 create mode 100644 paddlespeech/server/ws/asr_socket.py

diff --git a/paddlespeech/s2t/frontend/audio.py b/paddlespeech/s2t/frontend/audio.py
index d0368cc8..7f71e5dd 100644
--- a/paddlespeech/s2t/frontend/audio.py
+++ b/paddlespeech/s2t/frontend/audio.py
@@ -208,6 +208,18 @@ class AudioSegment():
             io.BytesIO(bytes), dtype='float32')
         return cls(samples, sample_rate)
 
+    @classmethod
+    def from_pcm(cls, samples, sample_rate):
+        """Create audio segment from a byte string containing audio samples.
+        :param samples: Audio samples [num_samples x num_channels].
+        :type samples: numpy.ndarray
+        :param sample_rate: Audio sample rate.
+        :type sample_rate: int
+        :return: Audio segment instance.
+        :rtype: AudioSegment
+        """
+        return cls(samples, sample_rate)
+
     @classmethod
     def concatenate(cls, *segments):
         """Concatenate an arbitrary number of audio segments together.
diff --git a/paddlespeech/s2t/frontend/speech.py b/paddlespeech/s2t/frontend/speech.py
index 8fd661c9..0340831a 100644
--- a/paddlespeech/s2t/frontend/speech.py
+++ b/paddlespeech/s2t/frontend/speech.py
@@ -107,6 +107,22 @@ class SpeechSegment(AudioSegment):
         return cls(audio.samples, audio.sample_rate, transcript, tokens,
                    token_ids)
 
+    @classmethod
+    def from_pcm(cls, samples, sample_rate, transcript, tokens=None, token_ids=None):
+        """Create speech segment from pcm on online mode 
+        Args:
+            samples (numpy.ndarray): Audio samples [num_samples x num_channels].
+            sample_rate (int): Audio sample rate.
+            transcript (str): Transcript text for the speech.
+            tokens (List[str], optional): text tokens. Defaults to None.
+            token_ids (List[int], optional): text token ids. Defaults to None.
+        Returns: 
+            SpeechSegment: Speech segment instance.
+        """
+        audio = AudioSegment.from_pcm(samples, sample_rate)
+        return cls(audio.samples, audio.sample_rate, transcript, tokens,
+                   token_ids)
+
     @classmethod
     def concatenate(cls, *segments):
         """Concatenate an arbitrary number of speech segments together, both
diff --git a/paddlespeech/server/bin/main.py b/paddlespeech/server/bin/main.py
index de528299..45ded33d 100644
--- a/paddlespeech/server/bin/main.py
+++ b/paddlespeech/server/bin/main.py
@@ -17,7 +17,8 @@ import uvicorn
 from fastapi import FastAPI
 
 from paddlespeech.server.engine.engine_pool import init_engine_pool
-from paddlespeech.server.restful.api import setup_router
+from paddlespeech.server.restful.api import setup_router as setup_http_router
+from paddlespeech.server.ws.api import setup_router as setup_ws_router
 from paddlespeech.server.utils.config import get_config
 
 app = FastAPI(
@@ -35,7 +36,12 @@ def init(config):
     """
     # init api
     api_list = list(engine.split("_")[0] for engine in config.engine_list)
-    api_router = setup_router(api_list)
+    if config.protocol == "websocket":
+        api_router = setup_ws_router(api_list)
+    elif config.protocol == "http":
+        api_router = setup_http_router(api_list)
+    else:
+        raise Exception("unsupported protocol")
     app.include_router(api_router)
 
     if not init_engine_pool(config):
diff --git a/paddlespeech/server/conf/application.yaml b/paddlespeech/server/conf/application.yaml
index 2b1a0599..40de8e3b 100644
--- a/paddlespeech/server/conf/application.yaml
+++ b/paddlespeech/server/conf/application.yaml
@@ -3,13 +3,18 @@
 #################################################################################
 #                             SERVER SETTING                                    #
 #################################################################################
-host: 127.0.0.1
+host: 0.0.0.0
 port: 8090
 
 # The task format in the engin_list is: <speech task>_<engine type>
 # task choices = ['asr_python', 'asr_inference', 'tts_python', 'tts_inference']
+# protocol: 'http'
+# engine_list: ['asr_python', 'tts_python', 'cls_python']
 
-engine_list: ['asr_python', 'tts_python', 'cls_python']
+
+# websocket, http (only choose one). websocket only support online engine type.
+protocol: 'websocket'
+engine_list: ['asr_online']
 
 
 #################################################################################
@@ -48,6 +53,24 @@ asr_inference:
         summary: True  # False -> do not show predictor config
 
 
+################### speech task: asr; engine_type: online #######################
+asr_online:
+    model_type: 'deepspeech2online_aishell'
+    am_model: # the pdmodel file of am static model [optional]
+    am_params:  # the pdiparams file of am static model [optional]
+    lang: 'zh'
+    sample_rate: 16000
+    cfg_path: 
+    decode_method: 
+    force_yes: True
+
+    am_predictor_conf:
+        device:  # set 'gpu:id' or 'cpu'
+        switch_ir_optim: True
+        glog_info: False  # True -> print glog
+        summary: True  # False -> do not show predictor config
+
+
 ################################### TTS #########################################
 ################### speech task: tts; engine_type: python #######################
 tts_python: 
diff --git a/paddlespeech/server/engine/asr/online/__init__.py b/paddlespeech/server/engine/asr/online/__init__.py
new file mode 100644
index 00000000..97043fd7
--- /dev/null
+++ b/paddlespeech/server/engine/asr/online/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/paddlespeech/server/engine/asr/online/asr_engine.py b/paddlespeech/server/engine/asr/online/asr_engine.py
new file mode 100644
index 00000000..d5c1aa7b
--- /dev/null
+++ b/paddlespeech/server/engine/asr/online/asr_engine.py
@@ -0,0 +1,355 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import io
+import os
+import time
+from typing import Optional
+import pickle
+import numpy as np
+from numpy import float32
+import soundfile
+
+import paddle
+from yacs.config import CfgNode
+
+from paddlespeech.s2t.frontend.speech import SpeechSegment
+from paddlespeech.cli.asr.infer import ASRExecutor
+from paddlespeech.cli.log import logger
+from paddlespeech.cli.utils import MODEL_HOME
+from paddlespeech.s2t.frontend.featurizer.text_featurizer import TextFeaturizer
+from paddlespeech.s2t.modules.ctc import CTCDecoder
+from paddlespeech.s2t.utils.utility import UpdateConfig
+from paddlespeech.server.engine.base_engine import BaseEngine
+from paddlespeech.server.utils.config import get_config
+from paddlespeech.server.utils.paddle_predictor import init_predictor
+from paddlespeech.server.utils.paddle_predictor import run_model
+
+__all__ = ['ASREngine']
+
+pretrained_models = {
+    "deepspeech2online_aishell-zh-16k": {
+        'url':
+        'https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_online_aishell_ckpt_0.1.1.model.tar.gz',
+        'md5':
+        'd5e076217cf60486519f72c217d21b9b',
+        'cfg_path':
+        'model.yaml',
+        'ckpt_path':
+        'exp/deepspeech2_online/checkpoints/avg_1',
+        'model':
+        'exp/deepspeech2_online/checkpoints/avg_1.jit.pdmodel',
+        'params':
+        'exp/deepspeech2_online/checkpoints/avg_1.jit.pdiparams',
+        'lm_url':
+        'https://deepspeech.bj.bcebos.com/zh_lm/zh_giga.no_cna_cmn.prune01244.klm',
+        'lm_md5':
+        '29e02312deb2e59b3c8686c7966d4fe3'
+    },
+}
+
+
+class ASRServerExecutor(ASRExecutor):
+    def __init__(self):
+        super().__init__()
+        pass
+
+    def _init_from_path(self,
+                        model_type: str='wenetspeech',
+                        am_model: Optional[os.PathLike]=None,
+                        am_params: Optional[os.PathLike]=None,
+                        lang: str='zh',
+                        sample_rate: int=16000,
+                        cfg_path: Optional[os.PathLike]=None,
+                        decode_method: str='attention_rescoring',
+                        am_predictor_conf: dict=None):
+        """
+        Init model and other resources from a specific path.
+        """
+
+        if cfg_path is None or am_model is None or am_params is None:
+            sample_rate_str = '16k' if sample_rate == 16000 else '8k'
+            tag = model_type + '-' + lang + '-' + sample_rate_str
+            res_path = self._get_pretrained_path(tag)  # wenetspeech_zh
+            self.res_path = res_path
+            self.cfg_path = os.path.join(res_path,
+                                         pretrained_models[tag]['cfg_path'])
+
+            self.am_model = os.path.join(res_path,
+                                         pretrained_models[tag]['model'])
+            self.am_params = os.path.join(res_path,
+                                          pretrained_models[tag]['params'])
+            logger.info(res_path)
+            logger.info(self.cfg_path)
+            logger.info(self.am_model)
+            logger.info(self.am_params)
+        else:
+            self.cfg_path = os.path.abspath(cfg_path)
+            self.am_model = os.path.abspath(am_model)
+            self.am_params = os.path.abspath(am_params)
+            self.res_path = os.path.dirname(
+                os.path.dirname(os.path.abspath(self.cfg_path)))
+
+        #Init body.
+        self.config = CfgNode(new_allowed=True)
+        self.config.merge_from_file(self.cfg_path)
+
+        with UpdateConfig(self.config):
+            if "deepspeech2online" in model_type or "deepspeech2offline" in model_type:
+                from paddlespeech.s2t.io.collator import SpeechCollator
+                self.vocab = self.config.vocab_filepath
+                self.config.decode.lang_model_path = os.path.join(
+                    MODEL_HOME, 'language_model',
+                    self.config.decode.lang_model_path)
+                self.collate_fn_test = SpeechCollator.from_config(self.config)
+                self.text_feature = TextFeaturizer(
+                    unit_type=self.config.unit_type, vocab=self.vocab)
+
+                lm_url = pretrained_models[tag]['lm_url']
+                lm_md5 = pretrained_models[tag]['lm_md5']
+                self.download_lm(
+                    lm_url,
+                    os.path.dirname(self.config.decode.lang_model_path), lm_md5)
+            elif "conformer" in model_type or "transformer" in model_type or "wenetspeech" in model_type:
+                raise Exception("wrong type")
+            else:
+                raise Exception("wrong type")
+
+        # AM predictor
+        self.am_predictor_conf = am_predictor_conf
+        self.am_predictor = init_predictor(
+            model_file=self.am_model,
+            params_file=self.am_params,
+            predictor_conf=self.am_predictor_conf)
+
+        # decoder
+        self.decoder = CTCDecoder(
+            odim=self.config.output_dim,  # <blank> is in  vocab
+            enc_n_units=self.config.rnn_layer_size * 2,
+            blank_id=self.config.blank_id,
+            dropout_rate=0.0,
+            reduction=True,  # sum
+            batch_average=True,  # sum / batch_size
+            grad_norm_type=self.config.get('ctc_grad_norm_type', None))
+        
+        # init decoder
+        cfg = self.config.decode
+        decode_batch_size = 1      # for online
+        self.decoder.init_decoder(
+            decode_batch_size, self.text_feature.vocab_list,
+            cfg.decoding_method, cfg.lang_model_path, cfg.alpha, cfg.beta,
+            cfg.beam_size, cfg.cutoff_prob, cfg.cutoff_top_n,
+            cfg.num_proc_bsearch)
+
+        # init state box
+        self.chunk_state_h_box = np.zeros(
+            (self.config.num_rnn_layers, 1, self.config.rnn_layer_size),
+            dtype=float32)
+        self.chunk_state_c_box = np.zeros(
+            (self.config.num_rnn_layers, 1, self.config.rnn_layer_size),
+            dtype=float32)
+
+    def reset_decoder_and_chunk(self):
+        """reset decoder and chunk state for an new audio
+        """
+        self.decoder.reset_decoder(batch_size=1)
+        # init state box, for new audio request
+        self.chunk_state_h_box = np.zeros(
+            (self.config.num_rnn_layers, 1, self.config.rnn_layer_size),
+            dtype=float32)
+        self.chunk_state_c_box = np.zeros(
+            (self.config.num_rnn_layers, 1, self.config.rnn_layer_size),
+            dtype=float32)
+
+    def decode_one_chunk(self, x_chunk, x_chunk_lens, model_type: str):
+        """decode one chunk
+
+        Args:
+            x_chunk (numpy.array): shape[B, T, D]
+            x_chunk_lens (numpy.array): shape[B]
+            model_type (str): online model type
+
+        Returns:
+            [type]: [description]
+        """
+        if "deepspeech2online" in model_type :
+            input_names = self.am_predictor.get_input_names()
+            audio_handle = self.am_predictor.get_input_handle(input_names[0])
+            audio_len_handle = self.am_predictor.get_input_handle(input_names[1])
+            h_box_handle = self.am_predictor.get_input_handle(input_names[2])
+            c_box_handle = self.am_predictor.get_input_handle(input_names[3])
+
+            audio_handle.reshape(x_chunk.shape)
+            audio_handle.copy_from_cpu(x_chunk)
+
+            audio_len_handle.reshape(x_chunk_lens.shape)
+            audio_len_handle.copy_from_cpu(x_chunk_lens)
+
+            h_box_handle.reshape(self.chunk_state_h_box.shape)
+            h_box_handle.copy_from_cpu(self.chunk_state_h_box)
+
+            c_box_handle.reshape(self.chunk_state_c_box.shape)
+            c_box_handle.copy_from_cpu(self.chunk_state_c_box)
+
+            output_names = self.am_predictor.get_output_names()
+            output_handle = self.am_predictor.get_output_handle(output_names[0])
+            output_lens_handle = self.am_predictor.get_output_handle(output_names[1])
+            output_state_h_handle = self.am_predictor.get_output_handle(
+                output_names[2])
+            output_state_c_handle = self.am_predictor.get_output_handle(
+                output_names[3])
+
+            self.am_predictor.run()
+
+            output_chunk_probs = output_handle.copy_to_cpu()
+            output_chunk_lens = output_lens_handle.copy_to_cpu()
+            self.chunk_state_h_box = output_state_h_handle.copy_to_cpu()
+            self.chunk_state_c_box = output_state_c_handle.copy_to_cpu()
+
+            self.decoder.next(output_chunk_probs, output_chunk_lens)
+            trans_best, trans_beam = self.decoder.decode()
+
+            return trans_best[0]
+
+        elif "conformer" in model_type or "transformer" in model_type:
+            raise Exception("invalid model name")
+        else:
+            raise Exception("invalid model name")
+
+    def _pcm16to32(self, audio):
+        """pcm int16 to float32
+
+        Args:
+            audio(numpy.array): numpy.int16
+
+        Returns:
+            audio(numpy.array): numpy.float32
+        """
+        if audio.dtype == np.int16:
+            audio = audio.astype("float32")
+            bits = np.iinfo(np.int16).bits
+            audio = audio / (2**(bits - 1))
+        return audio
+
+    def extract_feat(self, samples, sample_rate):
+        """extract feat
+
+        Args:
+            samples (numpy.array): numpy.float32
+            sample_rate (int): sample rate
+
+        Returns:
+            x_chunk (numpy.array): shape[B, T, D]
+            x_chunk_lens (numpy.array): shape[B]
+        """
+        # pcm16 -> pcm 32
+        samples = self._pcm16to32(samples)
+
+        # read audio
+        speech_segment = SpeechSegment.from_pcm(
+            samples, sample_rate, transcript=" ")
+        # audio augment
+        self.collate_fn_test.augmentation.transform_audio(speech_segment)
+
+        # extract speech feature
+        spectrum, transcript_part = self.collate_fn_test._speech_featurizer.featurize(
+            speech_segment, self.collate_fn_test.keep_transcription_text)
+        # CMVN spectrum
+        if self.collate_fn_test._normalizer:
+            spectrum = self.collate_fn_test._normalizer.apply(spectrum)
+
+        # spectrum augment
+        audio = self.collate_fn_test.augmentation.transform_feature(spectrum)
+
+        audio_len = audio.shape[0]
+        audio = paddle.to_tensor(audio, dtype='float32')
+        # audio_len = paddle.to_tensor(audio_len)
+        audio = paddle.unsqueeze(audio, axis=0)
+
+        x_chunk = audio.numpy()
+        x_chunk_lens = np.array([audio_len])
+
+        return x_chunk, x_chunk_lens
+
+
+class ASREngine(BaseEngine):
+    """ASR server engine
+
+    Args:
+        metaclass: Defaults to Singleton.
+    """
+
+    def __init__(self):
+        super(ASREngine, self).__init__()
+
+    def init(self, config: dict) -> bool:
+        """init engine resource
+
+        Args:
+            config_file (str): config file
+
+        Returns:
+            bool: init failed or success
+        """
+        self.input = None
+        self.output = ""
+        self.executor = ASRServerExecutor()
+        self.config = config
+
+        self.executor._init_from_path(
+            model_type=self.config.model_type,
+            am_model=self.config.am_model,
+            am_params=self.config.am_params,
+            lang=self.config.lang,
+            sample_rate=self.config.sample_rate,
+            cfg_path=self.config.cfg_path,
+            decode_method=self.config.decode_method,
+            am_predictor_conf=self.config.am_predictor_conf)
+
+        logger.info("Initialize ASR server engine successfully.")
+        return True
+
+    def preprocess(self, samples, sample_rate):
+        """preprocess
+
+        Args:
+            samples (numpy.array): numpy.float32
+            sample_rate (int): sample rate
+
+        Returns:
+            x_chunk (numpy.array): shape[B, T, D]
+            x_chunk_lens (numpy.array): shape[B]
+        """
+        x_chunk, x_chunk_lens = self.executor.extract_feat(samples, sample_rate)
+        return x_chunk, x_chunk_lens
+
+    def run(self, x_chunk, x_chunk_lens, decoder_chunk_size=1):
+        """run online engine
+
+        Args:
+            x_chunk (numpy.array): shape[B, T, D]
+            x_chunk_lens (numpy.array): shape[B]
+            decoder_chunk_size(int)
+        """
+        self.output = self.executor.decode_one_chunk(x_chunk, x_chunk_lens, self.config.model_type)
+
+    def postprocess(self):
+        """postprocess
+        """
+        return self.output
+
+    def reset(self):
+        """reset engine decoder and inference state
+        """
+        self.executor.reset_decoder_and_chunk()
+        self.output = ""
diff --git a/paddlespeech/server/engine/engine_factory.py b/paddlespeech/server/engine/engine_factory.py
index c39c44ca..2a39fb79 100644
--- a/paddlespeech/server/engine/engine_factory.py
+++ b/paddlespeech/server/engine/engine_factory.py
@@ -25,6 +25,9 @@ class EngineFactory(object):
         elif engine_name == 'asr' and engine_type == 'python':
             from paddlespeech.server.engine.asr.python.asr_engine import ASREngine
             return ASREngine()
+        elif engine_name == 'asr' and engine_type == 'online':
+            from paddlespeech.server.engine.asr.online.asr_engine import ASREngine
+            return ASREngine()
         elif engine_name == 'tts' and engine_type == 'inference':
             from paddlespeech.server.engine.tts.paddleinference.tts_engine import TTSEngine
             return TTSEngine()
diff --git a/paddlespeech/server/tests/asr/online/microphone_client.py b/paddlespeech/server/tests/asr/online/microphone_client.py
new file mode 100644
index 00000000..74d457c5
--- /dev/null
+++ b/paddlespeech/server/tests/asr/online/microphone_client.py
@@ -0,0 +1,154 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+record wave from the mic
+"""
+
+import threading
+import pyaudio
+import wave
+import logging
+import asyncio
+import websockets
+import json
+from signal import SIGINT, SIGTERM
+
+
+class ASRAudioHandler(threading.Thread):
+    def __init__(self,
+                 url="127.0.0.1",
+                 port=8090):
+        threading.Thread.__init__(self)
+        self.url = url
+        self.port = port
+        self.url = "ws://" + self.url + ":" + str(self.port) + "/ws/asr"
+        self.fileName = "./output.wav"
+        self.chunk = 5120
+        self.format = pyaudio.paInt16
+        self.channels = 1
+        self.rate = 16000
+        self._running = True
+        self._frames = []
+        self.data_backup = []
+
+    def startrecord(self):
+        """
+        start a new thread to record wave
+        """
+        threading._start_new_thread(self.recording, ())
+
+    def recording(self):
+        """
+        recording wave
+        """
+        self._running = True
+        self._frames = []
+        p = pyaudio.PyAudio()
+        stream = p.open(format=self.format,
+                               channels=self.channels,
+                               rate=self.rate,
+                               input=True,
+                               frames_per_buffer=self.chunk)
+        while(self._running):
+            data = stream.read(self.chunk)
+            self._frames.append(data)
+            self.data_backup.append(data)
+
+        stream.stop_stream()
+        stream.close()
+        p.terminate()
+
+    def save(self):
+        """
+        save wave data
+        """
+        p = pyaudio.PyAudio()
+        wf = wave.open(self.fileName, 'wb')
+        wf.setnchannels(self.channels)
+        wf.setsampwidth(p.get_sample_size(self.format))
+        wf.setframerate(self.rate)
+        wf.writeframes(b''.join(self.data_backup))
+        wf.close()
+        p.terminate()
+
+    def stoprecord(self):
+        """
+        stop recording
+        """
+        self._running = False
+
+    async def run(self):
+        aa = input("是否开始录音？   (y/n)")
+        if aa.strip() == "y":
+            self.startrecord()
+            logging.info("*" * 10 + "开始录音，请输入语音")
+
+            async with websockets.connect(self.url) as ws:
+                # 发送开始指令
+                audio_info = json.dumps({
+                                "name": "test.wav",
+                                "signal": "start",
+                                "nbest": 5
+                                }, sort_keys=True, indent=4, separators=(',', ': '))
+                await ws.send(audio_info)
+                msg = await ws.recv()
+                logging.info("receive msg={}".format(msg))
+
+                # send bytes data
+                logging.info("结束录音请: Ctrl + c。继续请按回车。")
+                try:
+                    while True:
+                        while len(self._frames) > 0:
+                            await ws.send(self._frames.pop(0))
+                            msg = await ws.recv()
+                            logging.info("receive msg={}".format(msg))
+                except asyncio.CancelledError:
+                    # quit
+                    # send finished 
+                    audio_info = json.dumps({
+                                    "name": "test.wav",
+                                    "signal": "end",
+                                    "nbest": 5
+                                    }, sort_keys=True, indent=4, separators=(',', ': '))
+                    await ws.send(audio_info)
+                    msg = await ws.recv()
+                    logging.info("receive msg={}".format(msg))
+
+                    self.stoprecord()
+                    logging.info("*" * 10 + "录音结束")
+                    self.save()
+        elif aa.strip() == "n":
+            exit()
+        else:
+            print("无效输入!")
+            exit()
+
+
+if __name__ == "__main__":
+
+    logging.basicConfig(level=logging.INFO)
+    logging.info("asr websocket client start")
+
+    handler = ASRAudioHandler("127.0.0.1", 8090)
+    loop = asyncio.get_event_loop()
+    main_task = asyncio.ensure_future(handler.run())
+    for signal in [SIGINT, SIGTERM]:
+        loop.add_signal_handler(signal, main_task.cancel)
+    try:
+        loop.run_until_complete(main_task)
+    finally:
+        loop.close()
+
+    logging.info("asr websocket client finished")
diff --git a/paddlespeech/server/tests/asr/online/websocket_client.py b/paddlespeech/server/tests/asr/online/websocket_client.py
new file mode 100644
index 00000000..d849ffea
--- /dev/null
+++ b/paddlespeech/server/tests/asr/online/websocket_client.py
@@ -0,0 +1,115 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+import argparse
+import logging
+import time
+import os
+import json
+import wave
+import numpy as np
+import asyncio
+import websockets
+import soundfile
+
+
+class ASRAudioHandler:
+    def __init__(self,
+                 url="127.0.0.1",
+                 port=8090):
+        self.url = url
+        self.port = port
+        self.url = "ws://" + self.url + ":" + str(self.port) + "/ws/asr"
+
+    def read_wave(self, wavfile_path: str):
+        samples, sample_rate = soundfile.read(wavfile_path, dtype='int16')
+        x_len = len(samples)
+        chunk_stride = 40 * 16      #40ms, sample_rate = 16kHz
+        chunk_size = 80 * 16        #80ms, sample_rate = 16kHz
+
+        if (x_len - chunk_size) % chunk_stride != 0:
+            padding_len_x = chunk_stride - (x_len - chunk_size
+                                            ) % chunk_stride
+        else:
+            padding_len_x = 0
+
+        padding = np.zeros(
+            (padding_len_x), dtype=samples.dtype)
+        padded_x = np.concatenate([samples, padding], axis=0)
+
+        num_chunk = (x_len + padding_len_x - chunk_size) / chunk_stride + 1
+        num_chunk = int(num_chunk)
+
+        for i in range(0, num_chunk):
+            start = i * chunk_stride
+            end = start + chunk_size
+            x_chunk = padded_x[start:end]
+            yield x_chunk
+
+    async def run(self, wavfile_path: str):
+        logging.info("send a message to the server")
+        # 读取音频
+        # self.read_wave()
+        # 发送 websocket 的 handshake 协议头
+        async with websockets.connect(self.url) as ws:
+            # server 端已经接收到 handshake 协议头
+            # 发送开始指令
+            audio_info = json.dumps({
+                            "name": "test.wav",
+                            "signal": "start",
+                            "nbest": 5
+                            }, sort_keys=True, indent=4, separators=(',', ': '))
+            await ws.send(audio_info)
+            msg = await ws.recv()
+            logging.info("receive msg={}".format(msg))
+
+            # send chunk audio data to engine
+            for chunk_data in self.read_wave(wavfile_path):
+                await ws.send(chunk_data.tobytes())
+                msg = await ws.recv()
+                logging.info("receive msg={}".format(msg))
+
+            # finished 
+            audio_info = json.dumps({
+                            "name": "test.wav",
+                            "signal": "end",
+                            "nbest": 5
+                            }, sort_keys=True, indent=4, separators=(',', ': '))
+            await ws.send(audio_info)
+            msg = await ws.recv()
+            logging.info("receive msg={}".format(msg))
+
+
+def main(args):
+    logging.basicConfig(level=logging.INFO)
+    logging.info("asr websocket client start")
+    handler = ASRAudioHandler("127.0.0.1", 8090)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(handler.run(args.wavfile))
+    logging.info("asr websocket client finished")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--wavfile",
+        action="store",
+        help="wav file path ",
+        default="./16_audio.wav")
+    args = parser.parse_args()
+
+    main(args)
diff --git a/paddlespeech/server/utils/buffer.py b/paddlespeech/server/utils/buffer.py
new file mode 100644
index 00000000..4c1a3958
--- /dev/null
+++ b/paddlespeech/server/utils/buffer.py
@@ -0,0 +1,59 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Frame(object):
+    """Represents a "frame" of audio data."""
+
+    def __init__(self, bytes, timestamp, duration):
+        self.bytes = bytes
+        self.timestamp = timestamp
+        self.duration = duration
+
+
+class ChunkBuffer(object):
+    def __init__(self,
+                 frame_duration_ms=80,
+                 shift_ms=40,
+                 sample_rate=16000,
+                 sample_width=2):
+        self.sample_rate = sample_rate
+        self.frame_duration_ms = frame_duration_ms
+        self.shift_ms = shift_ms
+        self.remained_audio = b''
+        self.sample_width = sample_width  # int16 = 2; float32 = 4
+
+    def frame_generator(self, audio):
+        """Generates audio frames from PCM audio data.
+        Takes the desired frame duration in milliseconds, the PCM data, and
+        the sample rate.
+        Yields Frames of the requested duration.
+        """
+        audio = self.remained_audio + audio
+        self.remained_audio = b''
+
+        n = int(self.sample_rate *
+                (self.frame_duration_ms / 1000.0) * self.sample_width)
+        shift_n = int(self.sample_rate *
+                      (self.shift_ms / 1000.0) * self.sample_width)
+        offset = 0
+        timestamp = 0.0
+        duration = (float(n) / self.sample_rate) / self.sample_width
+        shift_duration = (float(shift_n) / self.sample_rate) / self.sample_width
+        while offset + n <= len(audio):
+            yield Frame(audio[offset:offset + n], timestamp, duration)
+            timestamp += shift_duration
+            offset += shift_n
+
+        self.remained_audio += audio[offset:]
diff --git a/paddlespeech/server/utils/vad.py b/paddlespeech/server/utils/vad.py
new file mode 100644
index 00000000..e9b55717
--- /dev/null
+++ b/paddlespeech/server/utils/vad.py
@@ -0,0 +1,79 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import collections
+import logging
+
+import webrtcvad
+
+
+class VADAudio():
+    def __init__(self,
+                 aggressiveness,
+                 rate,
+                 frame_duration_ms,
+                 sample_width=2,
+                 padding_ms=200,
+                 padding_ratio=0.9):
+        """Initializes VAD with given aggressivenes and sets up internal queues"""
+        self.vad = webrtcvad.Vad(aggressiveness)
+        self.rate = rate
+        self.sample_width = sample_width
+        self.frame_duration_ms = frame_duration_ms
+        self._frame_length = int(rate * (frame_duration_ms / 1000.0) *
+                                 self.sample_width)
+        self._buffer_queue = collections.deque()
+        self.ring_buffer = collections.deque(maxlen=padding_ms //
+                                             frame_duration_ms)
+        self._ratio = padding_ratio
+        self.triggered = False
+
+    def add_audio(self, audio):
+        """Adds new audio to internal queue"""
+        for x in audio:
+            self._buffer_queue.append(x)
+
+    def frame_generator(self):
+        """Generator that yields audio frames of frame_duration_ms"""
+        while len(self._buffer_queue) > self._frame_length:
+            frame = bytearray()
+            for _ in range(self._frame_length):
+                frame.append(self._buffer_queue.popleft())
+            yield bytes(frame)
+
+    def vad_collector(self):
+        """Generator that yields series of consecutive audio frames comprising each utterence, separated by yielding a single None.
+            Determines voice activity by ratio of frames in padding_ms. Uses a buffer to include padding_ms prior to being triggered.
+            Example: (frame, ..., frame, None, frame, ..., frame, None, ...)
+                      |---utterence---|        |---utterence---|
+        """
+        for frame in self.frame_generator():
+            is_speech = self.vad.is_speech(frame, self.rate)
+            if not self.triggered:
+                self.ring_buffer.append((frame, is_speech))
+                num_voiced = len(
+                    [f for f, speech in self.ring_buffer if speech])
+                if num_voiced > self._ratio * self.ring_buffer.maxlen:
+                    self.triggered = True
+                    for f, s in self.ring_buffer:
+                        yield f
+                    self.ring_buffer.clear()
+            else:
+                yield frame
+                self.ring_buffer.append((frame, is_speech))
+                num_unvoiced = len(
+                    [f for f, speech in self.ring_buffer if not speech])
+                if num_unvoiced > self._ratio * self.ring_buffer.maxlen:
+                    self.triggered = False
+                    yield None
+                    self.ring_buffer.clear()
diff --git a/paddlespeech/server/ws/__init__.py b/paddlespeech/server/ws/__init__.py
new file mode 100644
index 00000000..97043fd7
--- /dev/null
+++ b/paddlespeech/server/ws/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/paddlespeech/server/ws/api.py b/paddlespeech/server/ws/api.py
new file mode 100644
index 00000000..10664d11
--- /dev/null
+++ b/paddlespeech/server/ws/api.py
@@ -0,0 +1,38 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List
+
+from fastapi import APIRouter
+
+from paddlespeech.server.ws.asr_socket import router as asr_router
+
+_router = APIRouter()
+
+
+def setup_router(api_list: List):
+    """setup router for fastapi
+    Args:
+        api_list (List): [asr, tts]
+    Returns:
+        APIRouter
+    """
+    for api_name in api_list:
+        if api_name == 'asr':
+            _router.include_router(asr_router)
+        elif api_name == 'tts':
+            pass
+        else:
+            pass
+
+    return _router
diff --git a/paddlespeech/server/ws/asr_socket.py b/paddlespeech/server/ws/asr_socket.py
new file mode 100644
index 00000000..5cc9472c
--- /dev/null
+++ b/paddlespeech/server/ws/asr_socket.py
@@ -0,0 +1,106 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import base64
+import traceback
+from typing import Union
+import random
+import numpy as np
+import json
+
+from fastapi import APIRouter
+from fastapi import WebSocket
+from fastapi import WebSocketDisconnect
+from starlette.websockets import WebSocketState as WebSocketState
+
+from paddlespeech.server.engine.asr.online.asr_engine import ASREngine
+from paddlespeech.server.engine.engine_pool import get_engine_pool
+from paddlespeech.server.utils.buffer import ChunkBuffer
+from paddlespeech.server.utils.vad import VADAudio
+
+
+router = APIRouter()
+
+@router.websocket('/ws/asr')
+async def websocket_endpoint(websocket: WebSocket):
+
+    await websocket.accept()
+
+    # init buffer
+    chunk_buffer = ChunkBuffer(sample_width=2)
+    # init vad
+    vad = VADAudio(2, 16000, 20)
+
+    try:
+        while True:
+            # careful here, changed the source code from starlette.websockets
+            assert websocket.application_state == WebSocketState.CONNECTED
+            message = await websocket.receive()
+            websocket._raise_on_disconnect(message)
+            if "text" in message:
+                message = json.loads(message["text"])
+                if 'signal' not in message:
+                    resp = {
+                            "status": "ok",
+                            "message": "no valid json data"
+                            }
+                    await websocket.send_json(resp)
+
+                if message['signal'] == 'start':
+                    resp = {
+                            "status": "ok",
+                            "signal": "server_ready"
+                            }
+                    # do something at begining here
+                    await websocket.send_json(resp)
+                elif message['signal'] == 'end':
+                    engine_pool = get_engine_pool()
+                    asr_engine = engine_pool['asr']
+                    # reset single  engine for an new connection
+                    asr_engine.reset()
+                    resp = {
+                            "status": "ok",
+                            "signal": "finished"
+                            }
+                    await websocket.send_json(resp)
+                    break
+                else:
+                    resp = {
+                            "status": "ok",
+                            "message": "no valid json data"
+                            }
+                    await websocket.send_json(resp)
+            elif "bytes" in message:
+                message = message["bytes"]
+
+                # vad for input bytes audio
+                vad.add_audio(message)
+                message = b''.join(f for f in vad.vad_collector() if f is not None)
+
+                engine_pool = get_engine_pool()
+                asr_engine = engine_pool['asr']
+                asr_results = ""
+                frames = chunk_buffer.frame_generator(message)
+                for frame in frames:
+                    samples = np.frombuffer(frame.bytes, dtype=np.int16)
+                    sample_rate = asr_engine.config.sample_rate
+                    x_chunk, x_chunk_lens = asr_engine.preprocess(samples, sample_rate)
+                    asr_engine.run(x_chunk, x_chunk_lens)
+                    asr_results = asr_engine.postprocess()
+
+                asr_results = asr_engine.postprocess()
+                resp = {'asr_results': asr_results}
+
+                await websocket.send_json(resp)
+    except WebSocketDisconnect:
+        pass

From 9602749a3cb166cf57e04af81c069f387364905a Mon Sep 17 00:00:00 2001
From: qingen <qingenz123@126.com>
Date: Wed, 30 Mar 2022 17:29:22 +0800
Subject: [PATCH 097/126] [vec][search] update client image url, test=doc fix
 #1608

---
 demos/README.md                           | 1 +
 demos/README_cn.md                        | 1 +
 demos/audio_searching/docker-compose.yaml | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/demos/README.md b/demos/README.md
index 4482aa19..36e93dbf 100644
--- a/demos/README.md
+++ b/demos/README.md
@@ -4,6 +4,7 @@
 
 The directory containes many speech applications in multi scenarios.
 
+* audio searching - mass audio similarity retrieval
 * audio tagging - multi-label tagging of an audio file
 * automatic_video_subtitiles - generate subtitles from a video
 * metaverse - 2D AR with TTS  
diff --git a/demos/README_cn.md b/demos/README_cn.md
index 242b4f07..add6e25f 100644
--- a/demos/README_cn.md
+++ b/demos/README_cn.md
@@ -4,6 +4,7 @@
 
 该目录包含基于 PaddleSpeech 开发的不同场景的语音应用 Demo：
 
+* 声音检索 - 海量音频相似性检索。
 * 声音分类 - 基于 AudioSet 的 527 类标签的音频多标签分类。 
 * 视频字幕生成 - 识别视频中语音的文本，并进行文本后处理。
 * 元宇宙 - 基于语音合成的 2D 增强现实。
diff --git a/demos/audio_searching/docker-compose.yaml b/demos/audio_searching/docker-compose.yaml
index 8916e76f..16ac054d 100644
--- a/demos/audio_searching/docker-compose.yaml
+++ b/demos/audio_searching/docker-compose.yaml
@@ -64,7 +64,7 @@ services:
 
   webclient:
     container_name: audio-webclient
-    image: qingen1/paddlespeech-audio-search-client:2.3
+    image: paddlepaddle/paddlespeech-audio-search-client:2.3
     networks:
       app_net:
         ipv4_address: 172.16.23.13

From 0fc79f474de35656d7577938308a272ee9324d92 Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Wed, 30 Mar 2022 11:02:08 +0000
Subject: [PATCH 098/126] add CNNDecoder, test=tts

---
 examples/csmsc/tts3/conf/cnndecoder.yaml      | 107 ++++++++++++++
 .../csmsc/tts3/local/synthesize_streaming.sh  |  92 ++++++++++++
 examples/csmsc/tts3/run_cnndecoder.sh         |  48 +++++++
 .../t2s/models/fastspeech2/fastspeech2.py     |  47 +++++--
 .../t2s/modules/transformer/encoder.py        | 133 ++++++++++++++++++
 5 files changed, 417 insertions(+), 10 deletions(-)
 create mode 100644 examples/csmsc/tts3/conf/cnndecoder.yaml
 create mode 100755 examples/csmsc/tts3/local/synthesize_streaming.sh
 create mode 100755 examples/csmsc/tts3/run_cnndecoder.sh

diff --git a/examples/csmsc/tts3/conf/cnndecoder.yaml b/examples/csmsc/tts3/conf/cnndecoder.yaml
new file mode 100644
index 00000000..8b46fea4
--- /dev/null
+++ b/examples/csmsc/tts3/conf/cnndecoder.yaml
@@ -0,0 +1,107 @@
+# use CNND
+###########################################################
+#                FEATURE EXTRACTION SETTING               #
+###########################################################
+
+fs: 24000          # sr
+n_fft: 2048        # FFT size (samples).
+n_shift: 300       # Hop size (samples). 12.5ms
+win_length: 1200   # Window length (samples). 50ms
+                   # If set to null, it will be the same as fft_size.
+window: "hann"     # Window function.
+
+# Only used for feats_type != raw
+
+fmin: 80           # Minimum frequency of Mel basis.
+fmax: 7600         # Maximum frequency of Mel basis.
+n_mels: 80         # The number of mel basis.
+
+# Only used for the model using pitch features (e.g. FastSpeech2)
+f0min: 80          # Minimum f0 for pitch extraction.
+f0max: 400         # Maximum f0 for pitch extraction.
+
+
+###########################################################
+#                       DATA SETTING                      #
+###########################################################
+batch_size: 64
+num_workers: 4
+
+
+###########################################################
+#                       MODEL SETTING                     #
+###########################################################
+model:
+    adim: 384         # attention dimension
+    aheads: 2         # number of attention heads
+    elayers: 4        # number of encoder layers
+    eunits: 1536      # number of encoder ff units
+    dlayers: 4        # number of decoder layers
+    dunits: 1536      # number of decoder ff units
+    positionwise_layer_type: conv1d   # type of position-wise layer
+    positionwise_conv_kernel_size: 3  # kernel size of position wise conv layer
+    duration_predictor_layers: 2      # number of layers of duration predictor
+    duration_predictor_chans: 256     # number of channels of duration predictor
+    duration_predictor_kernel_size: 3 # filter size of duration predictor
+    postnet_layers: 5                 # number of layers of postnset
+    postnet_filts: 5                  # filter size of conv layers in postnet
+    postnet_chans: 256                # number of channels of conv layers in postnet
+    use_scaled_pos_enc: True          # whether to use scaled positional encoding
+    encoder_normalize_before: True    # whether to perform layer normalization before the input
+    decoder_normalize_before: True    # whether to perform layer normalization before the input
+    reduction_factor: 1               # reduction factor
+    encoder_type: transformer           # encoder type
+    decoder_type: cnndecoder           # decoder type
+    init_type: xavier_uniform         # initialization type
+    init_enc_alpha: 1.0               # initial value of alpha of encoder scaled position encoding
+    init_dec_alpha: 1.0               # initial value of alpha of decoder scaled position encoding
+    transformer_enc_dropout_rate: 0.2            # dropout rate for transformer encoder layer
+    transformer_enc_positional_dropout_rate: 0.2 # dropout rate for transformer encoder positional encoding
+    transformer_enc_attn_dropout_rate: 0.2       # dropout rate for transformer encoder attention layer
+    cnn_dec_dropout_rate: 0.2                    # dropout rate for cnn decoder layer
+    cnn_postnet_dropout_rate: 0.2
+    cnn_postnet_resblock_kernel_sizes: [256, 256] # kernel sizes for residual block of cnn_postnet
+    cnn_postnet_kernel_size: 5                   # kernel size of cnn_postnet
+    cnn_decoder_embedding_dim: 256
+    pitch_predictor_layers: 5                  # number of conv layers in pitch predictor
+    pitch_predictor_chans: 256                 # number of channels of conv layers in pitch predictor
+    pitch_predictor_kernel_size: 5             # kernel size of conv leyers in pitch predictor
+    pitch_predictor_dropout: 0.5               # dropout rate in pitch predictor
+    pitch_embed_kernel_size: 1                 # kernel size of conv embedding layer for pitch
+    pitch_embed_dropout: 0.0                   # dropout rate after conv embedding layer for pitch
+    stop_gradient_from_pitch_predictor: True   # whether to stop the gradient from pitch predictor to encoder
+    energy_predictor_layers: 2                 # number of conv layers in energy predictor
+    energy_predictor_chans: 256                # number of channels of conv layers in energy predictor
+    energy_predictor_kernel_size: 3            # kernel size of conv leyers in energy predictor
+    energy_predictor_dropout: 0.5              # dropout rate in energy predictor
+    energy_embed_kernel_size: 1                # kernel size of conv embedding layer for energy
+    energy_embed_dropout: 0.0                  # dropout rate after conv embedding layer for energy
+    stop_gradient_from_energy_predictor: False # whether to stop the gradient from energy predictor to encoder
+
+
+
+###########################################################
+#                       UPDATER SETTING                   #
+###########################################################
+updater:
+    use_masking: True                 # whether to apply masking for padded part in loss calculation
+
+
+###########################################################
+#                     OPTIMIZER SETTING                   #
+###########################################################
+optimizer:
+  optim: adam              # optimizer type
+  learning_rate: 0.001     # learning rate
+
+###########################################################
+#                     TRAINING SETTING                    #
+###########################################################
+max_epoch: 1000
+num_snapshots: 5
+
+
+###########################################################
+#                       OTHER SETTING                     #
+###########################################################
+seed: 10086
diff --git a/examples/csmsc/tts3/local/synthesize_streaming.sh b/examples/csmsc/tts3/local/synthesize_streaming.sh
new file mode 100755
index 00000000..69bb22df
--- /dev/null
+++ b/examples/csmsc/tts3/local/synthesize_streaming.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+config_path=$1
+train_output_path=$2
+ckpt_name=$3
+
+stage=0
+stop_stage=0
+
+# pwgan
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize_streaming.py \
+        --am=fastspeech2_csmsc \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=pwgan_csmsc \
+        --voc_config=pwg_baker_ckpt_0.4/pwg_default.yaml \
+        --voc_ckpt=pwg_baker_ckpt_0.4/pwg_snapshot_iter_400000.pdz \
+        --voc_stat=pwg_baker_ckpt_0.4/pwg_stats.npy \
+        --lang=zh \
+        --text=${BIN_DIR}/../sentences.txt \
+        --output_dir=${train_output_path}/test_e2e \
+        --phones_dict=dump/phone_id_map.txt \
+        --inference_dir=${train_output_path}/inference
+fi
+
+# for more GAN Vocoders
+# multi band melgan
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize_streaming.py \
+        --am=fastspeech2_csmsc \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=mb_melgan_csmsc \
+        --voc_config=mb_melgan_csmsc_ckpt_0.1.1/default.yaml \
+        --voc_ckpt=mb_melgan_csmsc_ckpt_0.1.1/snapshot_iter_1000000.pdz\
+        --voc_stat=mb_melgan_csmsc_ckpt_0.1.1/feats_stats.npy \
+        --lang=zh \
+        --text=${BIN_DIR}/../sentences.txt \
+        --output_dir=${train_output_path}/test_e2e \
+        --phones_dict=dump/phone_id_map.txt \
+        --inference_dir=${train_output_path}/inference
+fi
+
+# the pretrained models haven't release now
+# style melgan
+# style melgan's Dygraph to Static Graph is not ready now
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize_streaming.py \
+        --am=fastspeech2_csmsc \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=style_melgan_csmsc \
+        --voc_config=style_melgan_csmsc_ckpt_0.1.1/default.yaml \
+        --voc_ckpt=style_melgan_csmsc_ckpt_0.1.1/snapshot_iter_1500000.pdz \
+        --voc_stat=style_melgan_csmsc_ckpt_0.1.1/feats_stats.npy \
+        --lang=zh \
+        --text=${BIN_DIR}/../sentences.txt \
+        --output_dir=${train_output_path}/test_e2e \
+        --phones_dict=dump/phone_id_map.txt
+        # --inference_dir=${train_output_path}/inference
+fi
+
+# hifigan
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+    echo "in hifigan syn_e2e"
+    FLAGS_allocator_strategy=naive_best_fit \
+    FLAGS_fraction_of_gpu_memory_to_use=0.01 \
+    python3 ${BIN_DIR}/../synthesize_streaming.py \
+        --am=fastspeech2_csmsc \
+        --am_config=${config_path} \
+        --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \
+        --am_stat=dump/train/speech_stats.npy \
+        --voc=hifigan_csmsc \
+        --voc_config=hifigan_csmsc_ckpt_0.1.1/default.yaml \
+        --voc_ckpt=hifigan_csmsc_ckpt_0.1.1/snapshot_iter_2500000.pdz \
+        --voc_stat=hifigan_csmsc_ckpt_0.1.1/feats_stats.npy \
+        --lang=zh \
+        --text=${BIN_DIR}/../sentences.txt \
+        --output_dir=${train_output_path}/test_e2e \
+        --phones_dict=dump/phone_id_map.txt \
+        --inference_dir=${train_output_path}/inference
+fi
diff --git a/examples/csmsc/tts3/run_cnndecoder.sh b/examples/csmsc/tts3/run_cnndecoder.sh
new file mode 100755
index 00000000..5cccef01
--- /dev/null
+++ b/examples/csmsc/tts3/run_cnndecoder.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+source path.sh
+
+gpus=0,1
+stage=0
+stop_stage=100
+
+conf_path=conf/cnndecoder.yaml
+train_output_path=exp/cnndecoder
+ckpt_name=snapshot_iter_153.pdz
+
+# with the following command, you can choose the stage range you want to run
+# such as `./run.sh --stage 0 --stop-stage 0`
+# this can not be mixed use with `$1`, `$2` ...
+source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    # prepare data
+    ./local/preprocess.sh ${conf_path} || exit -1
+fi
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    # train model, all `ckpt` under `train_output_path/checkpoints/` dir
+    CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1
+fi
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    # synthesize, vocoder is pwgan
+    CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1
+fi
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+    # synthesize_e2e, vocoder is pwgan
+    CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize_e2e.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1
+fi
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+    # inference with static model
+    CUDA_VISIBLE_DEVICES=${gpus} ./local/inference.sh ${train_output_path} || exit -1
+fi
+
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+    # synthesize_e2e, vocoder is pwgan
+    CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize_streaming.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1
+fi
+
diff --git a/paddlespeech/t2s/models/fastspeech2/fastspeech2.py b/paddlespeech/t2s/models/fastspeech2/fastspeech2.py
index 73f5498e..1c805051 100644
--- a/paddlespeech/t2s/models/fastspeech2/fastspeech2.py
+++ b/paddlespeech/t2s/models/fastspeech2/fastspeech2.py
@@ -14,6 +14,7 @@
 # Modified from espnet(https://github.com/espnet/espnet)
 """Fastspeech2 related modules for paddle"""
 from typing import Dict
+from typing import List
 from typing import Sequence
 from typing import Tuple
 from typing import Union
@@ -32,6 +33,8 @@ from paddlespeech.t2s.modules.predictor.duration_predictor import DurationPredic
 from paddlespeech.t2s.modules.predictor.length_regulator import LengthRegulator
 from paddlespeech.t2s.modules.predictor.variance_predictor import VariancePredictor
 from paddlespeech.t2s.modules.tacotron2.decoder import Postnet
+from paddlespeech.t2s.modules.transformer.encoder import CNNDecoder
+from paddlespeech.t2s.modules.transformer.encoder import CNNPostnet
 from paddlespeech.t2s.modules.transformer.encoder import ConformerEncoder
 from paddlespeech.t2s.modules.transformer.encoder import TransformerEncoder
 
@@ -97,6 +100,12 @@ class FastSpeech2(nn.Layer):
             zero_triu: bool=False,
             conformer_enc_kernel_size: int=7,
             conformer_dec_kernel_size: int=31,
+            # for CNN Decoder
+            cnn_dec_dropout_rate: float=0.2,
+            cnn_postnet_dropout_rate: float=0.2,
+            cnn_postnet_resblock_kernel_sizes: List[int]=[256, 256],
+            cnn_postnet_kernel_size: int=5,
+            cnn_decoder_embedding_dim: int=256,
             # duration predictor
             duration_predictor_layers: int=2,
             duration_predictor_chans: int=384,
@@ -392,6 +401,13 @@ class FastSpeech2(nn.Layer):
                 activation_type=conformer_activation_type,
                 use_cnn_module=use_cnn_in_conformer,
                 cnn_module_kernel=conformer_dec_kernel_size, )
+        elif decoder_type == 'cnndecoder':
+            self.decoder = CNNDecoder(
+                emb_dim=adim,
+                odim=odim,
+                kernel_size=cnn_postnet_kernel_size,
+                dropout_rate=cnn_dec_dropout_rate,
+                resblock_kernel_sizes=cnn_postnet_resblock_kernel_sizes)
         else:
             raise ValueError(f"{decoder_type} is not supported.")
 
@@ -399,14 +415,21 @@ class FastSpeech2(nn.Layer):
         self.feat_out = nn.Linear(adim, odim * reduction_factor)
 
         # define postnet
-        self.postnet = (None if postnet_layers == 0 else Postnet(
-            idim=idim,
-            odim=odim,
-            n_layers=postnet_layers,
-            n_chans=postnet_chans,
-            n_filts=postnet_filts,
-            use_batch_norm=use_batch_norm,
-            dropout_rate=postnet_dropout_rate, ))
+        if decoder_type == 'cnndecoder':
+            self.postnet = CNNPostnet(
+                odim=odim,
+                kernel_size=cnn_postnet_kernel_size,
+                dropout_rate=cnn_postnet_dropout_rate,
+                resblock_kernel_sizes=cnn_postnet_resblock_kernel_sizes)
+        else:
+            self.postnet = (None if postnet_layers == 0 else Postnet(
+                idim=idim,
+                odim=odim,
+                n_layers=postnet_layers,
+                n_chans=postnet_chans,
+                n_filts=postnet_filts,
+                use_batch_norm=use_batch_norm,
+                dropout_rate=postnet_dropout_rate, ))
 
         nn.initializer.set_global_initializer(None)
 
@@ -562,6 +585,7 @@ class FastSpeech2(nn.Layer):
                     [olen // self.reduction_factor for olen in olens.numpy()])
             else:
                 olens_in = olens
+            # (B, 1, T)
             h_masks = self._source_mask(olens_in)
         else:
             h_masks = None
@@ -569,8 +593,11 @@ class FastSpeech2(nn.Layer):
 
         zs, _ = self.decoder(hs, h_masks)
         # (B, Lmax, odim)
-        before_outs = self.feat_out(zs).reshape(
-            (paddle.shape(zs)[0], -1, self.odim))
+        if self.decoder_type == 'cnndecoder':
+            before_outs = zs
+        else:
+            before_outs = self.feat_out(zs).reshape(
+                (paddle.shape(zs)[0], -1, self.odim))
 
         # postnet -> (B, Lmax//r * r, odim)
         if self.postnet is None:
diff --git a/paddlespeech/t2s/modules/transformer/encoder.py b/paddlespeech/t2s/modules/transformer/encoder.py
index 2b3ee788..25a11ff6 100644
--- a/paddlespeech/t2s/modules/transformer/encoder.py
+++ b/paddlespeech/t2s/modules/transformer/encoder.py
@@ -515,3 +515,136 @@ class ConformerEncoder(BaseEncoder):
         if self.intermediate_layers is not None:
             return xs, masks, intermediate_outputs
         return xs, masks
+
+
+class Conv1dResidualBlock(nn.Layer):
+    """
+    Special module for simplified version of Encoder class.
+    """
+
+    def __init__(self,
+                 idim: int=256,
+                 odim: int=256,
+                 kernel_size: int=5,
+                 dropout_rate: float=0.2):
+        super().__init__()
+        self.main_block = nn.Sequential(
+            nn.Conv1D(
+                idim, odim, kernel_size=kernel_size, padding=kernel_size // 2),
+            nn.ReLU(),
+            nn.BatchNorm1D(odim),
+            nn.Dropout(p=dropout_rate))
+        self.conv1d_residual = nn.Conv1D(idim, odim, kernel_size=1)
+
+    def forward(self, xs):
+        """Encode input sequence.
+        Args:
+            xs (Tensor): Input tensor (#batch, idim, T).
+        Returns:
+            Tensor: Output tensor (#batch, odim, T).
+        """
+        outputs = self.main_block(xs)
+        outputs = self.conv1d_residual(xs) + outputs
+        return outputs
+
+
+class CNNDecoder(nn.Layer):
+    """
+    Much simplified decoder than the original one with Prenet.
+    """
+
+    def __init__(
+            self,
+            emb_dim: int=256,
+            odim: int=80,
+            kernel_size: int=5,
+            dropout_rate: float=0.2,
+            resblock_kernel_sizes: List[int]=[256, 256], ):
+
+        super().__init__()
+
+        input_shape = emb_dim
+        out_sizes = resblock_kernel_sizes
+        out_sizes.append(out_sizes[-1])
+
+        in_sizes = [input_shape] + out_sizes[:-1]
+        self.residual_blocks = nn.LayerList([
+            Conv1dResidualBlock(
+                idim=in_channels,
+                odim=out_channels,
+                kernel_size=kernel_size,
+                dropout_rate=dropout_rate, )
+            for in_channels, out_channels in zip(in_sizes, out_sizes)
+        ])
+        self.conv1d = nn.Conv1D(
+            in_channels=out_sizes[-1], out_channels=odim, kernel_size=1)
+
+    def forward(self, xs, masks=None):
+        """Encode input sequence.
+        Args:
+            xs (Tensor): Input tensor (#batch, time, idim).
+            masks (Tensor): Mask tensor (#batch, 1, time).
+        Returns:
+            Tensor: Output tensor (#batch, time, odim).
+        """
+        # print("input.shape in CNNDecoder:",xs.shape)
+        # exchange the temporal dimension and the feature dimension
+        xs = xs.transpose([0, 2, 1])
+        if masks is not None:
+            xs = xs * masks
+
+        for layer in self.residual_blocks:
+            outputs = layer(xs)
+            if masks is not None:
+                # input_mask B * 1 * T
+                outputs = outputs * masks
+            xs = outputs
+        outputs = self.conv1d(outputs)
+        if masks is not None:
+            outputs = outputs * masks
+        outputs = outputs.transpose([0, 2, 1])
+        # print("outputs.shape in CNNDecoder:",outputs.shape)
+        return outputs, masks
+
+
+class CNNPostnet(nn.Layer):
+    def __init__(
+            self,
+            odim: int=80,
+            kernel_size: int=5,
+            dropout_rate: float=0.2,
+            resblock_kernel_sizes: List[int]=[256, 256], ):
+        super().__init__()
+        out_sizes = resblock_kernel_sizes
+        in_sizes = [odim] + out_sizes[:-1]
+        self.residual_blocks = nn.LayerList([
+            Conv1dResidualBlock(
+                idim=in_channels,
+                odim=out_channels,
+                kernel_size=kernel_size,
+                dropout_rate=dropout_rate)
+            for in_channels, out_channels in zip(in_sizes, out_sizes)
+        ])
+        self.conv1d = nn.Conv1D(
+            in_channels=out_sizes[-1], out_channels=odim, kernel_size=1)
+
+    def forward(self, xs, masks=None):
+        """Encode input sequence.
+        Args:
+            xs (Tensor): Input tensor (#batch, odim, time).
+            masks (Tensor): Mask tensor (#batch, 1, time).
+        Returns:
+            Tensor: Output tensor (#batch, odim, time).
+        """
+        # print("xs.shape in CNNPostnet:",xs.shape)
+        for layer in self.residual_blocks:
+            outputs = layer(xs)
+            if masks is not None:
+                # input_mask B * 1 * T
+                outputs = outputs * masks
+            xs = outputs
+        outputs = self.conv1d(outputs)
+        if masks is not None:
+            outputs = outputs * masks
+        # print("outputs.shape in CNNPostnet:",outputs.shape)
+        return outputs

From aa0cb236e2f78f3cb1b494650fca4c3a867591bf Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 31 Mar 2022 03:20:05 +0000
Subject: [PATCH 099/126] fix openfst patch

---
 speechx/cmake/external/openfst.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/speechx/cmake/external/openfst.cmake b/speechx/cmake/external/openfst.cmake
index dc9cdff6..9acf530a 100644
--- a/speechx/cmake/external/openfst.cmake
+++ b/speechx/cmake/external/openfst.cmake
@@ -13,7 +13,7 @@ ExternalProject_Add(openfst
                       "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR}"
                       "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}"
                       "LIBS=-lgflags_nothreads -lglog -lpthread"
-  COMMAND           ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR}
+  COMMAND           ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR}
   BUILD_COMMAND     make -j 4
 )
 link_directories(${openfst_PREFIX_DIR}/lib)

From 21c4132edacd436ceb2709f64ae5cdf2f39b3bd9 Mon Sep 17 00:00:00 2001
From: liangym <34430015+lym0302@users.noreply.github.com>
Date: Thu, 31 Mar 2022 15:16:32 +0800
Subject: [PATCH 100/126] Update paddlespeech_client.py

---
 paddlespeech/server/bin/paddlespeech_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/paddlespeech/server/bin/paddlespeech_client.py b/paddlespeech/server/bin/paddlespeech_client.py
index 40f17c63..413f0087 100644
--- a/paddlespeech/server/bin/paddlespeech_client.py
+++ b/paddlespeech/server/bin/paddlespeech_client.py
@@ -150,7 +150,7 @@ class TTSClientExecutor(BaseExecutor):
 
         res = requests.post(url, json.dumps(request))
         response_dict = res.json()
-        if not output:
+        if output is not None:
             self.postprocess(response_dict["result"]["audio"], output)
         return res
 

From f17347f454fd1000ce235e74713c443777c7a79a Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 31 Mar 2022 07:21:28 +0000
Subject: [PATCH 101/126] glog to stderr

---
 demos/audio_searching/src/encode.py              |  4 ----
 speechx/examples/CMakeLists.txt                  |  2 ++
 speechx/examples/README.md                       |  3 ++-
 speechx/examples/decoder/offline_decoder_main.cc |  1 +
 speechx/examples/decoder/run.sh                  |  5 ++++-
 speechx/examples/feat/linear_spectrogram_main.cc | 10 ++++++++++
 speechx/examples/feat/run.sh                     |  1 +
 7 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/demos/audio_searching/src/encode.py b/demos/audio_searching/src/encode.py
index cf5f29a4..35805784 100644
--- a/demos/audio_searching/src/encode.py
+++ b/demos/audio_searching/src/encode.py
@@ -11,11 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-
-import librosa
 import numpy as np
-from config import DEFAULT_TABLE
 from logs import LOGGER
 
 from paddlespeech.cli import VectorExecutor
diff --git a/speechx/examples/CMakeLists.txt b/speechx/examples/CMakeLists.txt
index ef0a72b8..7f1543c2 100644
--- a/speechx/examples/CMakeLists.txt
+++ b/speechx/examples/CMakeLists.txt
@@ -3,3 +3,5 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 add_subdirectory(feat)
 add_subdirectory(nnet)
 add_subdirectory(decoder)
+
+add_subdirectory(glog)
\ No newline at end of file
diff --git a/speechx/examples/README.md b/speechx/examples/README.md
index 941c4272..705ca200 100644
--- a/speechx/examples/README.md
+++ b/speechx/examples/README.md
@@ -1,8 +1,9 @@
 # Examples
 
-* decoder - online decoder to work as offline
+* glog - glog usage
 * feat - mfcc, linear 
 * nnet - ds2 nn
+* decoder - online decoder to work as offline
 
 ## How to run
 
diff --git a/speechx/examples/decoder/offline_decoder_main.cc b/speechx/examples/decoder/offline_decoder_main.cc
index 44127c73..eccd7c09 100644
--- a/speechx/examples/decoder/offline_decoder_main.cc
+++ b/speechx/examples/decoder/offline_decoder_main.cc
@@ -63,6 +63,7 @@ int main(int argc, char* argv[]) {
 
     int32 chunk_size = 35;
     decoder.InitDecoder();
+    LOG(INFO) << "chunk size: " << chunk_size;
 
     for (; !feature_reader.Done(); feature_reader.Next()) {
         string utt = feature_reader.Key();
diff --git a/speechx/examples/decoder/run.sh b/speechx/examples/decoder/run.sh
index fc5e9182..1e5a678c 100755
--- a/speechx/examples/decoder/run.sh
+++ b/speechx/examples/decoder/run.sh
@@ -25,6 +25,9 @@ model_dir=../paddle_asr_model
 feat_wspecifier=./feats.ark
 cmvn=./cmvn.ark
 
+
+export GLOG_logtostderr=1
+
 # 3. run feat
 linear_spectrogram_main \
     --wav_rspecifier=scp:$model_dir/wav.scp \
@@ -37,4 +40,4 @@ offline_decoder_main \
     --model_path=$model_dir/avg_1.jit.pdmodel \
     --param_path=$model_dir/avg_1.jit.pdparams \
     --dict_file=$model_dir/vocab.txt \
-    --lm_path=$model_dir/avg_1.jit.klm
\ No newline at end of file
+    --lm_path=$model_dir/avg_1.jit.klm
diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc
index 9ed4d6f9..a27db56f 100644
--- a/speechx/examples/feat/linear_spectrogram_main.cc
+++ b/speechx/examples/feat/linear_spectrogram_main.cc
@@ -25,6 +25,8 @@
 #include "kaldi/util/kaldi-io.h"
 #include "kaldi/util/table-types.h"
 
+#include <glog/logging.h>
+
 DEFINE_string(wav_rspecifier, "", "test wav scp path");
 DEFINE_string(feature_wspecifier, "", "output feats wspecifier");
 DEFINE_string(cmvn_write_path, "./cmvn.ark", "write cmvn");
@@ -172,6 +174,9 @@ int main(int argc, char* argv[]) {
     ppspeech::LinearSpectrogramOptions opt;
     opt.frame_opts.frame_length_ms = 20;
     opt.frame_opts.frame_shift_ms = 10;
+    LOG(INFO) << "frame length (ms):" << opt.frame_opts.frame_length_ms;
+    LOG(INFO) << "frame shift (ms):" << opt.frame_opts.frame_shift_ms;
+
     ppspeech::DecibelNormalizerOptions db_norm_opt;
     std::unique_ptr<ppspeech::FeatureExtractorInterface> base_feature_extractor(
         new ppspeech::DecibelNormalizer(db_norm_opt, std::move(data_source)));
@@ -190,6 +195,11 @@ int main(int argc, char* argv[]) {
     int sample_rate = 16000;
     int chunk_sample_size = streaming_chunk * sample_rate;
 
+    LOG(INFO) << "sr:" << sample_rate;
+    LOG(INFO) << "chunk size (s):" << streaming_chunk;
+    LOG(INFO) << "chunk size (sample):" << chunk_sample_size;
+
+
     for (; !wav_reader.Done(); wav_reader.Next()) {
         std::string utt = wav_reader.Key();
         const kaldi::WaveData& wave_data = wav_reader.Value();
diff --git a/speechx/examples/feat/run.sh b/speechx/examples/feat/run.sh
index bd21bd7f..29c49d32 100755
--- a/speechx/examples/feat/run.sh
+++ b/speechx/examples/feat/run.sh
@@ -25,6 +25,7 @@ feat_wspecifier=./feats.ark
 cmvn=./cmvn.ark
 
 # 3. run feat
+export GLOG_logtostderr=1
 linear_spectrogram_main \
     --wav_rspecifier=scp:$model_dir/wav.scp \
     --feature_wspecifier=ark,t:$feat_wspecifier \

From 2ec8d608bf1ad5b0be9c36dc4339702271c27a6e Mon Sep 17 00:00:00 2001
From: WilliamZhang06 <xd_0614@163.com>
Date: Thu, 31 Mar 2022 16:06:16 +0800
Subject: [PATCH 102/126] fixed comments, test=doc

---
 paddlespeech/server/conf/application.yaml     | 13 ++--
 paddlespeech/server/conf/ws_application.yaml  | 51 ++++++++++++++++
 .../tests/asr/online/microphone_client.py     | 61 +++++++++++--------
 .../tests/asr/online/websocket_client.py      | 56 ++++++++---------
 paddlespeech/server/utils/vad.py              |  7 +--
 paddlespeech/server/ws/asr_socket.py          | 48 +++++++--------
 6 files changed, 142 insertions(+), 94 deletions(-)
 create mode 100644 paddlespeech/server/conf/ws_application.yaml

diff --git a/paddlespeech/server/conf/application.yaml b/paddlespeech/server/conf/application.yaml
index 40de8e3b..849349c2 100644
--- a/paddlespeech/server/conf/application.yaml
+++ b/paddlespeech/server/conf/application.yaml
@@ -3,18 +3,15 @@
 #################################################################################
 #                             SERVER SETTING                                    #
 #################################################################################
-host: 0.0.0.0
+host: 127.0.0.1
 port: 8090
 
 # The task format in the engin_list is: <speech task>_<engine type>
 # task choices = ['asr_python', 'asr_inference', 'tts_python', 'tts_inference']
-# protocol: 'http'
-# engine_list: ['asr_python', 'tts_python', 'cls_python']
-
-
-# websocket, http (only choose one). websocket only support online engine type.
-protocol: 'websocket'
-engine_list: ['asr_online']
+# protocol = ['websocket', 'http'] (only one can be selected). 
+# http only support offline engine type.
+protocol: 'http'
+engine_list: ['asr_python', 'tts_python', 'cls_python']
 
 
 #################################################################################
diff --git a/paddlespeech/server/conf/ws_application.yaml b/paddlespeech/server/conf/ws_application.yaml
new file mode 100644
index 00000000..ef23593e
--- /dev/null
+++ b/paddlespeech/server/conf/ws_application.yaml
@@ -0,0 +1,51 @@
+# This is the parameter configuration file for PaddleSpeech Serving.
+
+#################################################################################
+#                             SERVER SETTING                                    #
+#################################################################################
+host: 0.0.0.0
+port: 8091
+
+# The task format in the engin_list is: <speech task>_<engine type>
+# task choices = ['asr_online', 'tts_online']
+# protocol = ['websocket', 'http'] (only one can be selected).
+# websocket only support online engine type.
+protocol: 'websocket'
+engine_list: ['asr_online']
+
+
+#################################################################################
+#                                ENGINE CONFIG                                  #
+#################################################################################
+
+################################### ASR #########################################
+################### speech task: asr; engine_type: online #######################
+asr_online:
+    model_type: 'deepspeech2online_aishell'
+    am_model: # the pdmodel file of am static model [optional]
+    am_params:  # the pdiparams file of am static model [optional]
+    lang: 'zh'
+    sample_rate: 16000
+    cfg_path: 
+    decode_method: 
+    force_yes: True
+
+    am_predictor_conf:
+        device:  # set 'gpu:id' or 'cpu'
+        switch_ir_optim: True
+        glog_info: False  # True -> print glog
+        summary: True  # False -> do not show predictor config
+
+    chunk_buffer_conf:
+        frame_duration_ms: 80
+        shift_ms: 40
+        sample_rate: 16000
+        sample_width: 2
+
+    vad_conf:
+        aggressiveness: 2
+        sample_rate: 16000
+        frame_duration_ms: 20
+        sample_width: 2
+        padding_ms: 200
+        padding_ratio: 0.9
diff --git a/paddlespeech/server/tests/asr/online/microphone_client.py b/paddlespeech/server/tests/asr/online/microphone_client.py
index 74d457c5..2ceaf6d0 100644
--- a/paddlespeech/server/tests/asr/online/microphone_client.py
+++ b/paddlespeech/server/tests/asr/online/microphone_client.py
@@ -11,25 +11,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """
 record wave from the mic
 """
-
+import asyncio
+import json
+import logging
 import threading
-import pyaudio
 import wave
-import logging
-import asyncio
+from signal import SIGINT
+from signal import SIGTERM
+
+import pyaudio
 import websockets
-import json
-from signal import SIGINT, SIGTERM
 
 
 class ASRAudioHandler(threading.Thread):
-    def __init__(self,
-                 url="127.0.0.1",
-                 port=8090):
+    def __init__(self, url="127.0.0.1", port=8091):
         threading.Thread.__init__(self)
         self.url = url
         self.port = port
@@ -56,12 +54,13 @@ class ASRAudioHandler(threading.Thread):
         self._running = True
         self._frames = []
         p = pyaudio.PyAudio()
-        stream = p.open(format=self.format,
-                               channels=self.channels,
-                               rate=self.rate,
-                               input=True,
-                               frames_per_buffer=self.chunk)
-        while(self._running):
+        stream = p.open(
+            format=self.format,
+            channels=self.channels,
+            rate=self.rate,
+            input=True,
+            frames_per_buffer=self.chunk)
+        while (self._running):
             data = stream.read(self.chunk)
             self._frames.append(data)
             self.data_backup.append(data)
@@ -97,11 +96,15 @@ class ASRAudioHandler(threading.Thread):
 
             async with websockets.connect(self.url) as ws:
                 # 发送开始指令
-                audio_info = json.dumps({
-                                "name": "test.wav",
-                                "signal": "start",
-                                "nbest": 5
-                                }, sort_keys=True, indent=4, separators=(',', ': '))
+                audio_info = json.dumps(
+                    {
+                        "name": "test.wav",
+                        "signal": "start",
+                        "nbest": 5
+                    },
+                    sort_keys=True,
+                    indent=4,
+                    separators=(',', ': '))
                 await ws.send(audio_info)
                 msg = await ws.recv()
                 logging.info("receive msg={}".format(msg))
@@ -117,11 +120,15 @@ class ASRAudioHandler(threading.Thread):
                 except asyncio.CancelledError:
                     # quit
                     # send finished 
-                    audio_info = json.dumps({
-                                    "name": "test.wav",
-                                    "signal": "end",
-                                    "nbest": 5
-                                    }, sort_keys=True, indent=4, separators=(',', ': '))
+                    audio_info = json.dumps(
+                        {
+                            "name": "test.wav",
+                            "signal": "end",
+                            "nbest": 5
+                        },
+                        sort_keys=True,
+                        indent=4,
+                        separators=(',', ': '))
                     await ws.send(audio_info)
                     msg = await ws.recv()
                     logging.info("receive msg={}".format(msg))
@@ -141,7 +148,7 @@ if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     logging.info("asr websocket client start")
 
-    handler = ASRAudioHandler("127.0.0.1", 8090)
+    handler = ASRAudioHandler("127.0.0.1", 8091)
     loop = asyncio.get_event_loop()
     main_task = asyncio.ensure_future(handler.run())
     for signal in [SIGINT, SIGTERM]:
diff --git a/paddlespeech/server/tests/asr/online/websocket_client.py b/paddlespeech/server/tests/asr/online/websocket_client.py
index d849ffea..58b1a452 100644
--- a/paddlespeech/server/tests/asr/online/websocket_client.py
+++ b/paddlespeech/server/tests/asr/online/websocket_client.py
@@ -11,26 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
-
 import argparse
-import logging
-import time
-import os
+import asyncio
 import json
-import wave
+import logging
+
 import numpy as np
-import asyncio
-import websockets
 import soundfile
+import websockets
 
 
 class ASRAudioHandler:
-    def __init__(self,
-                 url="127.0.0.1",
-                 port=8090):
+    def __init__(self, url="127.0.0.1", port=8090):
         self.url = url
         self.port = port
         self.url = "ws://" + self.url + ":" + str(self.port) + "/ws/asr"
@@ -38,17 +32,15 @@ class ASRAudioHandler:
     def read_wave(self, wavfile_path: str):
         samples, sample_rate = soundfile.read(wavfile_path, dtype='int16')
         x_len = len(samples)
-        chunk_stride = 40 * 16      #40ms, sample_rate = 16kHz
-        chunk_size = 80 * 16        #80ms, sample_rate = 16kHz
+        chunk_stride = 40 * 16  #40ms, sample_rate = 16kHz
+        chunk_size = 80 * 16  #80ms, sample_rate = 16kHz
 
         if (x_len - chunk_size) % chunk_stride != 0:
-            padding_len_x = chunk_stride - (x_len - chunk_size
-                                            ) % chunk_stride
+            padding_len_x = chunk_stride - (x_len - chunk_size) % chunk_stride
         else:
             padding_len_x = 0
 
-        padding = np.zeros(
-            (padding_len_x), dtype=samples.dtype)
+        padding = np.zeros((padding_len_x), dtype=samples.dtype)
         padded_x = np.concatenate([samples, padding], axis=0)
 
         num_chunk = (x_len + padding_len_x - chunk_size) / chunk_stride + 1
@@ -68,11 +60,15 @@ class ASRAudioHandler:
         async with websockets.connect(self.url) as ws:
             # server 端已经接收到 handshake 协议头
             # 发送开始指令
-            audio_info = json.dumps({
-                            "name": "test.wav",
-                            "signal": "start",
-                            "nbest": 5
-                            }, sort_keys=True, indent=4, separators=(',', ': '))
+            audio_info = json.dumps(
+                {
+                    "name": "test.wav",
+                    "signal": "start",
+                    "nbest": 5
+                },
+                sort_keys=True,
+                indent=4,
+                separators=(',', ': '))
             await ws.send(audio_info)
             msg = await ws.recv()
             logging.info("receive msg={}".format(msg))
@@ -84,11 +80,15 @@ class ASRAudioHandler:
                 logging.info("receive msg={}".format(msg))
 
             # finished 
-            audio_info = json.dumps({
-                            "name": "test.wav",
-                            "signal": "end",
-                            "nbest": 5
-                            }, sort_keys=True, indent=4, separators=(',', ': '))
+            audio_info = json.dumps(
+                {
+                    "name": "test.wav",
+                    "signal": "end",
+                    "nbest": 5
+                },
+                sort_keys=True,
+                indent=4,
+                separators=(',', ': '))
             await ws.send(audio_info)
             msg = await ws.recv()
             logging.info("receive msg={}".format(msg))
@@ -97,7 +97,7 @@ class ASRAudioHandler:
 def main(args):
     logging.basicConfig(level=logging.INFO)
     logging.info("asr websocket client start")
-    handler = ASRAudioHandler("127.0.0.1", 8090)
+    handler = ASRAudioHandler("127.0.0.1", 8091)
     loop = asyncio.get_event_loop()
     loop.run_until_complete(handler.run(args.wavfile))
     logging.info("asr websocket client finished")
diff --git a/paddlespeech/server/utils/vad.py b/paddlespeech/server/utils/vad.py
index e9b55717..a2dcf68b 100644
--- a/paddlespeech/server/utils/vad.py
+++ b/paddlespeech/server/utils/vad.py
@@ -12,16 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import collections
-import logging
 
 import webrtcvad
 
 
 class VADAudio():
     def __init__(self,
-                 aggressiveness,
-                 rate,
-                 frame_duration_ms,
+                 aggressiveness=2,
+                 rate=16000,
+                 frame_duration_ms=20,
                  sample_width=2,
                  padding_ms=200,
                  padding_ratio=0.9):
diff --git a/paddlespeech/server/ws/asr_socket.py b/paddlespeech/server/ws/asr_socket.py
index 5cc9472c..ea19816b 100644
--- a/paddlespeech/server/ws/asr_socket.py
+++ b/paddlespeech/server/ws/asr_socket.py
@@ -11,35 +11,39 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import base64
-import traceback
-from typing import Union
-import random
-import numpy as np
 import json
 
+import numpy as np
 from fastapi import APIRouter
 from fastapi import WebSocket
 from fastapi import WebSocketDisconnect
 from starlette.websockets import WebSocketState as WebSocketState
 
-from paddlespeech.server.engine.asr.online.asr_engine import ASREngine
 from paddlespeech.server.engine.engine_pool import get_engine_pool
 from paddlespeech.server.utils.buffer import ChunkBuffer
 from paddlespeech.server.utils.vad import VADAudio
 
-
 router = APIRouter()
 
+
 @router.websocket('/ws/asr')
 async def websocket_endpoint(websocket: WebSocket):
 
     await websocket.accept()
 
+    engine_pool = get_engine_pool()
+    asr_engine = engine_pool['asr']
     # init buffer
-    chunk_buffer = ChunkBuffer(sample_width=2)
+    chunk_buffer_conf = asr_engine.config.chunk_buffer_conf
+    chunk_buffer = ChunkBuffer(
+        sample_rate=chunk_buffer_conf['sample_rate'],
+        sample_width=chunk_buffer_conf['sample_width'])
     # init vad
-    vad = VADAudio(2, 16000, 20)
+    vad_conf = asr_engine.config.vad_conf
+    vad = VADAudio(
+        aggressiveness=vad_conf['aggressiveness'],
+        rate=vad_conf['sample_rate'],
+        frame_duration_ms=vad_conf['frame_duration_ms'])
 
     try:
         while True:
@@ -50,17 +54,11 @@ async def websocket_endpoint(websocket: WebSocket):
             if "text" in message:
                 message = json.loads(message["text"])
                 if 'signal' not in message:
-                    resp = {
-                            "status": "ok",
-                            "message": "no valid json data"
-                            }
+                    resp = {"status": "ok", "message": "no valid json data"}
                     await websocket.send_json(resp)
 
                 if message['signal'] == 'start':
-                    resp = {
-                            "status": "ok",
-                            "signal": "server_ready"
-                            }
+                    resp = {"status": "ok", "signal": "server_ready"}
                     # do something at begining here
                     await websocket.send_json(resp)
                 elif message['signal'] == 'end':
@@ -68,24 +66,19 @@ async def websocket_endpoint(websocket: WebSocket):
                     asr_engine = engine_pool['asr']
                     # reset single  engine for an new connection
                     asr_engine.reset()
-                    resp = {
-                            "status": "ok",
-                            "signal": "finished"
-                            }
+                    resp = {"status": "ok", "signal": "finished"}
                     await websocket.send_json(resp)
                     break
                 else:
-                    resp = {
-                            "status": "ok",
-                            "message": "no valid json data"
-                            }
+                    resp = {"status": "ok", "message": "no valid json data"}
                     await websocket.send_json(resp)
             elif "bytes" in message:
                 message = message["bytes"]
 
                 # vad for input bytes audio
                 vad.add_audio(message)
-                message = b''.join(f for f in vad.vad_collector() if f is not None)
+                message = b''.join(f for f in vad.vad_collector()
+                                   if f is not None)
 
                 engine_pool = get_engine_pool()
                 asr_engine = engine_pool['asr']
@@ -94,7 +87,8 @@ async def websocket_endpoint(websocket: WebSocket):
                 for frame in frames:
                     samples = np.frombuffer(frame.bytes, dtype=np.int16)
                     sample_rate = asr_engine.config.sample_rate
-                    x_chunk, x_chunk_lens = asr_engine.preprocess(samples, sample_rate)
+                    x_chunk, x_chunk_lens = asr_engine.preprocess(samples,
+                                                                  sample_rate)
                     asr_engine.run(x_chunk, x_chunk_lens)
                     asr_results = asr_engine.postprocess()
 

From cb66b742ab75287e35915e13b59cf71bf42d4146 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 31 Mar 2022 09:03:01 +0000
Subject: [PATCH 103/126] more comment

---
 .../examples/decoder/offline_decoder_main.cc  | 41 ++++++++++++++-----
 speechx/examples/decoder/run.sh               |  2 +-
 speechx/examples/feat/feature-mfcc-test.cc    |  1 -
 .../examples/feat/linear_spectrogram_main.cc  | 38 +++++++++--------
 speechx/speechx/frontend/feature_cache.h      |  4 +-
 speechx/speechx/frontend/raw_audio.h          |  3 +-
 6 files changed, 57 insertions(+), 32 deletions(-)

diff --git a/speechx/examples/decoder/offline_decoder_main.cc b/speechx/examples/decoder/offline_decoder_main.cc
index eccd7c09..3a94cc94 100644
--- a/speechx/examples/decoder/offline_decoder_main.cc
+++ b/speechx/examples/decoder/offline_decoder_main.cc
@@ -22,11 +22,12 @@
 #include "nnet/decodable.h"
 #include "nnet/paddle_nnet.h"
 
-DEFINE_string(feature_respecifier, "", "test feature rspecifier");
+DEFINE_string(feature_respecifier, "", "feature matrix rspecifier");
 DEFINE_string(model_path, "avg_1.jit.pdmodel", "paddle nnet model");
 DEFINE_string(param_path, "avg_1.jit.pdiparams", "paddle nnet model param");
 DEFINE_string(dict_file, "vocab.txt", "vocabulary of lm");
 DEFINE_string(lm_path, "lm.klm", "language model");
+DEFINE_int32(chunk_size, 35, "feat chunk size");
 
 
 using kaldi::BaseFloat;
@@ -43,14 +44,16 @@ int main(int argc, char* argv[]) {
     std::string model_params = FLAGS_param_path;
     std::string dict_file = FLAGS_dict_file;
     std::string lm_path = FLAGS_lm_path;
+    int32 chunk_size = FLAGS_chunk_size;
+    LOG(INFO) << "model path: " << model_graph;
+    LOG(INFO) << "model param: " << model_params;
+    LOG(INFO) << "dict path: " << dict_file;
+    LOG(INFO) << "lm path: " << lm_path;
+    LOG(INFO) << "chunk size (frame): " << chunk_size;
 
     int32 num_done = 0, num_err = 0;
 
-    ppspeech::CTCBeamSearchOptions opts;
-    opts.dict_file = dict_file;
-    opts.lm_path = lm_path;
-    ppspeech::CTCBeamSearch decoder(opts);
-
+    // frontend + nnet is decodable
     ppspeech::ModelOptions model_opts;
     model_opts.model_path = model_graph;
     model_opts.params_path = model_params;
@@ -60,34 +63,50 @@ int main(int argc, char* argv[]) {
         new ppspeech::RawDataCache());
     std::shared_ptr<ppspeech::Decodable> decodable(
         new ppspeech::Decodable(nnet, raw_data));
+    LOG(INFO) << "Init decodeable.";
 
-    int32 chunk_size = 35;
-    decoder.InitDecoder();
-    LOG(INFO) << "chunk size: " << chunk_size;
+    // init decoder
+    ppspeech::CTCBeamSearchOptions opts;
+    opts.dict_file = dict_file;
+    opts.lm_path = lm_path;
+    ppspeech::CTCBeamSearch decoder(opts);
+    LOG(INFO) << "Init decoder.";
 
+    decoder.InitDecoder();
     for (; !feature_reader.Done(); feature_reader.Next()) {
         string utt = feature_reader.Key();
         const kaldi::Matrix<BaseFloat> feature = feature_reader.Value();
+        LOG(INFO) << "utt: " << utt;
+
+        // feat dim
         raw_data->SetDim(feature.NumCols());
+        LOG(INFO) << "dim: " << raw_data->Dim();
+
         int32 row_idx = 0;
         int32 num_chunks = feature.NumRows() / chunk_size;
+        LOG(INFO) << "n chunks: " << num_chunks;
         for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) {
+            // feat chunk
             kaldi::Vector<kaldi::BaseFloat> feature_chunk(chunk_size *
                                                           feature.NumCols());
             for (int row_id = 0; row_id < chunk_size; ++row_id) {
-                kaldi::SubVector<kaldi::BaseFloat> tmp(feature, row_idx);
+                kaldi::SubVector<kaldi::BaseFloat> feat_one_row(feature,
+                                                                row_idx);
                 kaldi::SubVector<kaldi::BaseFloat> f_chunk_tmp(
                     feature_chunk.Data() + row_id * feature.NumCols(),
                     feature.NumCols());
-                f_chunk_tmp.CopyFromVec(tmp);
+                f_chunk_tmp.CopyFromVec(feat_one_row);
                 row_idx++;
             }
+            // feed to raw cache
             raw_data->Accept(feature_chunk);
             if (chunk_idx == num_chunks - 1) {
                 raw_data->SetFinished();
             }
+            // decode step
             decoder.AdvanceDecode(decodable);
         }
+
         std::string result;
         result = decoder.GetFinalBestPath();
         KALDI_LOG << " the result of " << utt << " is " << result;
diff --git a/speechx/examples/decoder/run.sh b/speechx/examples/decoder/run.sh
index 1e5a678c..ddda8970 100755
--- a/speechx/examples/decoder/run.sh
+++ b/speechx/examples/decoder/run.sh
@@ -28,7 +28,7 @@ cmvn=./cmvn.ark
 
 export GLOG_logtostderr=1
 
-# 3. run feat
+# 3. gen linear feat
 linear_spectrogram_main \
     --wav_rspecifier=scp:$model_dir/wav.scp \
     --feature_wspecifier=ark,t:$feat_wspecifier \
diff --git a/speechx/examples/feat/feature-mfcc-test.cc b/speechx/examples/feat/feature-mfcc-test.cc
index ae32aba9..48a9e1c2 100644
--- a/speechx/examples/feat/feature-mfcc-test.cc
+++ b/speechx/examples/feat/feature-mfcc-test.cc
@@ -41,7 +41,6 @@
 
 using namespace kaldi;
 
-
 static void UnitTestReadWave() {
     std::cout << "=== UnitTestReadWave() ===\n";
 
diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc
index a27db56f..cde78c4d 100644
--- a/speechx/examples/feat/linear_spectrogram_main.cc
+++ b/speechx/examples/feat/linear_spectrogram_main.cc
@@ -151,7 +151,7 @@ void WriteMatrix() {
         cmvn_stats(1, idx) = variance_[idx];
     }
     cmvn_stats(0, mean_.size()) = count_;
-    kaldi::WriteKaldiObject(cmvn_stats, FLAGS_cmvn_write_path, true);
+    kaldi::WriteKaldiObject(cmvn_stats, FLAGS_cmvn_write_path, false);
 }
 
 int main(int argc, char* argv[]) {
@@ -163,51 +163,56 @@ int main(int argc, char* argv[]) {
     kaldi::BaseFloatMatrixWriter feat_writer(FLAGS_feature_wspecifier);
     WriteMatrix();
 
-    // test feature linear_spectorgram: wave --> decibel_normalizer --> hanning
-    // window -->linear_spectrogram --> cmvn
+
     int32 num_done = 0, num_err = 0;
+
+    // feature pipeline: wave cache --> decibel_normalizer --> hanning
+    // window -->linear_spectrogram --> global cmvn -> feat cache
+
     // std::unique_ptr<ppspeech::FeatureExtractorInterface> data_source(new
     // ppspeech::RawDataCache());
     std::unique_ptr<ppspeech::FeatureExtractorInterface> data_source(
         new ppspeech::RawAudioCache());
 
+    ppspeech::DecibelNormalizerOptions db_norm_opt;
+    std::unique_ptr<ppspeech::FeatureExtractorInterface> db_norm(
+        new ppspeech::DecibelNormalizer(db_norm_opt, std::move(data_source)));
+
     ppspeech::LinearSpectrogramOptions opt;
     opt.frame_opts.frame_length_ms = 20;
     opt.frame_opts.frame_shift_ms = 10;
-    LOG(INFO) << "frame length (ms):" << opt.frame_opts.frame_length_ms;
-    LOG(INFO) << "frame shift (ms):" << opt.frame_opts.frame_shift_ms;
-
-    ppspeech::DecibelNormalizerOptions db_norm_opt;
-    std::unique_ptr<ppspeech::FeatureExtractorInterface> base_feature_extractor(
-        new ppspeech::DecibelNormalizer(db_norm_opt, std::move(data_source)));
+    LOG(INFO) << "frame length (ms): " << opt.frame_opts.frame_length_ms;
+    LOG(INFO) << "frame shift (ms): " << opt.frame_opts.frame_shift_ms;
 
     std::unique_ptr<ppspeech::FeatureExtractorInterface> linear_spectrogram(
-        new ppspeech::LinearSpectrogram(opt,
-                                        std::move(base_feature_extractor)));
+        new ppspeech::LinearSpectrogram(opt, std::move(db_norm)));
 
     std::unique_ptr<ppspeech::FeatureExtractorInterface> cmvn(
         new ppspeech::CMVN(FLAGS_cmvn_write_path,
                            std::move(linear_spectrogram)));
 
     ppspeech::FeatureCache feature_cache(kint16max, std::move(cmvn));
+    LOG(INFO) << "feat dim: " << feature_cache.Dim();
 
-    float streaming_chunk = 0.36;
     int sample_rate = 16000;
+    float streaming_chunk = 0.36;
     int chunk_sample_size = streaming_chunk * sample_rate;
-
-    LOG(INFO) << "sr:" << sample_rate;
-    LOG(INFO) << "chunk size (s):" << streaming_chunk;
-    LOG(INFO) << "chunk size (sample):" << chunk_sample_size;
+    LOG(INFO) << "sr: " << sample_rate;
+    LOG(INFO) << "chunk size (s): " << streaming_chunk;
+    LOG(INFO) << "chunk size (sample): " << chunk_sample_size;
 
 
     for (; !wav_reader.Done(); wav_reader.Next()) {
         std::string utt = wav_reader.Key();
         const kaldi::WaveData& wave_data = wav_reader.Value();
+        LOG(INFO) << "process utt: " << utt;
 
         int32 this_channel = 0;
         kaldi::SubVector<kaldi::BaseFloat> waveform(wave_data.Data(),
                                                     this_channel);
         int tot_samples = waveform.Dim();
+        LOG(INFO) << "wav len (sample): " << tot_samples;
+
         int sample_offset = 0;
         std::vector<kaldi::Vector<BaseFloat>> feats;
         int feature_rows = 0;
@@ -219,6 +224,7 @@ int main(int argc, char* argv[]) {
             for (int i = 0; i < cur_chunk_size; ++i) {
                 wav_chunk(i) = waveform(sample_offset + i);
             }
+
             kaldi::Vector<BaseFloat> features;
             feature_cache.Accept(wav_chunk);
             if (cur_chunk_size < chunk_sample_size) {
diff --git a/speechx/speechx/frontend/feature_cache.h b/speechx/speechx/frontend/feature_cache.h
index b6bbdf3c..f52b9b0f 100644
--- a/speechx/speechx/frontend/feature_cache.h
+++ b/speechx/speechx/frontend/feature_cache.h
@@ -28,10 +28,10 @@ class FeatureCache : public FeatureExtractorInterface {
     // Feed feats or waves
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs);
 
-    // feats dim = num_frames * feature_dim
+    // feats size = num_frames * feat_dim
     virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* feats);
 
-    // feature cache only cache feature which from base extractor
+    // feat dim
     virtual size_t Dim() const { return base_extractor_->Dim(); }
 
     virtual void SetFinished() {
diff --git a/speechx/speechx/frontend/raw_audio.h b/speechx/speechx/frontend/raw_audio.h
index ce75c137..7a28f2c9 100644
--- a/speechx/speechx/frontend/raw_audio.h
+++ b/speechx/speechx/frontend/raw_audio.h
@@ -68,9 +68,10 @@ class RawDataCache : public FeatureExtractorInterface {
         data_.Resize(0);
         return true;
     }
-    virtual size_t Dim() const { return dim_; }
+
     virtual void SetFinished() { finished_ = true; }
     virtual bool IsFinished() const { return finished_; }
+    virtual size_t Dim() const { return dim_; }
     void SetDim(int32 dim) { dim_ = dim; }
     virtual void Reset() { finished_ = true; }
 

From 143ab13679f6221b0a3757899a4c6b715ba06037 Mon Sep 17 00:00:00 2001
From: Yang Zhou <goat.zhou@qq.com>
Date: Thu, 31 Mar 2022 20:13:05 +0800
Subject: [PATCH 104/126] add decoder_test_main

---
 speechx/examples/decoder/CMakeLists.txt       |   9 ++
 speechx/examples/decoder/decoder_test_main.cc |  69 ++++++++++
 .../examples/decoder/offline_decoder_main.cc  |   1 +
 .../offline_decoder_sliding_chunk_main.cc     | 119 ++++++++++++++++++
 .../decoder/ctc_beam_search_decoder.cc        |   6 +-
 .../speechx/decoder/ctc_beam_search_decoder.h |   4 +-
 speechx/speechx/nnet/decodable.cc             |   8 +-
 7 files changed, 209 insertions(+), 7 deletions(-)
 create mode 100644 speechx/examples/decoder/decoder_test_main.cc
 create mode 100644 speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc

diff --git a/speechx/examples/decoder/CMakeLists.txt b/speechx/examples/decoder/CMakeLists.txt
index 4bd5c6cf..ded423e9 100644
--- a/speechx/examples/decoder/CMakeLists.txt
+++ b/speechx/examples/decoder/CMakeLists.txt
@@ -1,5 +1,14 @@
 cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
+add_executable(offline_decoder_sliding_chunk_main ${CMAKE_CURRENT_SOURCE_DIR}/offline_decoder_sliding_chunk_main.cc)
+target_include_directories(offline_decoder_sliding_chunk_main PRIVATE ${SPEECHX_ROOT} ${SPEECHX_ROOT}/kaldi)
+target_link_libraries(offline_decoder_sliding_chunk_main PUBLIC nnet decoder fst utils gflags glog kaldi-base kaldi-matrix kaldi-util ${DEPS})
+
 add_executable(offline_decoder_main ${CMAKE_CURRENT_SOURCE_DIR}/offline_decoder_main.cc)
 target_include_directories(offline_decoder_main PRIVATE ${SPEECHX_ROOT} ${SPEECHX_ROOT}/kaldi)
 target_link_libraries(offline_decoder_main PUBLIC nnet decoder fst utils gflags glog kaldi-base kaldi-matrix kaldi-util ${DEPS})
+
+add_executable(decoder_test_main ${CMAKE_CURRENT_SOURCE_DIR}/decoder_test_main.cc)
+target_include_directories(decoder_test_main PRIVATE ${SPEECHX_ROOT} ${SPEECHX_ROOT}/kaldi)
+target_link_libraries(decoder_test_main PUBLIC nnet decoder fst utils gflags glog kaldi-base kaldi-matrix kaldi-util ${DEPS})
+
diff --git a/speechx/examples/decoder/decoder_test_main.cc b/speechx/examples/decoder/decoder_test_main.cc
new file mode 100644
index 00000000..79fe63fc
--- /dev/null
+++ b/speechx/examples/decoder/decoder_test_main.cc
@@ -0,0 +1,69 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// todo refactor, repalce with gtest
+
+#include "base/flags.h"
+#include "base/log.h"
+#include "decoder/ctc_beam_search_decoder.h"
+#include "kaldi/util/table-types.h"
+#include "nnet/decodable.h"
+
+DEFINE_string(nnet_prob_respecifier, "", "test nnet prob rspecifier");
+DEFINE_string(dict_file, "vocab.txt", "vocabulary of lm");
+DEFINE_string(lm_path, "lm.klm", "language model");
+
+
+using kaldi::BaseFloat;
+using kaldi::Matrix;
+using std::vector;
+
+int main(int argc, char* argv[]) {
+    gflags::ParseCommandLineFlags(&argc, &argv, false);
+    google::InitGoogleLogging(argv[0]);
+
+    kaldi::SequentialBaseFloatMatrixReader likelihood_reader(
+        FLAGS_nnet_prob_respecifier);
+    std::string dict_file = FLAGS_dict_file;
+    std::string lm_path = FLAGS_lm_path;
+
+    int32 num_done = 0, num_err = 0;
+
+    ppspeech::CTCBeamSearchOptions opts;
+    opts.dict_file = dict_file;
+    opts.lm_path = lm_path;
+    ppspeech::CTCBeamSearch decoder(opts);
+
+    std::shared_ptr<ppspeech::Decodable> decodable(
+        new ppspeech::Decodable(nullptr, nullptr));
+
+    decoder.InitDecoder();
+
+    for (; !likelihood_reader.Done(); likelihood_reader.Next()) {
+        string utt = likelihood_reader.Key();
+        const kaldi::Matrix<BaseFloat> likelihood = likelihood_reader.Value();
+        decodable->Acceptlikelihood(likelihood);
+        decoder.AdvanceDecode(decodable);
+        std::string result;
+        result = decoder.GetFinalBestPath();
+        KALDI_LOG << " the result of " << utt << " is " << result;
+        decodable->Reset();
+        decoder.Reset();
+        ++num_done;
+    }
+
+    KALDI_LOG << "Done " << num_done << " utterances, " << num_err
+              << " with errors.";
+    return (num_done != 0 ? 0 : 1);
+}
diff --git a/speechx/examples/decoder/offline_decoder_main.cc b/speechx/examples/decoder/offline_decoder_main.cc
index 44127c73..63ca868b 100644
--- a/speechx/examples/decoder/offline_decoder_main.cc
+++ b/speechx/examples/decoder/offline_decoder_main.cc
@@ -52,6 +52,7 @@ int main(int argc, char* argv[]) {
     ppspeech::CTCBeamSearch decoder(opts);
 
     ppspeech::ModelOptions model_opts;
+    model_opts.cache_shape = "5-1-1024,5-1-1024";
     model_opts.model_path = model_graph;
     model_opts.params_path = model_params;
     std::shared_ptr<ppspeech::PaddleNnet> nnet(
diff --git a/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
new file mode 100644
index 00000000..ad72b772
--- /dev/null
+++ b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
@@ -0,0 +1,119 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// todo refactor, repalce with gtest
+
+#include "base/flags.h"
+#include "base/log.h"
+#include "decoder/ctc_beam_search_decoder.h"
+#include "frontend/raw_audio.h"
+#include "kaldi/util/table-types.h"
+#include "nnet/decodable.h"
+#include "nnet/paddle_nnet.h"
+
+DEFINE_string(feature_respecifier, "", "test feature rspecifier");
+DEFINE_string(model_path, "avg_1.jit.pdmodel", "paddle nnet model");
+DEFINE_string(param_path, "avg_1.jit.pdiparams", "paddle nnet model param");
+DEFINE_string(dict_file, "vocab.txt", "vocabulary of lm");
+DEFINE_string(lm_path, "lm.klm", "language model");
+
+
+using kaldi::BaseFloat;
+using kaldi::Matrix;
+using std::vector;
+
+int main(int argc, char* argv[]) {
+    gflags::ParseCommandLineFlags(&argc, &argv, false);
+    google::InitGoogleLogging(argv[0]);
+
+    kaldi::SequentialBaseFloatMatrixReader feature_reader(
+        FLAGS_feature_respecifier);
+    std::string model_graph = FLAGS_model_path;
+    std::string model_params = FLAGS_param_path;
+    std::string dict_file = FLAGS_dict_file;
+    std::string lm_path = FLAGS_lm_path;
+
+    int32 num_done = 0, num_err = 0;
+
+    ppspeech::CTCBeamSearchOptions opts;
+    opts.dict_file = dict_file;
+    opts.lm_path = lm_path;
+    ppspeech::CTCBeamSearch decoder(opts);
+
+    ppspeech::ModelOptions model_opts;
+    model_opts.model_path = model_graph;
+    model_opts.params_path = model_params;
+    model_opts.cache_shape = "5-1-1024,5-1-1024";
+    std::shared_ptr<ppspeech::PaddleNnet> nnet(
+        new ppspeech::PaddleNnet(model_opts));
+    std::shared_ptr<ppspeech::RawDataCache> raw_data(
+        new ppspeech::RawDataCache());
+    std::shared_ptr<ppspeech::Decodable> decodable(
+        new ppspeech::Decodable(nnet, raw_data));
+
+    int32 chunk_size = 7;
+    int32 chunk_stride = 4;
+    int32 receptive_field_length = 7;
+    decoder.InitDecoder();
+
+    for (; !feature_reader.Done(); feature_reader.Next()) {
+        string utt = feature_reader.Key();
+        kaldi::Matrix<BaseFloat> feature = feature_reader.Value();
+        raw_data->SetDim(feature.NumCols());
+        int32 row_idx = 0;
+        int32 padding_len = 0;
+        int32 ori_feature_len = feature.NumRows(); 
+        if ( (feature.NumRows() - chunk_size) % chunk_stride != 0) {
+            padding_len = chunk_stride - (feature.NumRows() - chunk_size) % chunk_stride;
+            feature.Resize(feature.NumRows() + padding_len, feature.NumCols(), kaldi::kCopyData);
+        }
+        int32 num_chunks = (feature.NumRows() - chunk_size) / chunk_stride + 1;
+        for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) {
+            kaldi::Vector<kaldi::BaseFloat> feature_chunk(chunk_size *
+                                                          feature.NumCols());
+            int32 feature_chunk_size = 0; 
+            if ( ori_feature_len > chunk_idx * chunk_stride) {
+                feature_chunk_size = std::min(ori_feature_len - chunk_idx * chunk_stride, chunk_size);
+            }
+            if (feature_chunk_size < receptive_field_length) break;
+
+            int32 start = chunk_idx * chunk_stride;
+            int32 end = start + chunk_size;
+
+            for (int row_id = 0; row_id < chunk_size; ++row_id) {
+                kaldi::SubVector<kaldi::BaseFloat> tmp(feature, start);
+                kaldi::SubVector<kaldi::BaseFloat> f_chunk_tmp(
+                    feature_chunk.Data() + row_id * feature.NumCols(),
+                    feature.NumCols());
+                f_chunk_tmp.CopyFromVec(tmp);
+                ++start;
+            }
+            raw_data->Accept(feature_chunk);
+            if (chunk_idx == num_chunks - 1) {
+                raw_data->SetFinished();
+            }
+            decoder.AdvanceDecode(decodable);
+        }
+        std::string result;
+        result = decoder.GetFinalBestPath();
+        KALDI_LOG << " the result of " << utt << " is " << result;
+        decodable->Reset();
+        decoder.Reset();
+        ++num_done;
+    }
+
+    KALDI_LOG << "Done " << num_done << " utterances, " << num_err
+              << " with errors.";
+    return (num_done != 0 ? 0 : 1);
+}
diff --git a/speechx/speechx/decoder/ctc_beam_search_decoder.cc b/speechx/speechx/decoder/ctc_beam_search_decoder.cc
index 84f1453c..5d7a4f77 100644
--- a/speechx/speechx/decoder/ctc_beam_search_decoder.cc
+++ b/speechx/speechx/decoder/ctc_beam_search_decoder.cc
@@ -38,8 +38,10 @@ CTCBeamSearch::CTCBeamSearch(const CTCBeamSearchOptions& opts)
               << vocabulary_.size();
 
     LOG(INFO) << "language model path: " << opts_.lm_path;
-    init_ext_scorer_ = std::make_shared<Scorer>(
-        opts_.alpha, opts_.beta, opts_.lm_path, vocabulary_);
+    if (opts_.lm_path != "") {
+        init_ext_scorer_ = std::make_shared<Scorer>(
+            opts_.alpha, opts_.beta, opts_.lm_path, vocabulary_);
+    }
 
     blank_id_ = 0;
     auto it = std::find(vocabulary_.begin(), vocabulary_.end(), " ");
diff --git a/speechx/speechx/decoder/ctc_beam_search_decoder.h b/speechx/speechx/decoder/ctc_beam_search_decoder.h
index 451f33c0..1387eee7 100644
--- a/speechx/speechx/decoder/ctc_beam_search_decoder.h
+++ b/speechx/speechx/decoder/ctc_beam_search_decoder.h
@@ -33,13 +33,13 @@ struct CTCBeamSearchOptions {
     int num_proc_bsearch;
     CTCBeamSearchOptions()
         : dict_file("vocab.txt"),
-          lm_path("lm.klm"),
+          lm_path(""),
           alpha(1.9f),
           beta(5.0),
           beam_size(300),
           cutoff_prob(0.99f),
           cutoff_top_n(40),
-          num_proc_bsearch(0) {}
+          num_proc_bsearch(10) {}
 
     void Register(kaldi::OptionsItf* opts) {
         opts->Register("dict", &dict_file, "dict file ");
diff --git a/speechx/speechx/nnet/decodable.cc b/speechx/speechx/nnet/decodable.cc
index 6c0909ca..cd72bf76 100644
--- a/speechx/speechx/nnet/decodable.cc
+++ b/speechx/speechx/nnet/decodable.cc
@@ -26,6 +26,7 @@ Decodable::Decodable(const std::shared_ptr<NnetInterface>& nnet,
     : frontend_(frontend), nnet_(nnet), frame_offset_(0), frames_ready_(0) {}
 
 void Decodable::Acceptlikelihood(const Matrix<BaseFloat>& likelihood) {
+    nnet_cache_ = likelihood;
     frames_ready_ += likelihood.NumRows();
 }
 
@@ -53,7 +54,7 @@ bool Decodable::EnsureFrameHaveComputed(int32 frame) {
 
 bool Decodable::AdvanceChunk() {
     Vector<BaseFloat> features;
-    if (frontend_->Read(&features) == false) {
+    if (frontend_ == NULL || frontend_->Read(&features) == false) {
         return false;
     }
     int32 nnet_dim = 0;
@@ -77,10 +78,11 @@ bool Decodable::FrameLogLikelihood(int32 frame, vector<BaseFloat>* likelihood) {
 }
 
 void Decodable::Reset() {
-    frontend_->Reset();
-    nnet_->Reset();
+    if (frontend_ != nullptr) frontend_->Reset();
+    if (nnet_ != nullptr) nnet_->Reset();
     frame_offset_ = 0;
     frames_ready_ = 0;
+    nnet_cache_.Resize(0,0);
 }
 
 }  // namespace ppspeech
\ No newline at end of file

From 4d7cd0e0638818bef61ebc52e535f545be43d590 Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Thu, 31 Mar 2022 12:57:51 +0000
Subject: [PATCH 105/126] add streaming synthesize, test=tts

---
 .../csmsc/tts3/local/synthesize_streaming.sh  |  18 +-
 paddlespeech/t2s/exps/synthesize_streaming.py | 269 ++++++++++++++++++
 .../t2s/models/fastspeech2/fastspeech2.py     |  49 +++-
 .../t2s/modules/transformer/encoder.py        |   4 -
 4 files changed, 316 insertions(+), 24 deletions(-)
 create mode 100644 paddlespeech/t2s/exps/synthesize_streaming.py

diff --git a/examples/csmsc/tts3/local/synthesize_streaming.sh b/examples/csmsc/tts3/local/synthesize_streaming.sh
index 69bb22df..7606c238 100755
--- a/examples/csmsc/tts3/local/synthesize_streaming.sh
+++ b/examples/csmsc/tts3/local/synthesize_streaming.sh
@@ -22,9 +22,9 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
         --voc_stat=pwg_baker_ckpt_0.4/pwg_stats.npy \
         --lang=zh \
         --text=${BIN_DIR}/../sentences.txt \
-        --output_dir=${train_output_path}/test_e2e \
+        --output_dir=${train_output_path}/test_e2e_streaming \
         --phones_dict=dump/phone_id_map.txt \
-        --inference_dir=${train_output_path}/inference
+        --am_streaming=True
 fi
 
 # for more GAN Vocoders
@@ -43,9 +43,9 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         --voc_stat=mb_melgan_csmsc_ckpt_0.1.1/feats_stats.npy \
         --lang=zh \
         --text=${BIN_DIR}/../sentences.txt \
-        --output_dir=${train_output_path}/test_e2e \
+        --output_dir=${train_output_path}/test_e2e_streaming \
         --phones_dict=dump/phone_id_map.txt \
-        --inference_dir=${train_output_path}/inference
+        --am_streaming=True
 fi
 
 # the pretrained models haven't release now
@@ -65,9 +65,9 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
         --voc_stat=style_melgan_csmsc_ckpt_0.1.1/feats_stats.npy \
         --lang=zh \
         --text=${BIN_DIR}/../sentences.txt \
-        --output_dir=${train_output_path}/test_e2e \
-        --phones_dict=dump/phone_id_map.txt
-        # --inference_dir=${train_output_path}/inference
+        --output_dir=${train_output_path}/test_e2e_streaming \
+        --phones_dict=dump/phone_id_map.txt \
+        --am_streaming=True
 fi
 
 # hifigan
@@ -86,7 +86,7 @@ if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
         --voc_stat=hifigan_csmsc_ckpt_0.1.1/feats_stats.npy \
         --lang=zh \
         --text=${BIN_DIR}/../sentences.txt \
-        --output_dir=${train_output_path}/test_e2e \
+        --output_dir=${train_output_path}/test_e2e_streaming \
         --phones_dict=dump/phone_id_map.txt \
-        --inference_dir=${train_output_path}/inference
+        --am_streaming=True
 fi
diff --git a/paddlespeech/t2s/exps/synthesize_streaming.py b/paddlespeech/t2s/exps/synthesize_streaming.py
new file mode 100644
index 00000000..62915539
--- /dev/null
+++ b/paddlespeech/t2s/exps/synthesize_streaming.py
@@ -0,0 +1,269 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import math
+from pathlib import Path
+
+import numpy as np
+import paddle
+import soundfile as sf
+import yaml
+from timer import timer
+from yacs.config import CfgNode
+
+from paddlespeech.s2t.utils.dynamic_import import dynamic_import
+from paddlespeech.t2s.exps.syn_utils import get_frontend
+from paddlespeech.t2s.exps.syn_utils import get_sentences
+from paddlespeech.t2s.exps.syn_utils import get_voc_inference
+from paddlespeech.t2s.exps.syn_utils import model_alias
+from paddlespeech.t2s.utils import str2bool
+
+
+def denorm(data, mean, std):
+    return data * std + mean
+
+
+def get_chunks(data, chunk_size, pad_size):
+    data_len = data.shape[1]
+    chunks = []
+    n = math.ceil(data_len / chunk_size)
+    for i in range(n):
+        start = max(0, i * chunk_size - pad_size)
+        end = min((i + 1) * chunk_size + pad_size, data_len)
+        chunks.append(data[:, start:end, :])
+    return chunks
+
+
+def evaluate(args):
+
+    # Init body.
+    with open(args.am_config) as f:
+        am_config = CfgNode(yaml.safe_load(f))
+    with open(args.voc_config) as f:
+        voc_config = CfgNode(yaml.safe_load(f))
+
+    print("========Args========")
+    print(yaml.safe_dump(vars(args)))
+    print("========Config========")
+    print(am_config)
+    print(voc_config)
+
+    sentences = get_sentences(args)
+
+    # frontend
+    frontend = get_frontend(args)
+
+    with open(args.phones_dict, "r") as f:
+        phn_id = [line.strip().split() for line in f.readlines()]
+    vocab_size = len(phn_id)
+    print("vocab_size:", vocab_size)
+
+    # acoustic model, only support fastspeech2 here now!
+    # am_inference, am_name, am_dataset = get_am_inference(args, am_config)
+    # model: {model_name}_{dataset}
+    am_name = args.am[:args.am.rindex('_')]
+    am_dataset = args.am[args.am.rindex('_') + 1:]
+    odim = am_config.n_mels
+
+    am_class = dynamic_import(am_name, model_alias)
+    am = am_class(idim=vocab_size, odim=odim, **am_config["model"])
+    am.set_state_dict(paddle.load(args.am_ckpt)["main_params"])
+    am.eval()
+    am_mu, am_std = np.load(args.am_stat)
+    am_mu = paddle.to_tensor(am_mu)
+    am_std = paddle.to_tensor(am_std)
+
+    # vocoder
+    voc_inference = get_voc_inference(args, voc_config)
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    merge_sentences = True
+
+    N = 0
+    T = 0
+    chunk_size = 42
+    pad_size = 12
+
+    for utt_id, sentence in sentences:
+        with timer() as t:
+            get_tone_ids = False
+
+            if args.lang == 'zh':
+                input_ids = frontend.get_input_ids(
+                    sentence,
+                    merge_sentences=merge_sentences,
+                    get_tone_ids=get_tone_ids)
+
+                phone_ids = input_ids["phone_ids"]
+            else:
+                print("lang should in be 'zh' here!")
+            # merge_sentences=False here, so we only use the first item of phone_ids
+            phone_ids = phone_ids[0]
+            with paddle.no_grad():
+                # acoustic model
+                orig_hs, h_masks = am.encoder_infer(phone_ids)
+
+                if args.am_streaming:
+                    hss = get_chunks(orig_hs, chunk_size, pad_size)
+                    chunk_num = len(hss)
+                    mel_list = []
+                    for i, hs in enumerate(hss):
+                        before_outs, _ = am.decoder(hs)
+                        after_outs = before_outs + am.postnet(
+                            before_outs.transpose((0, 2, 1))).transpose(
+                                (0, 2, 1))
+                        normalized_mel = after_outs[0]
+                        sub_mel = denorm(normalized_mel, am_mu, am_std)
+                        # clip output part of pad
+                        if i == 0:
+                            sub_mel = sub_mel[:-pad_size]
+                        elif i == chunk_num - 1:
+                            # 最后一块的右侧一定没有 pad 够
+                            sub_mel = sub_mel[pad_size:]
+                        else:
+                            # 倒数几块的右侧也可能没有 pad 够
+                            sub_mel = sub_mel[pad_size:(chunk_size + pad_size) -
+                                              sub_mel.shape[0]]
+                        mel_list.append(sub_mel)
+                    mel = paddle.concat(mel_list, axis=0)
+
+                else:
+                    before_outs, _ = am.decoder(orig_hs)
+                    after_outs = before_outs + am.postnet(
+                        before_outs.transpose((0, 2, 1))).transpose((0, 2, 1))
+                    normalized_mel = after_outs[0]
+                    mel = denorm(normalized_mel, am_mu, am_std)
+
+                # vocoder
+                wav = voc_inference(mel)
+
+        wav = wav.numpy()
+        N += wav.size
+        T += t.elapse
+        speed = wav.size / t.elapse
+        rtf = am_config.fs / speed
+        print(
+            f"{utt_id}, mel: {mel.shape}, wave: {wav.shape}, time: {t.elapse}s, Hz: {speed}, RTF: {rtf}."
+        )
+        sf.write(
+            str(output_dir / (utt_id + ".wav")), wav, samplerate=am_config.fs)
+        print(f"{utt_id} done!")
+    print(f"generation speed: {N / T}Hz, RTF: {am_config.fs / (N / T) }")
+
+
+def parse_args():
+    # parse args and config and redirect to train_sp
+    parser = argparse.ArgumentParser(
+        description="Synthesize with acoustic model & vocoder")
+    # acoustic model
+    parser.add_argument(
+        '--am',
+        type=str,
+        default='fastspeech2_csmsc',
+        choices=['fastspeech2_csmsc'],
+        help='Choose acoustic model type of tts task.')
+    parser.add_argument(
+        '--am_config',
+        type=str,
+        default=None,
+        help='Config of acoustic model. Use deault config when it is None.')
+    parser.add_argument(
+        '--am_ckpt',
+        type=str,
+        default=None,
+        help='Checkpoint file of acoustic model.')
+    parser.add_argument(
+        "--am_stat",
+        type=str,
+        default=None,
+        help="mean and standard deviation used to normalize spectrogram when training acoustic model."
+    )
+    parser.add_argument(
+        "--phones_dict", type=str, default=None, help="phone vocabulary file.")
+    parser.add_argument(
+        "--tones_dict", type=str, default=None, help="tone vocabulary file.")
+
+    # vocoder
+    parser.add_argument(
+        '--voc',
+        type=str,
+        default='pwgan_csmsc',
+        choices=[
+            'pwgan_csmsc',
+            'pwgan_ljspeech',
+            'pwgan_aishell3',
+            'pwgan_vctk',
+            'mb_melgan_csmsc',
+            'style_melgan_csmsc',
+            'hifigan_csmsc',
+            'hifigan_ljspeech',
+            'hifigan_aishell3',
+            'hifigan_vctk',
+            'wavernn_csmsc',
+        ],
+        help='Choose vocoder type of tts task.')
+    parser.add_argument(
+        '--voc_config',
+        type=str,
+        default=None,
+        help='Config of voc. Use deault config when it is None.')
+    parser.add_argument(
+        '--voc_ckpt', type=str, default=None, help='Checkpoint file of voc.')
+    parser.add_argument(
+        "--voc_stat",
+        type=str,
+        default=None,
+        help="mean and standard deviation used to normalize spectrogram when training voc."
+    )
+    # other
+    parser.add_argument(
+        '--lang',
+        type=str,
+        default='zh',
+        help='Choose model language. zh or en')
+
+    parser.add_argument(
+        "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.")
+    parser.add_argument(
+        "--text",
+        type=str,
+        help="text to synthesize, a 'utt_id sentence' pair per line.")
+
+    parser.add_argument(
+        "--am_streaming",
+        type=str2bool,
+        default=False,
+        help="whether use streaming acoustic model")
+    parser.add_argument("--output_dir", type=str, help="output dir.")
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+
+    if args.ngpu == 0:
+        paddle.set_device("cpu")
+    elif args.ngpu > 0:
+        paddle.set_device("gpu")
+    else:
+        print("ngpu should >= 0 !")
+
+    evaluate(args)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/paddlespeech/t2s/models/fastspeech2/fastspeech2.py b/paddlespeech/t2s/models/fastspeech2/fastspeech2.py
index 1c805051..c2f1e218 100644
--- a/paddlespeech/t2s/models/fastspeech2/fastspeech2.py
+++ b/paddlespeech/t2s/models/fastspeech2/fastspeech2.py
@@ -509,6 +509,7 @@ class FastSpeech2(nn.Layer):
                  ps: paddle.Tensor=None,
                  es: paddle.Tensor=None,
                  is_inference: bool=False,
+                 return_after_enc=False,
                  alpha: float=1.0,
                  spk_emb=None,
                  spk_id=None,
@@ -589,8 +590,10 @@ class FastSpeech2(nn.Layer):
             h_masks = self._source_mask(olens_in)
         else:
             h_masks = None
-        # (B, Lmax, adim)
 
+        if return_after_enc:
+            return hs, h_masks
+        # (B, Lmax, adim)
         zs, _ = self.decoder(hs, h_masks)
         # (B, Lmax, odim)
         if self.decoder_type == 'cnndecoder':
@@ -608,10 +611,42 @@ class FastSpeech2(nn.Layer):
 
         return before_outs, after_outs, d_outs, p_outs, e_outs
 
+    def encoder_infer(
+            self,
+            text: paddle.Tensor,
+            alpha: float=1.0,
+            spk_emb=None,
+            spk_id=None,
+            tone_id=None,
+    ) -> Tuple[paddle.Tensor, paddle.Tensor, paddle.Tensor]:
+        # input of embedding must be int64
+        x = paddle.cast(text, 'int64')
+        # setup batch axis
+        ilens = paddle.shape(x)[0]
+
+        xs = x.unsqueeze(0)
+
+        if spk_emb is not None:
+            spk_emb = spk_emb.unsqueeze(0)
+
+        if tone_id is not None:
+            tone_id = tone_id.unsqueeze(0)
+
+        # (1, L, odim)
+        hs, h_masks = self._forward(
+            xs,
+            ilens,
+            is_inference=True,
+            return_after_enc=True,
+            alpha=alpha,
+            spk_emb=spk_emb,
+            spk_id=spk_id,
+            tone_id=tone_id)
+        return hs, h_masks
+
     def inference(
             self,
             text: paddle.Tensor,
-            speech: paddle.Tensor=None,
             durations: paddle.Tensor=None,
             pitch: paddle.Tensor=None,
             energy: paddle.Tensor=None,
@@ -625,7 +660,6 @@ class FastSpeech2(nn.Layer):
 
         Args:
             text(Tensor(int64)): Input sequence of characters (T,).
-            speech(Tensor, optional): Feature sequence to extract style (N, idim).
             durations(Tensor, optional (int64)): Groundtruth of duration (T,).
             pitch(Tensor, optional): Groundtruth of token-averaged pitch (T, 1).
             energy(Tensor, optional): Groundtruth of token-averaged energy (T, 1).
@@ -642,15 +676,11 @@ class FastSpeech2(nn.Layer):
         """
         # input of embedding must be int64
         x = paddle.cast(text, 'int64')
-        y = speech
         d, p, e = durations, pitch, energy
         # setup batch axis
         ilens = paddle.shape(x)[0]
 
-        xs, ys = x.unsqueeze(0), None
-
-        if y is not None:
-            ys = y.unsqueeze(0)
+        xs = x.unsqueeze(0)
 
         if spk_emb is not None:
             spk_emb = spk_emb.unsqueeze(0)
@@ -668,7 +698,6 @@ class FastSpeech2(nn.Layer):
             _, outs, d_outs, p_outs, e_outs = self._forward(
                 xs,
                 ilens,
-                ys,
                 ds=ds,
                 ps=ps,
                 es=es,
@@ -681,7 +710,6 @@ class FastSpeech2(nn.Layer):
             _, outs, d_outs, p_outs, e_outs = self._forward(
                 xs,
                 ilens,
-                ys,
                 is_inference=True,
                 alpha=alpha,
                 spk_emb=spk_emb,
@@ -829,7 +857,6 @@ class StyleFastSpeech2Inference(FastSpeech2Inference):
 
         Args:
             text(Tensor(int64)): Input sequence of characters (T,).
-            speech(Tensor, optional): Feature sequence to extract style (N, idim).
             durations(paddle.Tensor/np.ndarray, optional (int64)): Groundtruth of duration (T,), this will overwrite the set of durations_scale and durations_bias
             durations_scale(int/float, optional): 
             durations_bias(int/float, optional): 
diff --git a/paddlespeech/t2s/modules/transformer/encoder.py b/paddlespeech/t2s/modules/transformer/encoder.py
index 25a11ff6..f6420282 100644
--- a/paddlespeech/t2s/modules/transformer/encoder.py
+++ b/paddlespeech/t2s/modules/transformer/encoder.py
@@ -587,7 +587,6 @@ class CNNDecoder(nn.Layer):
         Returns:
             Tensor: Output tensor (#batch, time, odim).
         """
-        # print("input.shape in CNNDecoder:",xs.shape)
         # exchange the temporal dimension and the feature dimension
         xs = xs.transpose([0, 2, 1])
         if masks is not None:
@@ -603,7 +602,6 @@ class CNNDecoder(nn.Layer):
         if masks is not None:
             outputs = outputs * masks
         outputs = outputs.transpose([0, 2, 1])
-        # print("outputs.shape in CNNDecoder:",outputs.shape)
         return outputs, masks
 
 
@@ -636,7 +634,6 @@ class CNNPostnet(nn.Layer):
         Returns:
             Tensor: Output tensor (#batch, odim, time).
         """
-        # print("xs.shape in CNNPostnet:",xs.shape)
         for layer in self.residual_blocks:
             outputs = layer(xs)
             if masks is not None:
@@ -646,5 +643,4 @@ class CNNPostnet(nn.Layer):
         outputs = self.conv1d(outputs)
         if masks is not None:
             outputs = outputs * masks
-        # print("outputs.shape in CNNPostnet:",outputs.shape)
         return outputs

From 760e5d4418702f88f9dbe3dfc6bedc22e99b7a03 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 31 Mar 2022 12:24:15 +0000
Subject: [PATCH 106/126] refactor cache

---
 .../examples/decoder/offline_decoder_main.cc  |  5 +-
 .../examples/feat/linear_spectrogram_main.cc  |  7 ++-
 speechx/speechx/frontend/CMakeLists.txt       |  4 +-
 .../frontend/{raw_audio.cc => audio_cache.cc} | 43 +++++++------
 speechx/speechx/frontend/audio_cache.h        | 61 +++++++++++++++++++
 .../frontend/{raw_audio.h => data_cache.h}    | 46 +++-----------
 6 files changed, 101 insertions(+), 65 deletions(-)
 rename speechx/speechx/frontend/{raw_audio.cc => audio_cache.cc} (64%)
 create mode 100644 speechx/speechx/frontend/audio_cache.h
 rename speechx/speechx/frontend/{raw_audio.h => data_cache.h} (54%)

diff --git a/speechx/examples/decoder/offline_decoder_main.cc b/speechx/examples/decoder/offline_decoder_main.cc
index 3a858ad1..c73d5968 100644
--- a/speechx/examples/decoder/offline_decoder_main.cc
+++ b/speechx/examples/decoder/offline_decoder_main.cc
@@ -17,7 +17,7 @@
 #include "base/flags.h"
 #include "base/log.h"
 #include "decoder/ctc_beam_search_decoder.h"
-#include "frontend/raw_audio.h"
+#include "frontend/data_cache.h"
 #include "kaldi/util/table-types.h"
 #include "nnet/decodable.h"
 #include "nnet/paddle_nnet.h"
@@ -60,8 +60,7 @@ int main(int argc, char* argv[]) {
     model_opts.params_path = model_params;
     std::shared_ptr<ppspeech::PaddleNnet> nnet(
         new ppspeech::PaddleNnet(model_opts));
-    std::shared_ptr<ppspeech::RawDataCache> raw_data(
-        new ppspeech::RawDataCache());
+    std::shared_ptr<ppspeech::DataCache> raw_data(new ppspeech::DataCache());
     std::shared_ptr<ppspeech::Decodable> decodable(
         new ppspeech::Decodable(nnet, raw_data));
     LOG(INFO) << "Init decodeable.";
diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc
index cde78c4d..e1f0a895 100644
--- a/speechx/examples/feat/linear_spectrogram_main.cc
+++ b/speechx/examples/feat/linear_spectrogram_main.cc
@@ -17,10 +17,11 @@
 #include "frontend/linear_spectrogram.h"
 #include "base/flags.h"
 #include "base/log.h"
+#include "frontend/audio_cache.h"
+#include "frontend/data_cache.h"
 #include "frontend/feature_cache.h"
 #include "frontend/feature_extractor_interface.h"
 #include "frontend/normalizer.h"
-#include "frontend/raw_audio.h"
 #include "kaldi/feat/wave-reader.h"
 #include "kaldi/util/kaldi-io.h"
 #include "kaldi/util/table-types.h"
@@ -170,9 +171,9 @@ int main(int argc, char* argv[]) {
     // window -->linear_spectrogram --> global cmvn -> feat cache
 
     // std::unique_ptr<ppspeech::FeatureExtractorInterface> data_source(new
-    // ppspeech::RawDataCache());
+    // ppspeech::DataCache());
     std::unique_ptr<ppspeech::FeatureExtractorInterface> data_source(
-        new ppspeech::RawAudioCache());
+        new ppspeech::AudioCache());
 
     ppspeech::DecibelNormalizerOptions db_norm_opt;
     std::unique_ptr<ppspeech::FeatureExtractorInterface> db_norm(
diff --git a/speechx/speechx/frontend/CMakeLists.txt b/speechx/speechx/frontend/CMakeLists.txt
index 44ca52cd..d0ec008e 100644
--- a/speechx/speechx/frontend/CMakeLists.txt
+++ b/speechx/speechx/frontend/CMakeLists.txt
@@ -3,8 +3,8 @@ project(frontend)
 add_library(frontend STATIC
   normalizer.cc
   linear_spectrogram.cc
-  raw_audio.cc
+  audio_cache.cc
   feature_cache.cc
 )
 
-target_link_libraries(frontend PUBLIC kaldi-matrix)
+target_link_libraries(frontend PUBLIC kaldi-matrix)
\ No newline at end of file
diff --git a/speechx/speechx/frontend/raw_audio.cc b/speechx/speechx/frontend/audio_cache.cc
similarity index 64%
rename from speechx/speechx/frontend/raw_audio.cc
rename to speechx/speechx/frontend/audio_cache.cc
index 21f64362..d44ed592 100644
--- a/speechx/speechx/frontend/raw_audio.cc
+++ b/speechx/speechx/frontend/audio_cache.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "frontend/raw_audio.h"
+#include "frontend/audio_cache.h"
 #include "kaldi/base/timer.h"
 
 namespace ppspeech {
@@ -21,38 +21,43 @@ using kaldi::BaseFloat;
 using kaldi::VectorBase;
 using kaldi::Vector;
 
-RawAudioCache::RawAudioCache(int buffer_size)
-    : finished_(false), data_length_(0), start_(0), timeout_(1) {
-    ring_buffer_.resize(buffer_size);
+AudioCache::AudioCache(int buffer_size)
+    : finished_(false),
+      capacity_(buffer_size),
+      size_(0),
+      offset_(0),
+      timeout_(1) {
+    ring_buffer_.resize(capacity_);
 }
 
-void RawAudioCache::Accept(const VectorBase<BaseFloat>& waves) {
+void AudioCache::Accept(const VectorBase<BaseFloat>& waves) {
     std::unique_lock<std::mutex> lock(mutex_);
-    while (data_length_ + waves.Dim() > ring_buffer_.size()) {
+    while (size_ + waves.Dim() > ring_buffer_.size()) {
         ready_feed_condition_.wait(lock);
     }
     for (size_t idx = 0; idx < waves.Dim(); ++idx) {
-        int32 buffer_idx = (idx + start_) % ring_buffer_.size();
+        int32 buffer_idx = (idx + offset_) % ring_buffer_.size();
         ring_buffer_[buffer_idx] = waves(idx);
     }
-    data_length_ += waves.Dim();
+    size_ += waves.Dim();
 }
 
-bool RawAudioCache::Read(Vector<BaseFloat>* waves) {
+bool AudioCache::Read(Vector<BaseFloat>* waves) {
     size_t chunk_size = waves->Dim();
     kaldi::Timer timer;
     std::unique_lock<std::mutex> lock(mutex_);
-    while (chunk_size > data_length_) {
+    while (chunk_size > size_) {
         // when audio is empty and no more data feed
-        // ready_read_condition will block in dead lock. so replace with
-        // timeout_
+        // ready_read_condition will block in dead lock,
+        // so replace with timeout_
         // ready_read_condition_.wait(lock);
         int32 elapsed = static_cast<int32>(timer.Elapsed() * 1000);
         if (elapsed > timeout_) {
-            if (finished_ == true) {  // read last chunk data
+            if (finished_ == true) {
+                // read last chunk data
                 break;
             }
-            if (chunk_size > data_length_) {
+            if (chunk_size > size_) {
                 return false;
             }
         }
@@ -60,17 +65,17 @@ bool RawAudioCache::Read(Vector<BaseFloat>* waves) {
     }
 
     // read last chunk data
-    if (chunk_size > data_length_) {
-        chunk_size = data_length_;
+    if (chunk_size > size_) {
+        chunk_size = size_;
         waves->Resize(chunk_size);
     }
 
     for (size_t idx = 0; idx < chunk_size; ++idx) {
-        int buff_idx = (start_ + idx) % ring_buffer_.size();
+        int buff_idx = (offset_ + idx) % ring_buffer_.size();
         waves->Data()[idx] = ring_buffer_[buff_idx];
     }
-    data_length_ -= chunk_size;
-    start_ = (start_ + chunk_size) % ring_buffer_.size();
+    size_ -= chunk_size;
+    offset_ = (offset_ + chunk_size) % ring_buffer_.size();
     ready_feed_condition_.notify_one();
     return true;
 }
diff --git a/speechx/speechx/frontend/audio_cache.h b/speechx/speechx/frontend/audio_cache.h
new file mode 100644
index 00000000..b6c82c69
--- /dev/null
+++ b/speechx/speechx/frontend/audio_cache.h
@@ -0,0 +1,61 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#pragma once
+
+#include "base/common.h"
+#include "frontend/feature_extractor_interface.h"
+
+namespace ppspeech {
+
+// waves cache
+class AudioCache : public FeatureExtractorInterface {
+  public:
+    explicit AudioCache(int buffer_size = kint16max);
+
+    virtual void Accept(const kaldi::VectorBase<BaseFloat>& waves);
+
+    virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* waves);
+
+    // the audio dim is 1, one sample
+    virtual size_t Dim() const { return 1; }
+
+    virtual void SetFinished() {
+        std::lock_guard<std::mutex> lock(mutex_);
+        finished_ = true;
+    }
+
+    virtual bool IsFinished() const { return finished_; }
+
+    virtual void Reset() {
+        offset_ = 0;
+        size_ = 0;
+        finished_ = false;
+    }
+
+  private:
+    std::vector<kaldi::BaseFloat> ring_buffer_;
+    size_t offset_;    // offset in ring_buffer_
+    size_t size_;      // samples in ring_buffer_ now
+    size_t capacity_;  // capacity of ring_buffer_
+    bool finished_;    // reach audio end
+    mutable std::mutex mutex_;
+    std::condition_variable ready_feed_condition_;
+    kaldi::int32 timeout_;  // millisecond
+
+    DISALLOW_COPY_AND_ASSIGN(AudioCache);
+};
+
+}  // namespace ppspeech
diff --git a/speechx/speechx/frontend/raw_audio.h b/speechx/speechx/frontend/data_cache.h
similarity index 54%
rename from speechx/speechx/frontend/raw_audio.h
rename to speechx/speechx/frontend/data_cache.h
index 7a28f2c9..dea51d76 100644
--- a/speechx/speechx/frontend/raw_audio.h
+++ b/speechx/speechx/frontend/data_cache.h
@@ -15,51 +15,22 @@
 
 #pragma once
 
+
 #include "base/common.h"
 #include "frontend/feature_extractor_interface.h"
 
-#pragma once
 
 namespace ppspeech {
-
-class RawAudioCache : public FeatureExtractorInterface {
+// A data source for testing different frontend module.
+// It accepts waves or feats.
+class DataCache : public FeatureExtractorInterface {
   public:
-    explicit RawAudioCache(int buffer_size = kint16max);
-    virtual void Accept(const kaldi::VectorBase<BaseFloat>& waves);
-    virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* waves);
-    // the audio dim is 1
-    virtual size_t Dim() const { return 1; }
-    virtual void SetFinished() {
-        std::lock_guard<std::mutex> lock(mutex_);
-        finished_ = true;
-    }
-    virtual bool IsFinished() const { return finished_; }
-    virtual void Reset() {
-        start_ = 0;
-        data_length_ = 0;
-        finished_ = false;
-    }
-
-  private:
-    std::vector<kaldi::BaseFloat> ring_buffer_;
-    size_t start_;
-    size_t data_length_;
-    bool finished_;
-    mutable std::mutex mutex_;
-    std::condition_variable ready_feed_condition_;
-    kaldi::int32 timeout_;
-
-    DISALLOW_COPY_AND_ASSIGN(RawAudioCache);
-};
+    explicit DataCache() { finished_ = false; }
 
-// it is a datasource for testing different frontend module.
-// it accepts waves or feats.
-class RawDataCache : public FeatureExtractorInterface {
-  public:
-    explicit RawDataCache() { finished_ = false; }
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs) {
         data_ = inputs;
     }
+
     virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* feats) {
         if (data_.Dim() == 0) {
             return false;
@@ -80,7 +51,6 @@ class RawDataCache : public FeatureExtractorInterface {
     bool finished_;
     int32 dim_;
 
-    DISALLOW_COPY_AND_ASSIGN(RawDataCache);
+    DISALLOW_COPY_AND_ASSIGN(DataCache);
 };
-
-}  // namespace ppspeech
+}
\ No newline at end of file

From cb39777a60e53cfbac4dd2382a28ce7ce10c8cef Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Thu, 31 Mar 2022 12:24:23 +0000
Subject: [PATCH 107/126] format code

---
 paddlespeech/s2t/frontend/speech.py           |  7 ++++-
 paddlespeech/server/bin/main.py               |  2 +-
 .../server/engine/asr/online/asr_engine.py    | 27 +++++++++----------
 paddlespeech/server/utils/buffer.py           |  8 +++---
 4 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/paddlespeech/s2t/frontend/speech.py b/paddlespeech/s2t/frontend/speech.py
index 0340831a..96997104 100644
--- a/paddlespeech/s2t/frontend/speech.py
+++ b/paddlespeech/s2t/frontend/speech.py
@@ -108,7 +108,12 @@ class SpeechSegment(AudioSegment):
                    token_ids)
 
     @classmethod
-    def from_pcm(cls, samples, sample_rate, transcript, tokens=None, token_ids=None):
+    def from_pcm(cls,
+                 samples,
+                 sample_rate,
+                 transcript,
+                 tokens=None,
+                 token_ids=None):
         """Create speech segment from pcm on online mode 
         Args:
             samples (numpy.ndarray): Audio samples [num_samples x num_channels].
diff --git a/paddlespeech/server/bin/main.py b/paddlespeech/server/bin/main.py
index 45ded33d..81824c85 100644
--- a/paddlespeech/server/bin/main.py
+++ b/paddlespeech/server/bin/main.py
@@ -18,8 +18,8 @@ from fastapi import FastAPI
 
 from paddlespeech.server.engine.engine_pool import init_engine_pool
 from paddlespeech.server.restful.api import setup_router as setup_http_router
-from paddlespeech.server.ws.api import setup_router as setup_ws_router
 from paddlespeech.server.utils.config import get_config
+from paddlespeech.server.ws.api import setup_router as setup_ws_router
 
 app = FastAPI(
     title="PaddleSpeech Serving API", description="Api", version="0.0.1")
diff --git a/paddlespeech/server/engine/asr/online/asr_engine.py b/paddlespeech/server/engine/asr/online/asr_engine.py
index d5c1aa7b..389175a0 100644
--- a/paddlespeech/server/engine/asr/online/asr_engine.py
+++ b/paddlespeech/server/engine/asr/online/asr_engine.py
@@ -11,29 +11,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import io
 import os
-import time
 from typing import Optional
-import pickle
-import numpy as np
-from numpy import float32
-import soundfile
 
+import numpy as np
 import paddle
+from numpy import float32
 from yacs.config import CfgNode
 
-from paddlespeech.s2t.frontend.speech import SpeechSegment
 from paddlespeech.cli.asr.infer import ASRExecutor
 from paddlespeech.cli.log import logger
 from paddlespeech.cli.utils import MODEL_HOME
 from paddlespeech.s2t.frontend.featurizer.text_featurizer import TextFeaturizer
+from paddlespeech.s2t.frontend.speech import SpeechSegment
 from paddlespeech.s2t.modules.ctc import CTCDecoder
 from paddlespeech.s2t.utils.utility import UpdateConfig
 from paddlespeech.server.engine.base_engine import BaseEngine
-from paddlespeech.server.utils.config import get_config
 from paddlespeech.server.utils.paddle_predictor import init_predictor
-from paddlespeech.server.utils.paddle_predictor import run_model
 
 __all__ = ['ASREngine']
 
@@ -141,10 +135,10 @@ class ASRServerExecutor(ASRExecutor):
             reduction=True,  # sum
             batch_average=True,  # sum / batch_size
             grad_norm_type=self.config.get('ctc_grad_norm_type', None))
-        
+
         # init decoder
         cfg = self.config.decode
-        decode_batch_size = 1      # for online
+        decode_batch_size = 1  # for online
         self.decoder.init_decoder(
             decode_batch_size, self.text_feature.vocab_list,
             cfg.decoding_method, cfg.lang_model_path, cfg.alpha, cfg.beta,
@@ -182,10 +176,11 @@ class ASRServerExecutor(ASRExecutor):
         Returns:
             [type]: [description]
         """
-        if "deepspeech2online" in model_type :
+        if "deepspeech2online" in model_type:
             input_names = self.am_predictor.get_input_names()
             audio_handle = self.am_predictor.get_input_handle(input_names[0])
-            audio_len_handle = self.am_predictor.get_input_handle(input_names[1])
+            audio_len_handle = self.am_predictor.get_input_handle(
+                input_names[1])
             h_box_handle = self.am_predictor.get_input_handle(input_names[2])
             c_box_handle = self.am_predictor.get_input_handle(input_names[3])
 
@@ -203,7 +198,8 @@ class ASRServerExecutor(ASRExecutor):
 
             output_names = self.am_predictor.get_output_names()
             output_handle = self.am_predictor.get_output_handle(output_names[0])
-            output_lens_handle = self.am_predictor.get_output_handle(output_names[1])
+            output_lens_handle = self.am_predictor.get_output_handle(
+                output_names[1])
             output_state_h_handle = self.am_predictor.get_output_handle(
                 output_names[2])
             output_state_c_handle = self.am_predictor.get_output_handle(
@@ -341,7 +337,8 @@ class ASREngine(BaseEngine):
             x_chunk_lens (numpy.array): shape[B]
             decoder_chunk_size(int)
         """
-        self.output = self.executor.decode_one_chunk(x_chunk, x_chunk_lens, self.config.model_type)
+        self.output = self.executor.decode_one_chunk(x_chunk, x_chunk_lens,
+                                                     self.config.model_type)
 
     def postprocess(self):
         """postprocess
diff --git a/paddlespeech/server/utils/buffer.py b/paddlespeech/server/utils/buffer.py
index 4c1a3958..682357b3 100644
--- a/paddlespeech/server/utils/buffer.py
+++ b/paddlespeech/server/utils/buffer.py
@@ -43,10 +43,10 @@ class ChunkBuffer(object):
         audio = self.remained_audio + audio
         self.remained_audio = b''
 
-        n = int(self.sample_rate *
-                (self.frame_duration_ms / 1000.0) * self.sample_width)
-        shift_n = int(self.sample_rate *
-                      (self.shift_ms / 1000.0) * self.sample_width)
+        n = int(self.sample_rate * (self.frame_duration_ms / 1000.0) *
+                self.sample_width)
+        shift_n = int(self.sample_rate * (self.shift_ms / 1000.0) *
+                      self.sample_width)
         offset = 0
         timestamp = 0.0
         duration = (float(n) / self.sample_rate) / self.sample_width

From 92d699c1f820090ae5575f37267c464ba8b4081d Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 03:02:14 +0000
Subject: [PATCH 108/126] fix raw data

---
 .../examples/decoder/offline_decoder_sliding_chunk_main.cc  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
index ad72b772..27bd7b1b 100644
--- a/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
+++ b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
@@ -17,7 +17,7 @@
 #include "base/flags.h"
 #include "base/log.h"
 #include "decoder/ctc_beam_search_decoder.h"
-#include "frontend/raw_audio.h"
+#include "frontend/data_cache.h"
 #include "kaldi/util/table-types.h"
 #include "nnet/decodable.h"
 #include "nnet/paddle_nnet.h"
@@ -57,8 +57,8 @@ int main(int argc, char* argv[]) {
     model_opts.cache_shape = "5-1-1024,5-1-1024";
     std::shared_ptr<ppspeech::PaddleNnet> nnet(
         new ppspeech::PaddleNnet(model_opts));
-    std::shared_ptr<ppspeech::RawDataCache> raw_data(
-        new ppspeech::RawDataCache());
+    std::shared_ptr<ppspeech::DataCache> raw_data(
+        new ppspeech::DataCache());
     std::shared_ptr<ppspeech::Decodable> decodable(
         new ppspeech::Decodable(nnet, raw_data));
 

From e90bacc277983fee7b49d3ed9cdb8d51864f0961 Mon Sep 17 00:00:00 2001
From: Yang Zhou <goat.zhou@qq.com>
Date: Fri, 1 Apr 2022 11:04:11 +0800
Subject: [PATCH 109/126] remove model opts in offline_decoder_main

---
 speechx/examples/decoder/offline_decoder_main.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/speechx/examples/decoder/offline_decoder_main.cc b/speechx/examples/decoder/offline_decoder_main.cc
index 3a858ad1..3a94cc94 100644
--- a/speechx/examples/decoder/offline_decoder_main.cc
+++ b/speechx/examples/decoder/offline_decoder_main.cc
@@ -55,7 +55,6 @@ int main(int argc, char* argv[]) {
 
     // frontend + nnet is decodable
     ppspeech::ModelOptions model_opts;
-    model_opts.cache_shape = "5-1-1024,5-1-1024";
     model_opts.model_path = model_graph;
     model_opts.params_path = model_params;
     std::shared_ptr<ppspeech::PaddleNnet> nnet(

From 78219cef7b7f6cd822a63256a90f3144349acfb2 Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Fri, 1 Apr 2022 03:23:12 +0000
Subject: [PATCH 110/126] add cnndecoder pretrained model, test=doc

---
 examples/csmsc/tts3/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/csmsc/tts3/README.md b/examples/csmsc/tts3/README.md
index 08bf2ee9..ae8f7af6 100644
--- a/examples/csmsc/tts3/README.md
+++ b/examples/csmsc/tts3/README.md
@@ -226,6 +226,7 @@ CUDA_VISIBLE_DEVICES=${gpus} ./local/inference.sh ${train_output_path}
 Pretrained FastSpeech2 model with no silence in the edge of audios:
 - [fastspeech2_nosil_baker_ckpt_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_ckpt_0.4.zip)
 - [fastspeech2_conformer_baker_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_conformer_baker_ckpt_0.5.zip)
+- [fastspeech2_cnndecoder_csmsc_ckpt_1.0.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_cnndecoder_csmsc_ckpt_1.0.0.zip)
 
 The static model can be downloaded here:
 - [fastspeech2_nosil_baker_static_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_static_0.4.zip)

From 3aec266ca5f36934f694673db864d3c3f7faa8af Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Fri, 1 Apr 2022 03:42:06 +0000
Subject: [PATCH 111/126] add chunk size and pad size in args, test=doc

---
 paddlespeech/t2s/exps/synthesize_streaming.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/paddlespeech/t2s/exps/synthesize_streaming.py b/paddlespeech/t2s/exps/synthesize_streaming.py
index 62915539..f38b2d35 100644
--- a/paddlespeech/t2s/exps/synthesize_streaming.py
+++ b/paddlespeech/t2s/exps/synthesize_streaming.py
@@ -93,8 +93,8 @@ def evaluate(args):
 
     N = 0
     T = 0
-    chunk_size = 42
-    pad_size = 12
+    chunk_size = args.chunk_size
+    pad_size = args.pad_size
 
     for utt_id, sentence in sentences:
         with timer() as t:
@@ -109,7 +109,7 @@ def evaluate(args):
                 phone_ids = input_ids["phone_ids"]
             else:
                 print("lang should in be 'zh' here!")
-            # merge_sentences=False here, so we only use the first item of phone_ids
+            # merge_sentences=True here, so we only use the first item of phone_ids
             phone_ids = phone_ids[0]
             with paddle.no_grad():
                 # acoustic model
@@ -246,6 +246,11 @@ def parse_args():
         type=str2bool,
         default=False,
         help="whether use streaming acoustic model")
+    parser.add_argument(
+        "--chunk_size", type=int, default=42, help="chunk size of am streaming")
+    parser.add_argument(
+        "--pad_size", type=int, default=12, help="pad size of am streaming")
+
     parser.add_argument("--output_dir", type=str, help="output dir.")
 
     args = parser.parse_args()

From 2a8b44c525030b3c5d98611c4dd65ec4a95ea4d1 Mon Sep 17 00:00:00 2001
From: KP <109694228@qq.com>
Date: Fri, 1 Apr 2022 14:37:23 +0800
Subject: [PATCH 112/126] Update README.md

Update readme in docs. test=doc
---
 paddleaudio/docs/README.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/paddleaudio/docs/README.md b/paddleaudio/docs/README.md
index 8e4fccc5..f35d1d00 100644
--- a/paddleaudio/docs/README.md
+++ b/paddleaudio/docs/README.md
@@ -1,14 +1,15 @@
 # Build docs for PaddleAudio
 
+Execute the following steps in **current directory**.
+
 ## 1. Install
 
-`pip install Sphinx`
-`pip install sphinx_rtd_theme`
+`pip install Sphinx sphinx_rtd_theme`
 
 
 ## 2. Generate API docs
 
-Exclude `paddleaudio.utils`
+Generate API docs from doc string `*py`
 
 `sphinx-apidoc -fMeT -o source ../paddleaudio ../paddleaudio/utils --templatedir source/_templates`
 
@@ -16,3 +17,8 @@ Exclude `paddleaudio.utils`
 ## 3. Build
 
 `sphinx-build source _html`
+
+
+## 4. Preview
+
+Open `_html/index.html` for page preview.

From fd345180dd7513ea8e73e987f03bb7460e3bfe6d Mon Sep 17 00:00:00 2001
From: KP <109694228@qq.com>
Date: Fri, 1 Apr 2022 14:59:41 +0800
Subject: [PATCH 113/126] Add readme. test=doc

---
 paddleaudio/README.md | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 paddleaudio/README.md

diff --git a/paddleaudio/README.md b/paddleaudio/README.md
new file mode 100644
index 00000000..697c0173
--- /dev/null
+++ b/paddleaudio/README.md
@@ -0,0 +1,7 @@
+# PaddleAudio
+
+PaddleAudio is an audio library for PaddlePaddle.
+
+## Install
+
+`pip install .`

From 585eb9aec5fd69011d9121868385574b58af8db4 Mon Sep 17 00:00:00 2001
From: KP <109694228@qq.com>
Date: Fri, 1 Apr 2022 15:00:31 +0800
Subject: [PATCH 114/126] Update readme. test=doc

---
 paddleaudio/docs/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/paddleaudio/docs/README.md b/paddleaudio/docs/README.md
index f35d1d00..20626f52 100644
--- a/paddleaudio/docs/README.md
+++ b/paddleaudio/docs/README.md
@@ -9,7 +9,7 @@ Execute the following steps in **current directory**.
 
 ## 2. Generate API docs
 
-Generate API docs from doc string `*py`
+Generate API docs from doc string.
 
 `sphinx-apidoc -fMeT -o source ../paddleaudio ../paddleaudio/utils --templatedir source/_templates`
 

From 93c3e03bc846053889c823a349d7921686a329c3 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 07:49:53 +0000
Subject: [PATCH 115/126] more comment

---
 speechx/examples/decoder/decoder_test_main.cc |  7 ++-
 .../examples/decoder/offline_decoder_main.cc  |  2 +-
 .../offline_decoder_sliding_chunk_main.cc     | 48 ++++++++++++++-----
 speechx/speechx/nnet/decodable.cc             |  2 +-
 4 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/speechx/examples/decoder/decoder_test_main.cc b/speechx/examples/decoder/decoder_test_main.cc
index 79fe63fc..0e249cc6 100644
--- a/speechx/examples/decoder/decoder_test_main.cc
+++ b/speechx/examples/decoder/decoder_test_main.cc
@@ -24,11 +24,11 @@ DEFINE_string(nnet_prob_respecifier, "", "test nnet prob rspecifier");
 DEFINE_string(dict_file, "vocab.txt", "vocabulary of lm");
 DEFINE_string(lm_path, "lm.klm", "language model");
 
-
 using kaldi::BaseFloat;
 using kaldi::Matrix;
 using std::vector;
 
+// test decoder by feeding nnet posterior probability
 int main(int argc, char* argv[]) {
     gflags::ParseCommandLineFlags(&argc, &argv, false);
     google::InitGoogleLogging(argv[0]);
@@ -37,6 +37,8 @@ int main(int argc, char* argv[]) {
         FLAGS_nnet_prob_respecifier);
     std::string dict_file = FLAGS_dict_file;
     std::string lm_path = FLAGS_lm_path;
+    LOG(INFO) << "dict path: " << dict_file;
+    LOG(INFO) << "lm path: " << lm_path;
 
     int32 num_done = 0, num_err = 0;
 
@@ -53,6 +55,9 @@ int main(int argc, char* argv[]) {
     for (; !likelihood_reader.Done(); likelihood_reader.Next()) {
         string utt = likelihood_reader.Key();
         const kaldi::Matrix<BaseFloat> likelihood = likelihood_reader.Value();
+        LOG(INFO) << "process utt: " << utt;
+        LOG(INFO) << "rows: " << likelihood.NumRows();
+        LOG(INFO) << "cols: " << likelihood.NumCols();
         decodable->Acceptlikelihood(likelihood);
         decoder.AdvanceDecode(decodable);
         std::string result;
diff --git a/speechx/examples/decoder/offline_decoder_main.cc b/speechx/examples/decoder/offline_decoder_main.cc
index c73d5968..6bd83b9b 100644
--- a/speechx/examples/decoder/offline_decoder_main.cc
+++ b/speechx/examples/decoder/offline_decoder_main.cc
@@ -34,6 +34,7 @@ using kaldi::BaseFloat;
 using kaldi::Matrix;
 using std::vector;
 
+// test decoder by feeding speech feature, deprecated.
 int main(int argc, char* argv[]) {
     gflags::ParseCommandLineFlags(&argc, &argv, false);
     google::InitGoogleLogging(argv[0]);
@@ -55,7 +56,6 @@ int main(int argc, char* argv[]) {
 
     // frontend + nnet is decodable
     ppspeech::ModelOptions model_opts;
-    model_opts.cache_shape = "5-1-1024,5-1-1024";
     model_opts.model_path = model_graph;
     model_opts.params_path = model_params;
     std::shared_ptr<ppspeech::PaddleNnet> nnet(
diff --git a/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
index 27bd7b1b..4d5ffe14 100644
--- a/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
+++ b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
@@ -27,12 +27,19 @@ DEFINE_string(model_path, "avg_1.jit.pdmodel", "paddle nnet model");
 DEFINE_string(param_path, "avg_1.jit.pdiparams", "paddle nnet model param");
 DEFINE_string(dict_file, "vocab.txt", "vocabulary of lm");
 DEFINE_string(lm_path, "lm.klm", "language model");
-
+DEFINE_int32(receptive_field_length,
+             7,
+             "receptive field of two CNN(kernel=5) downsampling module.");
+DEFINE_int32(downsampling_rate,
+             4,
+             "two CNN(kernel=5) module downsampling rate.");
 
 using kaldi::BaseFloat;
 using kaldi::Matrix;
 using std::vector;
 
+
+// test ds2 online decoder by feeding speech feature
 int main(int argc, char* argv[]) {
     gflags::ParseCommandLineFlags(&argc, &argv, false);
     google::InitGoogleLogging(argv[0]);
@@ -43,6 +50,11 @@ int main(int argc, char* argv[]) {
     std::string model_params = FLAGS_param_path;
     std::string dict_file = FLAGS_dict_file;
     std::string lm_path = FLAGS_lm_path;
+    LOG(INFO) << "model path: " << model_graph;
+    LOG(INFO) << "model param: " << model_params;
+    LOG(INFO) << "dict path: " << dict_file;
+    LOG(INFO) << "lm path: " << lm_path;
+
 
     int32 num_done = 0, num_err = 0;
 
@@ -57,34 +69,44 @@ int main(int argc, char* argv[]) {
     model_opts.cache_shape = "5-1-1024,5-1-1024";
     std::shared_ptr<ppspeech::PaddleNnet> nnet(
         new ppspeech::PaddleNnet(model_opts));
-    std::shared_ptr<ppspeech::DataCache> raw_data(
-        new ppspeech::DataCache());
+    std::shared_ptr<ppspeech::DataCache> raw_data(new ppspeech::DataCache());
     std::shared_ptr<ppspeech::Decodable> decodable(
         new ppspeech::Decodable(nnet, raw_data));
 
-    int32 chunk_size = 7;
-    int32 chunk_stride = 4;
-    int32 receptive_field_length = 7;
+    int32 chunk_size = FLAGS_receptive_field_length;
+    int32 chunk_stride = FLAGS_downsampling_rate;
+    int32 receptive_field_length = FLAGS_receptive_field_length;
+    LOG(INFO) << "chunk size (frame): " << chunk_size;
+    LOG(INFO) << "chunk stride (frame): " << chunk_stride;
+    LOG(INFO) << "receptive field (frame): " << receptive_field_length;
     decoder.InitDecoder();
 
     for (; !feature_reader.Done(); feature_reader.Next()) {
         string utt = feature_reader.Key();
         kaldi::Matrix<BaseFloat> feature = feature_reader.Value();
         raw_data->SetDim(feature.NumCols());
+        LOG(INFO) << "process utt: " << utt;
+        LOG(INFO) << "rows: " << feature.NumRows();
+        LOG(INFO) << "cols: " << feature.NumCols();
+
         int32 row_idx = 0;
         int32 padding_len = 0;
-        int32 ori_feature_len = feature.NumRows(); 
-        if ( (feature.NumRows() - chunk_size) % chunk_stride != 0) {
-            padding_len = chunk_stride - (feature.NumRows() - chunk_size) % chunk_stride;
-            feature.Resize(feature.NumRows() + padding_len, feature.NumCols(), kaldi::kCopyData);
+        int32 ori_feature_len = feature.NumRows();
+        if ((feature.NumRows() - chunk_size) % chunk_stride != 0) {
+            padding_len =
+                chunk_stride - (feature.NumRows() - chunk_size) % chunk_stride;
+            feature.Resize(feature.NumRows() + padding_len,
+                           feature.NumCols(),
+                           kaldi::kCopyData);
         }
         int32 num_chunks = (feature.NumRows() - chunk_size) / chunk_stride + 1;
         for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) {
             kaldi::Vector<kaldi::BaseFloat> feature_chunk(chunk_size *
                                                           feature.NumCols());
-            int32 feature_chunk_size = 0; 
-            if ( ori_feature_len > chunk_idx * chunk_stride) {
-                feature_chunk_size = std::min(ori_feature_len - chunk_idx * chunk_stride, chunk_size);
+            int32 feature_chunk_size = 0;
+            if (ori_feature_len > chunk_idx * chunk_stride) {
+                feature_chunk_size = std::min(
+                    ori_feature_len - chunk_idx * chunk_stride, chunk_size);
             }
             if (feature_chunk_size < receptive_field_length) break;
 
diff --git a/speechx/speechx/nnet/decodable.cc b/speechx/speechx/nnet/decodable.cc
index cd72bf76..e6315d07 100644
--- a/speechx/speechx/nnet/decodable.cc
+++ b/speechx/speechx/nnet/decodable.cc
@@ -82,7 +82,7 @@ void Decodable::Reset() {
     if (nnet_ != nullptr) nnet_->Reset();
     frame_offset_ = 0;
     frames_ready_ = 0;
-    nnet_cache_.Resize(0,0);
+    nnet_cache_.Resize(0, 0);
 }
 
 }  // namespace ppspeech
\ No newline at end of file

From fea437abe3f6dc548408fcc92db36da769aaaf9b Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 08:19:47 +0000
Subject: [PATCH 116/126] add glog test

---
 .gitignore                                    |  2 +-
 .mergify.yml                                  |  2 +-
 speechx/examples/decoder/local/model.sh       |  3 +++
 speechx/examples/glog/CMakeLists.txt          |  8 ++++++
 speechx/examples/glog/README.md               | 25 +++++++++++++++++++
 .../examples/glog/glog_logtostderr_test.cc    | 25 +++++++++++++++++++
 speechx/examples/glog/glog_test.cc            | 23 +++++++++++++++++
 speechx/examples/glog/path.sh                 | 14 +++++++++++
 speechx/examples/glog/run.sh                  | 22 ++++++++++++++++
 9 files changed, 122 insertions(+), 2 deletions(-)
 create mode 100644 speechx/examples/decoder/local/model.sh
 create mode 100644 speechx/examples/glog/CMakeLists.txt
 create mode 100644 speechx/examples/glog/README.md
 create mode 100644 speechx/examples/glog/glog_logtostderr_test.cc
 create mode 100644 speechx/examples/glog/glog_test.cc
 create mode 100644 speechx/examples/glog/path.sh
 create mode 100755 speechx/examples/glog/run.sh

diff --git a/.gitignore b/.gitignore
index e25ec327..63947200 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,7 @@
 .DS_Store
 *.pyc
 .vscode
-*log
+*.log
 *.wav
 *.pdmodel
 *.pdiparams*
diff --git a/.mergify.yml b/.mergify.yml
index 6dae66d0..68b24810 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -52,7 +52,7 @@ pull_request_rules:
         add: ["T2S"]
   - name: "auto add label=Audio"
     conditions:
-      - files~=^audio/
+      - files~=^paddleaudio/
     actions:
       label:
         add: ["Audio"]
diff --git a/speechx/examples/decoder/local/model.sh b/speechx/examples/decoder/local/model.sh
new file mode 100644
index 00000000..5c609a6c
--- /dev/null
+++ b/speechx/examples/decoder/local/model.sh
@@ -0,0 +1,3 @@
+#!/bin/bash 
+
+
diff --git a/speechx/examples/glog/CMakeLists.txt b/speechx/examples/glog/CMakeLists.txt
new file mode 100644
index 00000000..b4b0e635
--- /dev/null
+++ b/speechx/examples/glog/CMakeLists.txt
@@ -0,0 +1,8 @@
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
+
+add_executable(glog_test ${CMAKE_CURRENT_SOURCE_DIR}/glog_test.cc)
+target_link_libraries(glog_test glog)
+
+
+add_executable(glog_logtostderr_test ${CMAKE_CURRENT_SOURCE_DIR}/glog_logtostderr_test.cc)
+target_link_libraries(glog_logtostderr_test glog)
\ No newline at end of file
diff --git a/speechx/examples/glog/README.md b/speechx/examples/glog/README.md
new file mode 100644
index 00000000..996e192e
--- /dev/null
+++ b/speechx/examples/glog/README.md
@@ -0,0 +1,25 @@
+# [GLOG](https://rpg.ifi.uzh.ch/docs/glog.html)
+
+Unless otherwise specified, glog writes to the filename `/tmp/<program name>.<hostname>.<user name>.log.<severity level>.<date>.<time>.<pid>` (e.g., "/tmp/hello_world.example.com.hamaji.log.INFO.20080709-222411.10474"). By default, glog copies the log messages of severity level ERROR or FATAL to standard error (stderr) in addition to log files.
+
+Several flags influence glog's output behavior. If the Google gflags library is installed on your machine, the configure script (see the INSTALL file in the package for detail of this script) will automatically detect and use it, allowing you to pass flags on the command line. For example, if you want to turn the flag --logtostderr on, you can start your application with the following command line:
+
+   `./your_application --logtostderr=1`
+
+If the Google gflags library isn't installed, you set flags via environment variables, prefixing the flag name with "GLOG_", e.g.
+
+   `GLOG_logtostderr=1 ./your_application`
+
+You can also modify flag values in your program by modifying global variables `FLAGS_*` . Most settings start working immediately after you update `FLAGS_*` . The exceptions are the flags related to destination files. For example, you might want to set `FLAGS_log_dir` before calling `google::InitGoogleLogging` . Here is an example:
+∂∂
+```c++
+   LOG(INFO) << "file";
+   // Most flags work immediately after updating values.
+   FLAGS_logtostderr = 1;
+   LOG(INFO) << "stderr";
+   FLAGS_logtostderr = 0;
+   // This won't change the log destination. If you want to set this
+   // value, you should do this before google::InitGoogleLogging .
+   FLAGS_log_dir = "/some/log/directory";
+   LOG(INFO) << "the same file";
+```
diff --git a/speechx/examples/glog/glog_logtostderr_test.cc b/speechx/examples/glog/glog_logtostderr_test.cc
new file mode 100644
index 00000000..b0616a7d
--- /dev/null
+++ b/speechx/examples/glog/glog_logtostderr_test.cc
@@ -0,0 +1,25 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <glog/logging.h>
+
+int main(int argc, char* argv[]) {
+    // Initialize Google’s logging library.
+    google::InitGoogleLogging(argv[0]);
+
+    FLAGS_logtostderr = 1;
+
+    LOG(INFO) << "Found " << 10 << " cookies";
+    LOG(ERROR) << "Found " << 10 << " error";
+}
\ No newline at end of file
diff --git a/speechx/examples/glog/glog_test.cc b/speechx/examples/glog/glog_test.cc
new file mode 100644
index 00000000..b6275119
--- /dev/null
+++ b/speechx/examples/glog/glog_test.cc
@@ -0,0 +1,23 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <glog/logging.h>
+
+int main(int argc, char* argv[]) {
+    // Initialize Google’s logging library.
+    google::InitGoogleLogging(argv[0]);
+
+    LOG(INFO) << "Found " << 10 << " cookies";
+    LOG(ERROR) << "Found " << 10 << " error";
+}
\ No newline at end of file
diff --git a/speechx/examples/glog/path.sh b/speechx/examples/glog/path.sh
new file mode 100644
index 00000000..e2c7b2fc
--- /dev/null
+++ b/speechx/examples/glog/path.sh
@@ -0,0 +1,14 @@
+# This contains the locations of binarys build required for running the examples.
+
+SPEECHX_ROOT=$PWD/../..
+SPEECHX_EXAMPLES=$SPEECHX_ROOT/build/examples
+
+SPEECHX_TOOLS=$SPEECHX_ROOT/tools
+TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin
+
+[ -d $SPEECHX_EXAMPLES ] || { echo "Error: 'build/examples' directory not found. please ensure that the project build successfully"; }
+
+export LC_AL=C
+
+SPEECHX_BIN=$SPEECHX_EXAMPLES/glog
+export PATH=$PATH:$SPEECHX_BIN:$TOOLS_BIN
diff --git a/speechx/examples/glog/run.sh b/speechx/examples/glog/run.sh
new file mode 100755
index 00000000..d3fcdb64
--- /dev/null
+++ b/speechx/examples/glog/run.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set +x
+set -e
+
+. ./path.sh
+
+# 1. compile
+if [ ! -d ${SPEECHX_EXAMPLES} ]; then
+    pushd ${SPEECHX_ROOT} 
+    bash build.sh
+    popd
+fi
+
+# 2. run 
+glog_test
+
+echo "------"
+export FLAGS_logtostderr=1 
+glog_test
+
+echo "------"
+glog_logtostderr_test

From a9f4ce47a34bbd62c88090ef9a6e3498dbfc669a Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 10:24:16 +0000
Subject: [PATCH 117/126] frontend itf

---
 speechx/examples/feat/linear_spectrogram_main.cc   | 12 ++++++------
 speechx/speechx/frontend/audio_cache.h             |  4 ++--
 speechx/speechx/frontend/data_cache.h              |  4 ++--
 speechx/speechx/frontend/fbank.h                   |  4 ++--
 speechx/speechx/frontend/feature_cache.cc          |  2 +-
 speechx/speechx/frontend/feature_cache.h           |  8 ++++----
 .../frontend/feature_extractor_controller.h        | 13 -------------
 .../frontend/feature_extractor_controller_impl.h   | 13 -------------
 ...eature_extractor_interface.h => frontend_itf.h} |  2 +-
 speechx/speechx/frontend/linear_spectrogram.cc     |  2 +-
 speechx/speechx/frontend/linear_spectrogram.h      |  8 ++++----
 speechx/speechx/frontend/normalizer.cc             |  4 ++--
 speechx/speechx/frontend/normalizer.h              | 14 +++++++-------
 speechx/speechx/nnet/decodable.cc                  |  2 +-
 speechx/speechx/nnet/decodable.h                   |  6 +++---
 15 files changed, 36 insertions(+), 62 deletions(-)
 delete mode 100644 speechx/speechx/frontend/feature_extractor_controller.h
 delete mode 100644 speechx/speechx/frontend/feature_extractor_controller_impl.h
 rename speechx/speechx/frontend/{feature_extractor_interface.h => frontend_itf.h} (97%)

diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc
index e1f0a895..c29d2b21 100644
--- a/speechx/examples/feat/linear_spectrogram_main.cc
+++ b/speechx/examples/feat/linear_spectrogram_main.cc
@@ -20,7 +20,7 @@
 #include "frontend/audio_cache.h"
 #include "frontend/data_cache.h"
 #include "frontend/feature_cache.h"
-#include "frontend/feature_extractor_interface.h"
+#include "frontend/frontend_itf.h"
 #include "frontend/normalizer.h"
 #include "kaldi/feat/wave-reader.h"
 #include "kaldi/util/kaldi-io.h"
@@ -170,13 +170,13 @@ int main(int argc, char* argv[]) {
     // feature pipeline: wave cache --> decibel_normalizer --> hanning
     // window -->linear_spectrogram --> global cmvn -> feat cache
 
-    // std::unique_ptr<ppspeech::FeatureExtractorInterface> data_source(new
+    // std::unique_ptr<ppspeech::FrontendInterface> data_source(new
     // ppspeech::DataCache());
-    std::unique_ptr<ppspeech::FeatureExtractorInterface> data_source(
+    std::unique_ptr<ppspeech::FrontendInterface> data_source(
         new ppspeech::AudioCache());
 
     ppspeech::DecibelNormalizerOptions db_norm_opt;
-    std::unique_ptr<ppspeech::FeatureExtractorInterface> db_norm(
+    std::unique_ptr<ppspeech::FrontendInterface> db_norm(
         new ppspeech::DecibelNormalizer(db_norm_opt, std::move(data_source)));
 
     ppspeech::LinearSpectrogramOptions opt;
@@ -185,10 +185,10 @@ int main(int argc, char* argv[]) {
     LOG(INFO) << "frame length (ms): " << opt.frame_opts.frame_length_ms;
     LOG(INFO) << "frame shift (ms): " << opt.frame_opts.frame_shift_ms;
 
-    std::unique_ptr<ppspeech::FeatureExtractorInterface> linear_spectrogram(
+    std::unique_ptr<ppspeech::FrontendInterface> linear_spectrogram(
         new ppspeech::LinearSpectrogram(opt, std::move(db_norm)));
 
-    std::unique_ptr<ppspeech::FeatureExtractorInterface> cmvn(
+    std::unique_ptr<ppspeech::FrontendInterface> cmvn(
         new ppspeech::CMVN(FLAGS_cmvn_write_path,
                            std::move(linear_spectrogram)));
 
diff --git a/speechx/speechx/frontend/audio_cache.h b/speechx/speechx/frontend/audio_cache.h
index b6c82c69..f48da12b 100644
--- a/speechx/speechx/frontend/audio_cache.h
+++ b/speechx/speechx/frontend/audio_cache.h
@@ -16,12 +16,12 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/feature_extractor_interface.h"
+#include "frontend/frontend_itf.h"
 
 namespace ppspeech {
 
 // waves cache
-class AudioCache : public FeatureExtractorInterface {
+class AudioCache : public FrontendInterface {
   public:
     explicit AudioCache(int buffer_size = kint16max);
 
diff --git a/speechx/speechx/frontend/data_cache.h b/speechx/speechx/frontend/data_cache.h
index dea51d76..b8ce6bf6 100644
--- a/speechx/speechx/frontend/data_cache.h
+++ b/speechx/speechx/frontend/data_cache.h
@@ -17,13 +17,13 @@
 
 
 #include "base/common.h"
-#include "frontend/feature_extractor_interface.h"
+#include "frontend/frontend_itf.h"
 
 
 namespace ppspeech {
 // A data source for testing different frontend module.
 // It accepts waves or feats.
-class DataCache : public FeatureExtractorInterface {
+class DataCache : public FrontendInterface {
   public:
     explicit DataCache() { finished_ = false; }
 
diff --git a/speechx/speechx/frontend/fbank.h b/speechx/speechx/frontend/fbank.h
index 7d9cf422..68267b3d 100644
--- a/speechx/speechx/frontend/fbank.h
+++ b/speechx/speechx/frontend/fbank.h
@@ -20,10 +20,10 @@
 
 namespace ppspeech {
 
-class FbankExtractor : FeatureExtractorInterface {
+class FbankExtractor : FrontendInterface {
   public:
     explicit FbankExtractor(const FbankOptions& opts,
-                            share_ptr<FeatureExtractorInterface> pre_extractor);
+                            share_ptr<FrontendInterface> pre_extractor);
     virtual void AcceptWaveform(
         const kaldi::Vector<kaldi::BaseFloat>& input) = 0;
     virtual void Read(kaldi::Vector<kaldi::BaseFloat>* feat) = 0;
diff --git a/speechx/speechx/frontend/feature_cache.cc b/speechx/speechx/frontend/feature_cache.cc
index dad6907c..53b7076d 100644
--- a/speechx/speechx/frontend/feature_cache.cc
+++ b/speechx/speechx/frontend/feature_cache.cc
@@ -24,7 +24,7 @@ using kaldi::SubVector;
 using std::unique_ptr;
 
 FeatureCache::FeatureCache(
-    int max_size, unique_ptr<FeatureExtractorInterface> base_extractor) {
+    int max_size, unique_ptr<FrontendInterface> base_extractor) {
     max_size_ = max_size;
     base_extractor_ = std::move(base_extractor);
 }
diff --git a/speechx/speechx/frontend/feature_cache.h b/speechx/speechx/frontend/feature_cache.h
index f52b9b0f..1281ec35 100644
--- a/speechx/speechx/frontend/feature_cache.h
+++ b/speechx/speechx/frontend/feature_cache.h
@@ -15,15 +15,15 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/feature_extractor_interface.h"
+#include "frontend/frontend_itf.h"
 
 namespace ppspeech {
 
-class FeatureCache : public FeatureExtractorInterface {
+class FeatureCache : public FrontendInterface {
   public:
     explicit FeatureCache(
         int32 max_size = kint16max,
-        std::unique_ptr<FeatureExtractorInterface> base_extractor = NULL);
+        std::unique_ptr<FrontendInterface> base_extractor = NULL);
 
     // Feed feats or waves
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs);
@@ -53,7 +53,7 @@ class FeatureCache : public FeatureExtractorInterface {
     bool Compute();
 
     size_t max_size_;
-    std::unique_ptr<FeatureExtractorInterface> base_extractor_;
+    std::unique_ptr<FrontendInterface> base_extractor_;
 
     std::mutex mutex_;
     std::queue<kaldi::Vector<BaseFloat>> cache_;
diff --git a/speechx/speechx/frontend/feature_extractor_controller.h b/speechx/speechx/frontend/feature_extractor_controller.h
deleted file mode 100644
index 0544a1e2..00000000
--- a/speechx/speechx/frontend/feature_extractor_controller.h
+++ /dev/null
@@ -1,13 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
diff --git a/speechx/speechx/frontend/feature_extractor_controller_impl.h b/speechx/speechx/frontend/feature_extractor_controller_impl.h
deleted file mode 100644
index 0544a1e2..00000000
--- a/speechx/speechx/frontend/feature_extractor_controller_impl.h
+++ /dev/null
@@ -1,13 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
diff --git a/speechx/speechx/frontend/feature_extractor_interface.h b/speechx/speechx/frontend/frontend_itf.h
similarity index 97%
rename from speechx/speechx/frontend/feature_extractor_interface.h
rename to speechx/speechx/frontend/frontend_itf.h
index 5da2526b..7913cc7c 100644
--- a/speechx/speechx/frontend/feature_extractor_interface.h
+++ b/speechx/speechx/frontend/frontend_itf.h
@@ -19,7 +19,7 @@
 
 namespace ppspeech {
 
-class FeatureExtractorInterface {
+class FrontendInterface {
   public:
     // Feed inputs: features(2D saved in 1D) or waveforms(1D).
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs) = 0;
diff --git a/speechx/speechx/frontend/linear_spectrogram.cc b/speechx/speechx/frontend/linear_spectrogram.cc
index 41bc8743..2ba00785 100644
--- a/speechx/speechx/frontend/linear_spectrogram.cc
+++ b/speechx/speechx/frontend/linear_spectrogram.cc
@@ -27,7 +27,7 @@ using std::vector;
 
 LinearSpectrogram::LinearSpectrogram(
     const LinearSpectrogramOptions& opts,
-    std::unique_ptr<FeatureExtractorInterface> base_extractor) {
+    std::unique_ptr<FrontendInterface> base_extractor) {
     opts_ = opts;
     base_extractor_ = std::move(base_extractor);
     int32 window_size = opts.frame_opts.WindowSize();
diff --git a/speechx/speechx/frontend/linear_spectrogram.h b/speechx/speechx/frontend/linear_spectrogram.h
index 10853904..136441ef 100644
--- a/speechx/speechx/frontend/linear_spectrogram.h
+++ b/speechx/speechx/frontend/linear_spectrogram.h
@@ -16,7 +16,7 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/feature_extractor_interface.h"
+#include "frontend/frontend_itf.h"
 #include "kaldi/feat/feature-window.h"
 
 namespace ppspeech {
@@ -35,11 +35,11 @@ struct LinearSpectrogramOptions {
     }
 };
 
-class LinearSpectrogram : public FeatureExtractorInterface {
+class LinearSpectrogram : public FrontendInterface {
   public:
     explicit LinearSpectrogram(
         const LinearSpectrogramOptions& opts,
-        std::unique_ptr<FeatureExtractorInterface> base_extractor);
+        std::unique_ptr<FrontendInterface> base_extractor);
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs);
     virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* feats);
     // the dim_ is the dim of single frame feature
@@ -61,7 +61,7 @@ class LinearSpectrogram : public FeatureExtractorInterface {
     std::vector<kaldi::BaseFloat> hanning_window_;
     kaldi::BaseFloat hanning_window_energy_;
     LinearSpectrogramOptions opts_;
-    std::unique_ptr<FeatureExtractorInterface> base_extractor_;
+    std::unique_ptr<FrontendInterface> base_extractor_;
     int chunk_sample_size_;
     DISALLOW_COPY_AND_ASSIGN(LinearSpectrogram);
 };
diff --git a/speechx/speechx/frontend/normalizer.cc b/speechx/speechx/frontend/normalizer.cc
index 52412561..26f11b69 100644
--- a/speechx/speechx/frontend/normalizer.cc
+++ b/speechx/speechx/frontend/normalizer.cc
@@ -28,7 +28,7 @@ using std::unique_ptr;
 
 DecibelNormalizer::DecibelNormalizer(
     const DecibelNormalizerOptions& opts,
-    std::unique_ptr<FeatureExtractorInterface> base_extractor) {
+    std::unique_ptr<FrontendInterface> base_extractor) {
     base_extractor_ = std::move(base_extractor);
     opts_ = opts;
     dim_ = 1;
@@ -92,7 +92,7 @@ bool DecibelNormalizer::Compute(VectorBase<BaseFloat>* waves) const {
 }
 
 CMVN::CMVN(std::string cmvn_file,
-           unique_ptr<FeatureExtractorInterface> base_extractor)
+           unique_ptr<FrontendInterface> base_extractor)
     : var_norm_(true) {
     base_extractor_ = std::move(base_extractor);
     bool binary;
diff --git a/speechx/speechx/frontend/normalizer.h b/speechx/speechx/frontend/normalizer.h
index 352d1e16..df181961 100644
--- a/speechx/speechx/frontend/normalizer.h
+++ b/speechx/speechx/frontend/normalizer.h
@@ -16,7 +16,7 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/feature_extractor_interface.h"
+#include "frontend/frontend_itf.h"
 #include "kaldi/matrix/kaldi-matrix.h"
 #include "kaldi/util/options-itf.h"
 
@@ -40,11 +40,11 @@ struct DecibelNormalizerOptions {
     }
 };
 
-class DecibelNormalizer : public FeatureExtractorInterface {
+class DecibelNormalizer : public FrontendInterface {
   public:
     explicit DecibelNormalizer(
         const DecibelNormalizerOptions& opts,
-        std::unique_ptr<FeatureExtractorInterface> base_extractor);
+        std::unique_ptr<FrontendInterface> base_extractor);
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& waves);
     virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* waves);
     // noramlize audio, the dim is 1.
@@ -57,15 +57,15 @@ class DecibelNormalizer : public FeatureExtractorInterface {
     bool Compute(kaldi::VectorBase<kaldi::BaseFloat>* waves) const;
     DecibelNormalizerOptions opts_;
     size_t dim_;
-    std::unique_ptr<FeatureExtractorInterface> base_extractor_;
+    std::unique_ptr<FrontendInterface> base_extractor_;
     kaldi::Vector<kaldi::BaseFloat> waveform_;
 };
 
 
-class CMVN : public FeatureExtractorInterface {
+class CMVN : public FrontendInterface {
   public:
     explicit CMVN(std::string cmvn_file,
-                  std::unique_ptr<FeatureExtractorInterface> base_extractor);
+                  std::unique_ptr<FrontendInterface> base_extractor);
     virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs);
 
     // the length of feats = feature_row * feature_dim,
@@ -81,7 +81,7 @@ class CMVN : public FeatureExtractorInterface {
     void Compute(kaldi::VectorBase<kaldi::BaseFloat>* feats) const;
     void ApplyCMVN(kaldi::MatrixBase<BaseFloat>* feats);
     kaldi::Matrix<double> stats_;
-    std::unique_ptr<FeatureExtractorInterface> base_extractor_;
+    std::unique_ptr<FrontendInterface> base_extractor_;
     size_t dim_;
     bool var_norm_;
 };
diff --git a/speechx/speechx/nnet/decodable.cc b/speechx/speechx/nnet/decodable.cc
index e6315d07..542168d2 100644
--- a/speechx/speechx/nnet/decodable.cc
+++ b/speechx/speechx/nnet/decodable.cc
@@ -22,7 +22,7 @@ using std::vector;
 using kaldi::Vector;
 
 Decodable::Decodable(const std::shared_ptr<NnetInterface>& nnet,
-                     const std::shared_ptr<FeatureExtractorInterface>& frontend)
+                     const std::shared_ptr<FrontendInterface>& frontend)
     : frontend_(frontend), nnet_(nnet), frame_offset_(0), frames_ready_(0) {}
 
 void Decodable::Acceptlikelihood(const Matrix<BaseFloat>& likelihood) {
diff --git a/speechx/speechx/nnet/decodable.h b/speechx/speechx/nnet/decodable.h
index 7938b582..ef17601f 100644
--- a/speechx/speechx/nnet/decodable.h
+++ b/speechx/speechx/nnet/decodable.h
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 #include "base/common.h"
-#include "frontend/feature_extractor_interface.h"
+#include "frontend/frontend_itf.h"
 #include "kaldi/matrix/kaldi-matrix.h"
 #include "nnet/decodable-itf.h"
 #include "nnet/nnet_interface.h"
@@ -26,7 +26,7 @@ class Decodable : public kaldi::DecodableInterface {
   public:
     explicit Decodable(
         const std::shared_ptr<NnetInterface>& nnet,
-        const std::shared_ptr<FeatureExtractorInterface>& frontend);
+        const std::shared_ptr<FrontendInterface>& frontend);
     // void Init(DecodableOpts config);
     virtual kaldi::BaseFloat LogLikelihood(int32 frame, int32 index);
     virtual bool IsLastFrame(int32 frame) const;
@@ -41,7 +41,7 @@ class Decodable : public kaldi::DecodableInterface {
 
   private:
     bool AdvanceChunk();
-    std::shared_ptr<FeatureExtractorInterface> frontend_;
+    std::shared_ptr<FrontendInterface> frontend_;
     std::shared_ptr<NnetInterface> nnet_;
     kaldi::Matrix<kaldi::BaseFloat> nnet_cache_;
     // std::vector<std::vector<kaldi::BaseFloat>> nnet_cache_;

From 8d66a254dae27dbe32f92921fa18be24326c689c Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 10:31:08 +0000
Subject: [PATCH 118/126] cmvn and db norm

---
 speechx/speechx/frontend/CMakeLists.txt       |  3 +-
 .../frontend/{normalizer.cc => cmvn.cc}       | 79 +--------------
 speechx/speechx/frontend/cmvn.h               | 34 +++++++
 speechx/speechx/frontend/db_norm.cc           | 95 +++++++++++++++++++
 speechx/speechx/frontend/db_norm.h            | 65 +++++++++++++
 speechx/speechx/frontend/normalizer.h         | 89 +----------------
 6 files changed, 199 insertions(+), 166 deletions(-)
 rename speechx/speechx/frontend/{normalizer.cc => cmvn.cc} (59%)
 create mode 100644 speechx/speechx/frontend/cmvn.h
 create mode 100644 speechx/speechx/frontend/db_norm.cc
 create mode 100644 speechx/speechx/frontend/db_norm.h

diff --git a/speechx/speechx/frontend/CMakeLists.txt b/speechx/speechx/frontend/CMakeLists.txt
index d0ec008e..35243b6e 100644
--- a/speechx/speechx/frontend/CMakeLists.txt
+++ b/speechx/speechx/frontend/CMakeLists.txt
@@ -1,7 +1,8 @@
 project(frontend)
 
 add_library(frontend STATIC
-  normalizer.cc
+  cmvn.cc
+  db_norm.cc
   linear_spectrogram.cc
   audio_cache.cc
   feature_cache.cc
diff --git a/speechx/speechx/frontend/normalizer.cc b/speechx/speechx/frontend/cmvn.cc
similarity index 59%
rename from speechx/speechx/frontend/normalizer.cc
rename to speechx/speechx/frontend/cmvn.cc
index 26f11b69..d9bba943 100644
--- a/speechx/speechx/frontend/normalizer.cc
+++ b/speechx/speechx/frontend/cmvn.cc
@@ -1,17 +1,3 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 
 #include "frontend/normalizer.h"
 #include "kaldi/feat/cmvn.h"
@@ -26,70 +12,7 @@ using std::vector;
 using kaldi::SubVector;
 using std::unique_ptr;
 
-DecibelNormalizer::DecibelNormalizer(
-    const DecibelNormalizerOptions& opts,
-    std::unique_ptr<FrontendInterface> base_extractor) {
-    base_extractor_ = std::move(base_extractor);
-    opts_ = opts;
-    dim_ = 1;
-}
-
-void DecibelNormalizer::Accept(const kaldi::VectorBase<BaseFloat>& waves) {
-    base_extractor_->Accept(waves);
-}
-
-bool DecibelNormalizer::Read(kaldi::Vector<BaseFloat>* waves) {
-    if (base_extractor_->Read(waves) == false || waves->Dim() == 0) {
-        return false;
-    }
-    Compute(waves);
-    return true;
-}
-
-bool DecibelNormalizer::Compute(VectorBase<BaseFloat>* waves) const {
-    // calculate db rms
-    BaseFloat rms_db = 0.0;
-    BaseFloat mean_square = 0.0;
-    BaseFloat gain = 0.0;
-    BaseFloat wave_float_normlization = 1.0f / (std::pow(2, 16 - 1));
-
-    vector<BaseFloat> samples;
-    samples.resize(waves->Dim());
-    for (size_t i = 0; i < samples.size(); ++i) {
-        samples[i] = (*waves)(i);
-    }
-
-    // square
-    for (auto& d : samples) {
-        if (opts_.convert_int_float) {
-            d = d * wave_float_normlization;
-        }
-        mean_square += d * d;
-    }
-
-    // mean
-    mean_square /= samples.size();
-    rms_db = 10 * std::log10(mean_square);
-    gain = opts_.target_db - rms_db;
-
-    if (gain > opts_.max_gain_db) {
-        LOG(ERROR)
-            << "Unable to normalize segment to " << opts_.target_db << "dB,"
-            << "because the the probable gain have exceeds opts_.max_gain_db"
-            << opts_.max_gain_db << "dB.";
-        return false;
-    }
-
-    // Note that this is an in-place transformation.
-    for (auto& item : samples) {
-        // python item *= 10.0 ** (gain / 20.0)
-        item *= std::pow(10.0, gain / 20.0);
-    }
 
-    std::memcpy(
-        waves->Data(), samples.data(), sizeof(BaseFloat) * samples.size());
-    return true;
-}
 
 CMVN::CMVN(std::string cmvn_file,
            unique_ptr<FrontendInterface> base_extractor)
@@ -185,4 +108,4 @@ void CMVN::ApplyCMVN(kaldi::MatrixBase<BaseFloat>* feats) {
     ApplyCmvn(stats_, var_norm_, feats);
 }
 
-}  // namespace ppspeech
+}  // namespace ppspeech
\ No newline at end of file
diff --git a/speechx/speechx/frontend/cmvn.h b/speechx/speechx/frontend/cmvn.h
new file mode 100644
index 00000000..fdf2a87a
--- /dev/null
+++ b/speechx/speechx/frontend/cmvn.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "base/common.h"
+#include "frontend/frontend_itf.h"
+#include "kaldi/matrix/kaldi-matrix.h"
+#include "kaldi/util/options-itf.h"
+
+namespace ppspeech {
+
+class CMVN : public FrontendInterface {
+  public:
+    explicit CMVN(std::string cmvn_file,
+                  std::unique_ptr<FrontendInterface> base_extractor);
+    virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs);
+
+    // the length of feats = feature_row * feature_dim,
+    // the Matrix is squashed into Vector
+    virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* feats);
+    // the dim_ is the feautre dim.
+    virtual size_t Dim() const { return dim_; }
+    virtual void SetFinished() { base_extractor_->SetFinished(); }
+    virtual bool IsFinished() const { return base_extractor_->IsFinished(); }
+    virtual void Reset() { base_extractor_->Reset(); }
+
+  private:
+    void Compute(kaldi::VectorBase<kaldi::BaseFloat>* feats) const;
+    void ApplyCMVN(kaldi::MatrixBase<BaseFloat>* feats);
+    kaldi::Matrix<double> stats_;
+    std::unique_ptr<FrontendInterface> base_extractor_;
+    size_t dim_;
+    bool var_norm_;
+};
+
+}  // namespace ppspeech
\ No newline at end of file
diff --git a/speechx/speechx/frontend/db_norm.cc b/speechx/speechx/frontend/db_norm.cc
new file mode 100644
index 00000000..830af13b
--- /dev/null
+++ b/speechx/speechx/frontend/db_norm.cc
@@ -0,0 +1,95 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#include "frontend/normalizer.h"
+#include "kaldi/feat/cmvn.h"
+#include "kaldi/util/kaldi-io.h"
+
+namespace ppspeech {
+
+using kaldi::Vector;
+using kaldi::VectorBase;
+using kaldi::BaseFloat;
+using std::vector;
+using kaldi::SubVector;
+using std::unique_ptr;
+
+DecibelNormalizer::DecibelNormalizer(
+    const DecibelNormalizerOptions& opts,
+    std::unique_ptr<FrontendInterface> base_extractor) {
+    base_extractor_ = std::move(base_extractor);
+    opts_ = opts;
+    dim_ = 1;
+}
+
+void DecibelNormalizer::Accept(const kaldi::VectorBase<BaseFloat>& waves) {
+    base_extractor_->Accept(waves);
+}
+
+bool DecibelNormalizer::Read(kaldi::Vector<BaseFloat>* waves) {
+    if (base_extractor_->Read(waves) == false || waves->Dim() == 0) {
+        return false;
+    }
+    Compute(waves);
+    return true;
+}
+
+bool DecibelNormalizer::Compute(VectorBase<BaseFloat>* waves) const {
+    // calculate db rms
+    BaseFloat rms_db = 0.0;
+    BaseFloat mean_square = 0.0;
+    BaseFloat gain = 0.0;
+    BaseFloat wave_float_normlization = 1.0f / (std::pow(2, 16 - 1));
+
+    vector<BaseFloat> samples;
+    samples.resize(waves->Dim());
+    for (size_t i = 0; i < samples.size(); ++i) {
+        samples[i] = (*waves)(i);
+    }
+
+    // square
+    for (auto& d : samples) {
+        if (opts_.convert_int_float) {
+            d = d * wave_float_normlization;
+        }
+        mean_square += d * d;
+    }
+
+    // mean
+    mean_square /= samples.size();
+    rms_db = 10 * std::log10(mean_square);
+    gain = opts_.target_db - rms_db;
+
+    if (gain > opts_.max_gain_db) {
+        LOG(ERROR)
+            << "Unable to normalize segment to " << opts_.target_db << "dB,"
+            << "because the the probable gain have exceeds opts_.max_gain_db"
+            << opts_.max_gain_db << "dB.";
+        return false;
+    }
+
+    // Note that this is an in-place transformation.
+    for (auto& item : samples) {
+        // python item *= 10.0 ** (gain / 20.0)
+        item *= std::pow(10.0, gain / 20.0);
+    }
+
+    std::memcpy(
+        waves->Data(), samples.data(), sizeof(BaseFloat) * samples.size());
+    return true;
+}
+
+
+}  // namespace ppspeech
diff --git a/speechx/speechx/frontend/db_norm.h b/speechx/speechx/frontend/db_norm.h
new file mode 100644
index 00000000..3d371071
--- /dev/null
+++ b/speechx/speechx/frontend/db_norm.h
@@ -0,0 +1,65 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#pragma once
+
+#include "base/common.h"
+#include "frontend/frontend_itf.h"
+#include "kaldi/matrix/kaldi-matrix.h"
+#include "kaldi/util/options-itf.h"
+
+namespace ppspeech {
+
+struct DecibelNormalizerOptions {
+    float target_db;
+    float max_gain_db;
+    bool convert_int_float;
+    DecibelNormalizerOptions()
+        : target_db(-20), max_gain_db(300.0), convert_int_float(false) {}
+
+    void Register(kaldi::OptionsItf* opts) {
+        opts->Register(
+            "target-db", &target_db, "target db for db normalization");
+        opts->Register(
+            "max-gain-db", &max_gain_db, "max gain db for db normalization");
+        opts->Register("convert-int-float",
+                       &convert_int_float,
+                       "if convert int samples to float");
+    }
+};
+
+class DecibelNormalizer : public FrontendInterface {
+  public:
+    explicit DecibelNormalizer(
+        const DecibelNormalizerOptions& opts,
+        std::unique_ptr<FrontendInterface> base_extractor);
+    virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& waves);
+    virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* waves);
+    // noramlize audio, the dim is 1.
+    virtual size_t Dim() const { return dim_; }
+    virtual void SetFinished() { base_extractor_->SetFinished(); }
+    virtual bool IsFinished() const { return base_extractor_->IsFinished(); }
+    virtual void Reset() { base_extractor_->Reset(); }
+
+  private:
+    bool Compute(kaldi::VectorBase<kaldi::BaseFloat>* waves) const;
+    DecibelNormalizerOptions opts_;
+    size_t dim_;
+    std::unique_ptr<FrontendInterface> base_extractor_;
+    kaldi::Vector<kaldi::BaseFloat> waveform_;
+};
+
+
+}  // namespace ppspeech
\ No newline at end of file
diff --git a/speechx/speechx/frontend/normalizer.h b/speechx/speechx/frontend/normalizer.h
index df181961..89599c2a 100644
--- a/speechx/speechx/frontend/normalizer.h
+++ b/speechx/speechx/frontend/normalizer.h
@@ -1,89 +1,4 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-
 #pragma once
 
-#include "base/common.h"
-#include "frontend/frontend_itf.h"
-#include "kaldi/matrix/kaldi-matrix.h"
-#include "kaldi/util/options-itf.h"
-
-namespace ppspeech {
-
-struct DecibelNormalizerOptions {
-    float target_db;
-    float max_gain_db;
-    bool convert_int_float;
-    DecibelNormalizerOptions()
-        : target_db(-20), max_gain_db(300.0), convert_int_float(false) {}
-
-    void Register(kaldi::OptionsItf* opts) {
-        opts->Register(
-            "target-db", &target_db, "target db for db normalization");
-        opts->Register(
-            "max-gain-db", &max_gain_db, "max gain db for db normalization");
-        opts->Register("convert-int-float",
-                       &convert_int_float,
-                       "if convert int samples to float");
-    }
-};
-
-class DecibelNormalizer : public FrontendInterface {
-  public:
-    explicit DecibelNormalizer(
-        const DecibelNormalizerOptions& opts,
-        std::unique_ptr<FrontendInterface> base_extractor);
-    virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& waves);
-    virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* waves);
-    // noramlize audio, the dim is 1.
-    virtual size_t Dim() const { return dim_; }
-    virtual void SetFinished() { base_extractor_->SetFinished(); }
-    virtual bool IsFinished() const { return base_extractor_->IsFinished(); }
-    virtual void Reset() { base_extractor_->Reset(); }
-
-  private:
-    bool Compute(kaldi::VectorBase<kaldi::BaseFloat>* waves) const;
-    DecibelNormalizerOptions opts_;
-    size_t dim_;
-    std::unique_ptr<FrontendInterface> base_extractor_;
-    kaldi::Vector<kaldi::BaseFloat> waveform_;
-};
-
-
-class CMVN : public FrontendInterface {
-  public:
-    explicit CMVN(std::string cmvn_file,
-                  std::unique_ptr<FrontendInterface> base_extractor);
-    virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& inputs);
-
-    // the length of feats = feature_row * feature_dim,
-    // the Matrix is squashed into Vector
-    virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* feats);
-    // the dim_ is the feautre dim.
-    virtual size_t Dim() const { return dim_; }
-    virtual void SetFinished() { base_extractor_->SetFinished(); }
-    virtual bool IsFinished() const { return base_extractor_->IsFinished(); }
-    virtual void Reset() { base_extractor_->Reset(); }
-
-  private:
-    void Compute(kaldi::VectorBase<kaldi::BaseFloat>* feats) const;
-    void ApplyCMVN(kaldi::MatrixBase<BaseFloat>* feats);
-    kaldi::Matrix<double> stats_;
-    std::unique_ptr<FrontendInterface> base_extractor_;
-    size_t dim_;
-    bool var_norm_;
-};
-
-}  // namespace ppspeech
\ No newline at end of file
+#include "frontend/cmvn.h"
+#include "frontend/db_norm.h"
\ No newline at end of file

From 42c7537ce629a7fc717d27e5fc36073d86f8fce6 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 10:43:39 +0000
Subject: [PATCH 119/126] frontend to audio dir

---
 speechx/examples/decoder/offline_decoder_main.cc  |  2 +-
 .../decoder/offline_decoder_sliding_chunk_main.cc |  2 +-
 speechx/examples/feat/linear_spectrogram_main.cc  | 13 ++++++-------
 speechx/speechx/frontend/CMakeLists.txt           | 11 +----------
 speechx/speechx/frontend/audio/CMakeLists.txt     | 11 +++++++++++
 .../speechx/frontend/{ => audio}/audio_cache.cc   |  2 +-
 .../speechx/frontend/{ => audio}/audio_cache.h    |  2 +-
 speechx/speechx/frontend/{ => audio}/cmvn.cc      |  3 +--
 speechx/speechx/frontend/{ => audio}/cmvn.h       |  2 +-
 speechx/speechx/frontend/{ => audio}/data_cache.h |  2 +-
 speechx/speechx/frontend/{ => audio}/db_norm.cc   |  2 +-
 speechx/speechx/frontend/{ => audio}/db_norm.h    |  2 +-
 speechx/speechx/frontend/{ => audio}/fbank.h      |  0
 .../speechx/frontend/{ => audio}/feature_cache.cc |  2 +-
 .../speechx/frontend/{ => audio}/feature_cache.h  |  2 +-
 .../speechx/frontend/{ => audio}/frontend_itf.h   |  0
 .../frontend/{ => audio}/linear_spectrogram.cc    |  2 +-
 .../frontend/{ => audio}/linear_spectrogram.h     |  2 +-
 speechx/speechx/frontend/{ => audio}/mfcc.h       |  0
 speechx/speechx/frontend/audio/normalizer.h       |  4 ++++
 speechx/speechx/frontend/normalizer.h             |  4 ----
 speechx/speechx/frontend/window.h                 | 15 ---------------
 speechx/speechx/nnet/decodable.h                  |  2 +-
 23 files changed, 36 insertions(+), 51 deletions(-)
 rename speechx/speechx/frontend/{ => audio}/audio_cache.cc (98%)
 rename speechx/speechx/frontend/{ => audio}/audio_cache.h (97%)
 rename speechx/speechx/frontend/{ => audio}/cmvn.cc (98%)
 rename speechx/speechx/frontend/{ => audio}/cmvn.h (94%)
 rename speechx/speechx/frontend/{ => audio}/data_cache.h (97%)
 rename speechx/speechx/frontend/{ => audio}/db_norm.cc (98%)
 rename speechx/speechx/frontend/{ => audio}/db_norm.h (97%)
 rename speechx/speechx/frontend/{ => audio}/fbank.h (100%)
 rename speechx/speechx/frontend/{ => audio}/feature_cache.cc (97%)
 rename speechx/speechx/frontend/{ => audio}/feature_cache.h (97%)
 rename speechx/speechx/frontend/{ => audio}/frontend_itf.h (100%)
 rename speechx/speechx/frontend/{ => audio}/linear_spectrogram.cc (98%)
 rename speechx/speechx/frontend/{ => audio}/linear_spectrogram.h (97%)
 rename speechx/speechx/frontend/{ => audio}/mfcc.h (100%)
 create mode 100644 speechx/speechx/frontend/audio/normalizer.h
 delete mode 100644 speechx/speechx/frontend/normalizer.h
 delete mode 100644 speechx/speechx/frontend/window.h

diff --git a/speechx/examples/decoder/offline_decoder_main.cc b/speechx/examples/decoder/offline_decoder_main.cc
index 6bd83b9b..9a9c14a0 100644
--- a/speechx/examples/decoder/offline_decoder_main.cc
+++ b/speechx/examples/decoder/offline_decoder_main.cc
@@ -17,7 +17,7 @@
 #include "base/flags.h"
 #include "base/log.h"
 #include "decoder/ctc_beam_search_decoder.h"
-#include "frontend/data_cache.h"
+#include "frontend/audio/data_cache.h"
 #include "kaldi/util/table-types.h"
 #include "nnet/decodable.h"
 #include "nnet/paddle_nnet.h"
diff --git a/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
index 4d5ffe14..7f6c572c 100644
--- a/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
+++ b/speechx/examples/decoder/offline_decoder_sliding_chunk_main.cc
@@ -17,7 +17,7 @@
 #include "base/flags.h"
 #include "base/log.h"
 #include "decoder/ctc_beam_search_decoder.h"
-#include "frontend/data_cache.h"
+#include "frontend/audio/data_cache.h"
 #include "kaldi/util/table-types.h"
 #include "nnet/decodable.h"
 #include "nnet/paddle_nnet.h"
diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc
index c29d2b21..8f32bac2 100644
--- a/speechx/examples/feat/linear_spectrogram_main.cc
+++ b/speechx/examples/feat/linear_spectrogram_main.cc
@@ -14,19 +14,18 @@
 
 // todo refactor, repalce with gtest
 
-#include "frontend/linear_spectrogram.h"
 #include "base/flags.h"
 #include "base/log.h"
-#include "frontend/audio_cache.h"
-#include "frontend/data_cache.h"
-#include "frontend/feature_cache.h"
-#include "frontend/frontend_itf.h"
-#include "frontend/normalizer.h"
 #include "kaldi/feat/wave-reader.h"
 #include "kaldi/util/kaldi-io.h"
 #include "kaldi/util/table-types.h"
 
-#include <glog/logging.h>
+#include "frontend/audio/linear_spectrogram.h"
+#include "frontend/audio/audio_cache.h"
+#include "frontend/audio/data_cache.h"
+#include "frontend/audio/feature_cache.h"
+#include "frontend/audio/frontend_itf.h"
+#include "frontend/audio/normalizer.h"
 
 DEFINE_string(wav_rspecifier, "", "test wav scp path");
 DEFINE_string(feature_wspecifier, "", "output feats wspecifier");
diff --git a/speechx/speechx/frontend/CMakeLists.txt b/speechx/speechx/frontend/CMakeLists.txt
index 35243b6e..7d10fdec 100644
--- a/speechx/speechx/frontend/CMakeLists.txt
+++ b/speechx/speechx/frontend/CMakeLists.txt
@@ -1,11 +1,2 @@
-project(frontend)
 
-add_library(frontend STATIC
-  cmvn.cc
-  db_norm.cc
-  linear_spectrogram.cc
-  audio_cache.cc
-  feature_cache.cc
-)
-
-target_link_libraries(frontend PUBLIC kaldi-matrix)
\ No newline at end of file
+add_subdirectory(audio)
\ No newline at end of file
diff --git a/speechx/speechx/frontend/audio/CMakeLists.txt b/speechx/speechx/frontend/audio/CMakeLists.txt
index e69de29b..35243b6e 100644
--- a/speechx/speechx/frontend/audio/CMakeLists.txt
+++ b/speechx/speechx/frontend/audio/CMakeLists.txt
@@ -0,0 +1,11 @@
+project(frontend)
+
+add_library(frontend STATIC
+  cmvn.cc
+  db_norm.cc
+  linear_spectrogram.cc
+  audio_cache.cc
+  feature_cache.cc
+)
+
+target_link_libraries(frontend PUBLIC kaldi-matrix)
\ No newline at end of file
diff --git a/speechx/speechx/frontend/audio_cache.cc b/speechx/speechx/frontend/audio/audio_cache.cc
similarity index 98%
rename from speechx/speechx/frontend/audio_cache.cc
rename to speechx/speechx/frontend/audio/audio_cache.cc
index d44ed592..c3233e59 100644
--- a/speechx/speechx/frontend/audio_cache.cc
+++ b/speechx/speechx/frontend/audio/audio_cache.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "frontend/audio_cache.h"
+#include "frontend/audio/audio_cache.h"
 #include "kaldi/base/timer.h"
 
 namespace ppspeech {
diff --git a/speechx/speechx/frontend/audio_cache.h b/speechx/speechx/frontend/audio/audio_cache.h
similarity index 97%
rename from speechx/speechx/frontend/audio_cache.h
rename to speechx/speechx/frontend/audio/audio_cache.h
index f48da12b..17e1a838 100644
--- a/speechx/speechx/frontend/audio_cache.h
+++ b/speechx/speechx/frontend/audio/audio_cache.h
@@ -16,7 +16,7 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/frontend_itf.h"
+#include "frontend/audio/frontend_itf.h"
 
 namespace ppspeech {
 
diff --git a/speechx/speechx/frontend/cmvn.cc b/speechx/speechx/frontend/audio/cmvn.cc
similarity index 98%
rename from speechx/speechx/frontend/cmvn.cc
rename to speechx/speechx/frontend/audio/cmvn.cc
index d9bba943..706492b7 100644
--- a/speechx/speechx/frontend/cmvn.cc
+++ b/speechx/speechx/frontend/audio/cmvn.cc
@@ -1,5 +1,5 @@
 
-#include "frontend/normalizer.h"
+#include "frontend/audio/cmvn.h"
 #include "kaldi/feat/cmvn.h"
 #include "kaldi/util/kaldi-io.h"
 
@@ -13,7 +13,6 @@ using kaldi::SubVector;
 using std::unique_ptr;
 
 
-
 CMVN::CMVN(std::string cmvn_file,
            unique_ptr<FrontendInterface> base_extractor)
     : var_norm_(true) {
diff --git a/speechx/speechx/frontend/cmvn.h b/speechx/speechx/frontend/audio/cmvn.h
similarity index 94%
rename from speechx/speechx/frontend/cmvn.h
rename to speechx/speechx/frontend/audio/cmvn.h
index fdf2a87a..b3cfbb11 100644
--- a/speechx/speechx/frontend/cmvn.h
+++ b/speechx/speechx/frontend/audio/cmvn.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/frontend_itf.h"
+#include "frontend/audio/frontend_itf.h"
 #include "kaldi/matrix/kaldi-matrix.h"
 #include "kaldi/util/options-itf.h"
 
diff --git a/speechx/speechx/frontend/data_cache.h b/speechx/speechx/frontend/audio/data_cache.h
similarity index 97%
rename from speechx/speechx/frontend/data_cache.h
rename to speechx/speechx/frontend/audio/data_cache.h
index b8ce6bf6..a812278c 100644
--- a/speechx/speechx/frontend/data_cache.h
+++ b/speechx/speechx/frontend/audio/data_cache.h
@@ -17,7 +17,7 @@
 
 
 #include "base/common.h"
-#include "frontend/frontend_itf.h"
+#include "frontend/audio/frontend_itf.h"
 
 
 namespace ppspeech {
diff --git a/speechx/speechx/frontend/db_norm.cc b/speechx/speechx/frontend/audio/db_norm.cc
similarity index 98%
rename from speechx/speechx/frontend/db_norm.cc
rename to speechx/speechx/frontend/audio/db_norm.cc
index 830af13b..931e932d 100644
--- a/speechx/speechx/frontend/db_norm.cc
+++ b/speechx/speechx/frontend/audio/db_norm.cc
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 
-#include "frontend/normalizer.h"
+#include "frontend/audio/db_norm.h"
 #include "kaldi/feat/cmvn.h"
 #include "kaldi/util/kaldi-io.h"
 
diff --git a/speechx/speechx/frontend/db_norm.h b/speechx/speechx/frontend/audio/db_norm.h
similarity index 97%
rename from speechx/speechx/frontend/db_norm.h
rename to speechx/speechx/frontend/audio/db_norm.h
index 3d371071..42597143 100644
--- a/speechx/speechx/frontend/db_norm.h
+++ b/speechx/speechx/frontend/audio/db_norm.h
@@ -16,7 +16,7 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/frontend_itf.h"
+#include "frontend/audio/frontend_itf.h"
 #include "kaldi/matrix/kaldi-matrix.h"
 #include "kaldi/util/options-itf.h"
 
diff --git a/speechx/speechx/frontend/fbank.h b/speechx/speechx/frontend/audio/fbank.h
similarity index 100%
rename from speechx/speechx/frontend/fbank.h
rename to speechx/speechx/frontend/audio/fbank.h
diff --git a/speechx/speechx/frontend/feature_cache.cc b/speechx/speechx/frontend/audio/feature_cache.cc
similarity index 97%
rename from speechx/speechx/frontend/feature_cache.cc
rename to speechx/speechx/frontend/audio/feature_cache.cc
index 53b7076d..d7bea61a 100644
--- a/speechx/speechx/frontend/feature_cache.cc
+++ b/speechx/speechx/frontend/audio/feature_cache.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "frontend/feature_cache.h"
+#include "frontend/audio/feature_cache.h"
 
 namespace ppspeech {
 
diff --git a/speechx/speechx/frontend/feature_cache.h b/speechx/speechx/frontend/audio/feature_cache.h
similarity index 97%
rename from speechx/speechx/frontend/feature_cache.h
rename to speechx/speechx/frontend/audio/feature_cache.h
index 1281ec35..99961b5e 100644
--- a/speechx/speechx/frontend/feature_cache.h
+++ b/speechx/speechx/frontend/audio/feature_cache.h
@@ -15,7 +15,7 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/frontend_itf.h"
+#include "frontend/audio/frontend_itf.h"
 
 namespace ppspeech {
 
diff --git a/speechx/speechx/frontend/frontend_itf.h b/speechx/speechx/frontend/audio/frontend_itf.h
similarity index 100%
rename from speechx/speechx/frontend/frontend_itf.h
rename to speechx/speechx/frontend/audio/frontend_itf.h
diff --git a/speechx/speechx/frontend/linear_spectrogram.cc b/speechx/speechx/frontend/audio/linear_spectrogram.cc
similarity index 98%
rename from speechx/speechx/frontend/linear_spectrogram.cc
rename to speechx/speechx/frontend/audio/linear_spectrogram.cc
index 2ba00785..827b8ecc 100644
--- a/speechx/speechx/frontend/linear_spectrogram.cc
+++ b/speechx/speechx/frontend/audio/linear_spectrogram.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "frontend/linear_spectrogram.h"
+#include "frontend/audio/linear_spectrogram.h"
 #include "kaldi/base/kaldi-math.h"
 #include "kaldi/matrix/matrix-functions.h"
 
diff --git a/speechx/speechx/frontend/linear_spectrogram.h b/speechx/speechx/frontend/audio/linear_spectrogram.h
similarity index 97%
rename from speechx/speechx/frontend/linear_spectrogram.h
rename to speechx/speechx/frontend/audio/linear_spectrogram.h
index 136441ef..bbf8d685 100644
--- a/speechx/speechx/frontend/linear_spectrogram.h
+++ b/speechx/speechx/frontend/audio/linear_spectrogram.h
@@ -16,7 +16,7 @@
 #pragma once
 
 #include "base/common.h"
-#include "frontend/frontend_itf.h"
+#include "frontend/audio/frontend_itf.h"
 #include "kaldi/feat/feature-window.h"
 
 namespace ppspeech {
diff --git a/speechx/speechx/frontend/mfcc.h b/speechx/speechx/frontend/audio/mfcc.h
similarity index 100%
rename from speechx/speechx/frontend/mfcc.h
rename to speechx/speechx/frontend/audio/mfcc.h
diff --git a/speechx/speechx/frontend/audio/normalizer.h b/speechx/speechx/frontend/audio/normalizer.h
new file mode 100644
index 00000000..df9e4b75
--- /dev/null
+++ b/speechx/speechx/frontend/audio/normalizer.h
@@ -0,0 +1,4 @@
+#pragma once
+
+#include "frontend/audio/cmvn.h"
+#include "frontend/audio/db_norm.h"
\ No newline at end of file
diff --git a/speechx/speechx/frontend/normalizer.h b/speechx/speechx/frontend/normalizer.h
deleted file mode 100644
index 89599c2a..00000000
--- a/speechx/speechx/frontend/normalizer.h
+++ /dev/null
@@ -1,4 +0,0 @@
-#pragma once
-
-#include "frontend/cmvn.h"
-#include "frontend/db_norm.h"
\ No newline at end of file
diff --git a/speechx/speechx/frontend/window.h b/speechx/speechx/frontend/window.h
deleted file mode 100644
index 70d6307e..00000000
--- a/speechx/speechx/frontend/window.h
+++ /dev/null
@@ -1,15 +0,0 @@
-// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// extract the window of kaldi feat.
diff --git a/speechx/speechx/nnet/decodable.h b/speechx/speechx/nnet/decodable.h
index ef17601f..c75a0f4d 100644
--- a/speechx/speechx/nnet/decodable.h
+++ b/speechx/speechx/nnet/decodable.h
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 #include "base/common.h"
-#include "frontend/frontend_itf.h"
+#include "frontend/audio/frontend_itf.h"
 #include "kaldi/matrix/kaldi-matrix.h"
 #include "nnet/decodable-itf.h"
 #include "nnet/nnet_interface.h"

From 9071b9597de42bccbd34202d664ee834fb5fc34b Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 10:49:03 +0000
Subject: [PATCH 120/126] format code

---
 .../examples/feat/linear_spectrogram_main.cc    |  7 +++----
 speechx/speechx/frontend/audio/cmvn.cc          | 17 +++++++++++++++--
 speechx/speechx/frontend/audio/cmvn.h           | 14 ++++++++++++++
 speechx/speechx/frontend/audio/feature_cache.cc |  4 ++--
 speechx/speechx/frontend/audio/normalizer.h     | 14 ++++++++++++++
 speechx/speechx/nnet/decodable.h                |  5 ++---
 6 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc
index 8f32bac2..ca76d85c 100644
--- a/speechx/examples/feat/linear_spectrogram_main.cc
+++ b/speechx/examples/feat/linear_spectrogram_main.cc
@@ -20,11 +20,11 @@
 #include "kaldi/util/kaldi-io.h"
 #include "kaldi/util/table-types.h"
 
-#include "frontend/audio/linear_spectrogram.h"
 #include "frontend/audio/audio_cache.h"
 #include "frontend/audio/data_cache.h"
 #include "frontend/audio/feature_cache.h"
 #include "frontend/audio/frontend_itf.h"
+#include "frontend/audio/linear_spectrogram.h"
 #include "frontend/audio/normalizer.h"
 
 DEFINE_string(wav_rspecifier, "", "test wav scp path");
@@ -187,9 +187,8 @@ int main(int argc, char* argv[]) {
     std::unique_ptr<ppspeech::FrontendInterface> linear_spectrogram(
         new ppspeech::LinearSpectrogram(opt, std::move(db_norm)));
 
-    std::unique_ptr<ppspeech::FrontendInterface> cmvn(
-        new ppspeech::CMVN(FLAGS_cmvn_write_path,
-                           std::move(linear_spectrogram)));
+    std::unique_ptr<ppspeech::FrontendInterface> cmvn(new ppspeech::CMVN(
+        FLAGS_cmvn_write_path, std::move(linear_spectrogram)));
 
     ppspeech::FeatureCache feature_cache(kint16max, std::move(cmvn));
     LOG(INFO) << "feat dim: " << feature_cache.Dim();
diff --git a/speechx/speechx/frontend/audio/cmvn.cc b/speechx/speechx/frontend/audio/cmvn.cc
index 706492b7..4c1ffd6a 100644
--- a/speechx/speechx/frontend/audio/cmvn.cc
+++ b/speechx/speechx/frontend/audio/cmvn.cc
@@ -1,3 +1,17 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 
 #include "frontend/audio/cmvn.h"
 #include "kaldi/feat/cmvn.h"
@@ -13,8 +27,7 @@ using kaldi::SubVector;
 using std::unique_ptr;
 
 
-CMVN::CMVN(std::string cmvn_file,
-           unique_ptr<FrontendInterface> base_extractor)
+CMVN::CMVN(std::string cmvn_file, unique_ptr<FrontendInterface> base_extractor)
     : var_norm_(true) {
     base_extractor_ = std::move(base_extractor);
     bool binary;
diff --git a/speechx/speechx/frontend/audio/cmvn.h b/speechx/speechx/frontend/audio/cmvn.h
index b3cfbb11..50ef5649 100644
--- a/speechx/speechx/frontend/audio/cmvn.h
+++ b/speechx/speechx/frontend/audio/cmvn.h
@@ -1,3 +1,17 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #pragma once
 
 #include "base/common.h"
diff --git a/speechx/speechx/frontend/audio/feature_cache.cc b/speechx/speechx/frontend/audio/feature_cache.cc
index d7bea61a..3f7f6502 100644
--- a/speechx/speechx/frontend/audio/feature_cache.cc
+++ b/speechx/speechx/frontend/audio/feature_cache.cc
@@ -23,8 +23,8 @@ using std::vector;
 using kaldi::SubVector;
 using std::unique_ptr;
 
-FeatureCache::FeatureCache(
-    int max_size, unique_ptr<FrontendInterface> base_extractor) {
+FeatureCache::FeatureCache(int max_size,
+                           unique_ptr<FrontendInterface> base_extractor) {
     max_size_ = max_size;
     base_extractor_ = std::move(base_extractor);
 }
diff --git a/speechx/speechx/frontend/audio/normalizer.h b/speechx/speechx/frontend/audio/normalizer.h
index df9e4b75..dcf721dd 100644
--- a/speechx/speechx/frontend/audio/normalizer.h
+++ b/speechx/speechx/frontend/audio/normalizer.h
@@ -1,3 +1,17 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #pragma once
 
 #include "frontend/audio/cmvn.h"
diff --git a/speechx/speechx/nnet/decodable.h b/speechx/speechx/nnet/decodable.h
index c75a0f4d..3f0aab04 100644
--- a/speechx/speechx/nnet/decodable.h
+++ b/speechx/speechx/nnet/decodable.h
@@ -24,9 +24,8 @@ struct DecodableOpts;
 
 class Decodable : public kaldi::DecodableInterface {
   public:
-    explicit Decodable(
-        const std::shared_ptr<NnetInterface>& nnet,
-        const std::shared_ptr<FrontendInterface>& frontend);
+    explicit Decodable(const std::shared_ptr<NnetInterface>& nnet,
+                       const std::shared_ptr<FrontendInterface>& frontend);
     // void Init(DecodableOpts config);
     virtual kaldi::BaseFloat LogLikelihood(int32 frame, int32 index);
     virtual bool IsLastFrame(int32 frame) const;

From f83ec41161ef3ef8969495f34587ca6870f7cc79 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Fri, 1 Apr 2022 11:11:08 +0000
Subject: [PATCH 121/126] rename nnet itf

---
 speechx/speechx/nnet/{nnet_interface.h => nnet_itf.h} | 0
 speechx/speechx/nnet/paddle_nnet.h                    | 7 ++++---
 2 files changed, 4 insertions(+), 3 deletions(-)
 rename speechx/speechx/nnet/{nnet_interface.h => nnet_itf.h} (100%)

diff --git a/speechx/speechx/nnet/nnet_interface.h b/speechx/speechx/nnet/nnet_itf.h
similarity index 100%
rename from speechx/speechx/nnet/nnet_interface.h
rename to speechx/speechx/nnet/nnet_itf.h
diff --git a/speechx/speechx/nnet/paddle_nnet.h b/speechx/speechx/nnet/paddle_nnet.h
index 30fbac9f..906994d0 100644
--- a/speechx/speechx/nnet/paddle_nnet.h
+++ b/speechx/speechx/nnet/paddle_nnet.h
@@ -15,13 +15,14 @@
 
 #pragma once
 
-#include "base/common.h"
-#include "nnet/nnet_interface.h"
-#include "paddle_inference_api.h"
 
 #include "kaldi/matrix/kaldi-matrix.h"
 #include "kaldi/util/options-itf.h"
 
+#include "base/common.h"
+#include "nnet/nnet_itf.h"
+#include "paddle_inference_api.h"
+
 #include <numeric>
 
 namespace ppspeech {

From 8dfd3c6a3bf22e6c9001756611ce799f4ed13cb8 Mon Sep 17 00:00:00 2001
From: Hui Zhang <zhtclz@foxmail.com>
Date: Sat, 2 Apr 2022 02:18:33 +0000
Subject: [PATCH 122/126] fix nnet itf header, test=doc

---
 speechx/speechx/nnet/decodable.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/speechx/speechx/nnet/decodable.h b/speechx/speechx/nnet/decodable.h
index 3f0aab04..f035a338 100644
--- a/speechx/speechx/nnet/decodable.h
+++ b/speechx/speechx/nnet/decodable.h
@@ -16,7 +16,7 @@
 #include "frontend/audio/frontend_itf.h"
 #include "kaldi/matrix/kaldi-matrix.h"
 #include "nnet/decodable-itf.h"
-#include "nnet/nnet_interface.h"
+#include "nnet/nnet_itf.h"
 
 namespace ppspeech {
 

From 828ee1440459959e405851e43df7e0037c801be2 Mon Sep 17 00:00:00 2001
From: TianYuan <white-sky@qq.com>
Date: Sat, 2 Apr 2022 02:31:52 +0000
Subject: [PATCH 123/126] add license and reference for some models, test=doc

---
 .../t2s/exps/fastspeech2/gen_gta_mel.py        |  1 -
 paddlespeech/t2s/models/hifigan/hifigan.py     | 18 ++++++++++++++----
 paddlespeech/t2s/models/tacotron2/tacotron2.py |  1 +
 paddlespeech/t2s/models/wavernn/wavernn.py     |  1 +
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py b/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py
index 3fded29b..4c92ad1c 100644
--- a/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py
+++ b/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py
@@ -13,7 +13,6 @@
 # limitations under the License.
 # generate mels using durations.txt
 # for mb melgan finetune
-# 长度和原本的 mel 不一致怎么办？
 import argparse
 import os
 from pathlib import Path
diff --git a/paddlespeech/t2s/models/hifigan/hifigan.py b/paddlespeech/t2s/models/hifigan/hifigan.py
index 116376ec..ac5ff204 100644
--- a/paddlespeech/t2s/models/hifigan/hifigan.py
+++ b/paddlespeech/t2s/models/hifigan/hifigan.py
@@ -1,7 +1,17 @@
-# -*- coding: utf-8 -*-
-"""HiFi-GAN Modules.
-This code is based on https://github.com/jik876/hifi-gan.
-"""
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This code is based on https://github.com/jik876/hifi-gan.
 import copy
 from typing import Any
 from typing import Dict
diff --git a/paddlespeech/t2s/models/tacotron2/tacotron2.py b/paddlespeech/t2s/models/tacotron2/tacotron2.py
index abb691b4..7b306e48 100644
--- a/paddlespeech/t2s/models/tacotron2/tacotron2.py
+++ b/paddlespeech/t2s/models/tacotron2/tacotron2.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Modified from espnet(https://github.com/espnet/espnet)
 """Tacotron 2 related modules for paddle"""
 import logging
 from typing import Dict
diff --git a/paddlespeech/t2s/models/wavernn/wavernn.py b/paddlespeech/t2s/models/wavernn/wavernn.py
index 95907043..b4b8b480 100644
--- a/paddlespeech/t2s/models/wavernn/wavernn.py
+++ b/paddlespeech/t2s/models/wavernn/wavernn.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Modified from https://github.com/fatchord/WaveRNN
 import sys
 import time
 from typing import List

From ca860e3d2f5fc60e09963ee717288f979c002ab5 Mon Sep 17 00:00:00 2001
From: huangyuxin <hyxin2014@126.com>
Date: Sat, 2 Apr 2022 03:25:18 +0000
Subject: [PATCH 124/126] supplement note

---
 paddlespeech/s2t/decoders/recog.py     | 1 +
 paddlespeech/s2t/decoders/recog_bin.py | 1 +
 paddlespeech/s2t/models/u2/u2.py       | 1 +
 paddlespeech/s2t/models/u2/updater.py  | 1 +
 paddlespeech/s2t/models/u2_st/u2_st.py | 1 +
 paddlespeech/s2t/utils/bleu_score.py   | 1 +
 paddlespeech/s2t/utils/text_grid.py    | 1 +
 7 files changed, 7 insertions(+)

diff --git a/paddlespeech/s2t/decoders/recog.py b/paddlespeech/s2t/decoders/recog.py
index 88955eac..2d2aa210 100644
--- a/paddlespeech/s2t/decoders/recog.py
+++ b/paddlespeech/s2t/decoders/recog.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # Reference espnet Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+# Modified from espnet(https://github.com/espnet/espnet)
 """V2 backend for `asr_recog.py` using py:class:`decoders.beam_search.BeamSearch`."""
 import jsonlines
 import paddle
diff --git a/paddlespeech/s2t/decoders/recog_bin.py b/paddlespeech/s2t/decoders/recog_bin.py
index 0470a034..37b49f3a 100644
--- a/paddlespeech/s2t/decoders/recog_bin.py
+++ b/paddlespeech/s2t/decoders/recog_bin.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # Reference espnet Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+# Modified from espnet(https://github.com/espnet/espnet)
 """End-to-end speech recognition model decoding script."""
 import logging
 import os
diff --git a/paddlespeech/s2t/models/u2/u2.py b/paddlespeech/s2t/models/u2/u2.py
index 51388586..6a98607b 100644
--- a/paddlespeech/s2t/models/u2/u2.py
+++ b/paddlespeech/s2t/models/u2/u2.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Modified from wenet(https://github.com/wenet-e2e/wenet)
 """U2 ASR Model
 Unified Streaming and Non-streaming Two-pass End-to-end Model for Speech Recognition
 (https://arxiv.org/pdf/2012.05481.pdf)
diff --git a/paddlespeech/s2t/models/u2/updater.py b/paddlespeech/s2t/models/u2/updater.py
index bb18fe41..c59090a8 100644
--- a/paddlespeech/s2t/models/u2/updater.py
+++ b/paddlespeech/s2t/models/u2/updater.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Modified from wenet(https://github.com/wenet-e2e/wenet)
 from contextlib import nullcontext
 
 import paddle
diff --git a/paddlespeech/s2t/models/u2_st/u2_st.py b/paddlespeech/s2t/models/u2_st/u2_st.py
index 999723e5..6447753c 100644
--- a/paddlespeech/s2t/models/u2_st/u2_st.py
+++ b/paddlespeech/s2t/models/u2_st/u2_st.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Modified from wenet(https://github.com/wenet-e2e/wenet)
 """U2 ASR Model
 Unified Streaming and Non-streaming Two-pass End-to-end Model for Speech Recognition
 (https://arxiv.org/pdf/2012.05481.pdf)
diff --git a/paddlespeech/s2t/utils/bleu_score.py b/paddlespeech/s2t/utils/bleu_score.py
index a50c000a..d7eb9c7c 100644
--- a/paddlespeech/s2t/utils/bleu_score.py
+++ b/paddlespeech/s2t/utils/bleu_score.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Modified from espnet(https://github.com/espnet/espnet)
 """This module provides functions to calculate bleu score in different level.
 e.g. wer for word-level, cer for char-level.
 """
diff --git a/paddlespeech/s2t/utils/text_grid.py b/paddlespeech/s2t/utils/text_grid.py
index 3af58c9b..cbd9856e 100644
--- a/paddlespeech/s2t/utils/text_grid.py
+++ b/paddlespeech/s2t/utils/text_grid.py
@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# Modified from wenet(https://github.com/wenet-e2e/wenet)
 from typing import Dict
 from typing import List
 from typing import Text

From ff051507641c6cc35aa7ae58365767e5fdb17f98 Mon Sep 17 00:00:00 2001
From: qingen <qingenz123@126.com>
Date: Sat, 2 Apr 2022 22:30:58 +0800
Subject: [PATCH 125/126] [vec][search] update README, test=doc fix #1608

---
 demos/audio_searching/README.md     | 46 +++++++++++++++++----------
 demos/audio_searching/README_cn.md  | 48 ++++++++++++++++++-----------
 demos/audio_searching/src/encode.py |  5 +--
 3 files changed, 62 insertions(+), 37 deletions(-)

diff --git a/demos/audio_searching/README.md b/demos/audio_searching/README.md
index c0df12ec..8a6f3863 100644
--- a/demos/audio_searching/README.md
+++ b/demos/audio_searching/README.md
@@ -25,10 +25,14 @@ You can choose one way from easy, meduim and hard to install paddlespeech.
 The audio similarity search system requires Milvus, MySQL services. We can start these containers with one click through [docker-compose.yaml](./docker-compose.yaml), so please make sure you have [installed Docker Engine](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) before running. then
 
 ```bash
+## Enter the audio_searching directory for the following example
+cd ~/PaddleSpeech/demos/audio_searching/
+
+## Then start the related services within the container
 docker-compose -f docker-compose.yaml up -d
 ```
 
-Then you will see the that all containers are created:
+You will see the that all containers are created:
 
 ```bash
 Creating network "quick_deploy_app_net" with driver "bridge"
@@ -47,7 +51,7 @@ b2bcf279e599  milvusdb/milvus:v2.0.1  "/tini -- milvus run…"  22 hours ago  Up
 d8ef4c84e25c  mysql:5.7 "docker-entrypoint.s…"  22 hours ago  Up 22 hours 0.0.0.0:3306->3306/tcp, 33060/tcp audio-mysql
 8fb501edb4f3  quay.io/coreos/etcd:v3.5.0  "etcd -advertise-cli…"  22 hours ago  Up 22 hours 2379-2380/tcp milvus-etcd
 ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"  22 hours ago  Up 22 hours (healthy) 9000/tcp  milvus-minio
-15c84a506754  qingen1/paddlespeech-audio-search-client:2.3  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
+15c84a506754  paddlepaddle/paddlespeech-audio-search-client:2.3  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
 ```
 
 ### 3. Start API Server
@@ -58,22 +62,27 @@ Then to start the system server, and it provides HTTP backend services.
   ```bash
   pip install -r requirements.txt
   ```
-- Set configuration
+- Set configuration(In the case of local running, you can skip this step.)
 
   ```bash
+  ## Method 1: Modify the source file
   vim src/config.py
+
+  ## Method 2: Modify the environment variables, as shown in
+  export MILVUS_HOST=127.0.0.1
+  export MYSQL_HOST=127.0.0.1
   ```
 
-  Modify the parameters according to your own environment. Here listing some parameters that need to be set, for more information please refer to [config.py](./src/config.py).
+  Here listing some parameters that need to be set, for more information please refer to [config.py](./src/config.py).
 
-  | **Parameter**    | **Description**                                       | **Default setting** |
-  | ---------------- | ----------------------------------------------------- | ------------------- |
-  | MILVUS_HOST      | The IP address of Milvus, you can get it by ifconfig. If running everything on one machine, most likely 127.0.0.1 | 127.0.0.1           |
-  | MILVUS_PORT      | Port of Milvus.                                       | 19530               |
-  | VECTOR_DIMENSION | Dimension of the vectors.                             | 2048                |
-  | MYSQL_HOST       | The IP address of Mysql.                              | 127.0.0.1           |
-  | MYSQL_PORT       | Port of Milvus.                                       | 3306                |
-  | DEFAULT_TABLE    | The milvus and mysql default collection name.         | audio_table          |
+  | **Parameter**    |**Description**         | **Default setting** |
+  | ---------------- | -----------------------| ------------------- |
+  | MILVUS_HOST      | The IP address of Milvus, you can get it by ifconfig. If running everything on one machine, most likely 127.0.0.1 | 127.0.0.1
+  | MILVUS_PORT      | Port of Milvus.    | 19530               |
+  | VECTOR_DIMENSION | Dimension of the vectors.        | 2048          |
+  | MYSQL_HOST       | The IP address of Mysql.    | 127.0.0.1           |
+  | MYSQL_PORT       | Port of Mysql.        | 3306                |
+  | DEFAULT_TABLE    | The milvus and mysql default collection name.  | audio_table          |
 
 - Run the code
 
@@ -102,7 +111,13 @@ Then to start the system server, and it provides HTTP backend services.
   ```bash
   wget -c https://www.openslr.org/resources/82/cn-celeb_v2.tar.gz && tar -xvf cn-celeb_v2.tar.gz 
   ```
-  Note: If you want to build a quick demo, you can use ./src/test_main.py:download_audio_data function, it downloads 20 audio files , Subsequent results show this collection as an example
+  **Note**: If you want to build a quick demo, you can use ./src/test_main.py:download_audio_data function, it downloads 20 audio files , Subsequent results show this collection as an example
+
+- Prepare model(Skip this step if you use the default model.)
+  ```bash
+  ## Modify model configuration parameters. Currently, only ecapatdnn_voxceleb12 is supported, and multiple types will be supported in the future
+  vim ./src/encode.py
+  ```
  
 - Scripts test (Recommended)
 
@@ -179,7 +194,7 @@ Then to start the system server, and it provides HTTP backend services.
   
     Navigate to 127.0.0.1:8068 in your browser to access the front-end interface.
 
-    Note: If the browser and the service are not on the same machine, then the IP needs to be changed to the IP of the machine where the service is located, and the corresponding API_URL in docker-compose.yaml needs to be changed, and the docker-compose.yaml file needs to be re-executed for the change to take effect.
+    **Note**: If the browser and the service are not on the same machine, then the IP needs to be changed to the IP of the machine where the service is located, and the corresponding API_URL in docker-compose.yaml needs to be changed, and the docker-compose.yaml file needs to be re-executed for the change to take effect.
 
     - Insert data
 
@@ -218,6 +233,3 @@ Here is a list of pretrained models released by PaddleSpeech :
 | Model | Sample Rate
 | :--- | :---: 
 | ecapa_tdnn | 16000
-| panns_cnn6| 32000
-| panns_cnn10| 32000
-| panns_cnn14| 32000
diff --git a/demos/audio_searching/README_cn.md b/demos/audio_searching/README_cn.md
index c851bd0f..0d0f42a0 100644
--- a/demos/audio_searching/README_cn.md
+++ b/demos/audio_searching/README_cn.md
@@ -20,16 +20,20 @@
 ### 1. PaddleSpeech 安装
 音频向量的提取需要用到基于 PaddleSpeech 训练的模型，所以请确保在运行之前已经安装了 PaddleSpeech，具体安装步骤，详见[安装文档](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/docs/source/install_cn.md)。
 
-你可以从 easy，medium，hard 三中方式中选择一种方式安装。
+你可以从 easy，medium，hard 三种方式中选择一种方式安装。
 
 ### 2. MySQL 和 Milvus 安装
 音频相似性的检索需要用到 Milvus, MySQL 服务。 我们可以通过 [docker-compose.yaml](./docker-compose.yaml) 一键启动这些容器，所以请确保在运行之前已经安装了 [Docker Engine](https://docs.docker.com/engine/install/) 和 [Docker Compose](https://docs.docker.com/compose/install/)。 即
 
 ```bash
+## 先进入到 audio_searching 目录，如下示例
+cd ~/PaddleSpeech/demos/audio_searching/
+
+## 然后启动容器内的相关服务
 docker-compose -f docker-compose.yaml up -d
 ```
 
-然后你会看到所有的容器都被创建: 
+你会看到所有的容器都被创建:
 
 ```bash
 Creating network "quick_deploy_app_net" with driver "bridge"
@@ -48,7 +52,7 @@ b2bcf279e599  milvusdb/milvus:v2.0.1  "/tini -- milvus run…"  22 hours ago  Up
 d8ef4c84e25c  mysql:5.7 "docker-entrypoint.s…"  22 hours ago  Up 22 hours 0.0.0.0:3306->3306/tcp, 33060/tcp audio-mysql
 8fb501edb4f3  quay.io/coreos/etcd:v3.5.0  "etcd -advertise-cli…"  22 hours ago  Up 22 hours 2379-2380/tcp milvus-etcd
 ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"  22 hours ago  Up 22 hours (healthy) 9000/tcp  milvus-minio
-15c84a506754  qingen1/paddlespeech-audio-search-client:2.3  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
+15c84a506754  paddlepaddle/paddlespeech-audio-search-client:2.3  "/bin/bash -c '/usr/…"  22 hours ago  Up 22 hours (healthy) 0.0.0.0:8068->80/tcp  audio-webclient
 
 ```
 
@@ -60,22 +64,27 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
   ```bash
   pip install -r requirements.txt
   ```
-- 修改配置
+- 修改配置(本地运行情况下，一般不用修改，可以跳过该步骤)
 
   ```bash
+  ## 方法一：修改源码文件
   vim src/config.py
+
+  ## 方法二：修改环境变量，如下所示
+  export MILVUS_HOST=127.0.0.1
+  export MYSQL_HOST=127.0.0.1
   ```
 
-  请根据实际环境进行修改。 这里列出了一些需要设置的参数，更多信息请参考 [config.py](./src/config.py)  
+  这里列出了一些需要设置的参数，更多信息请参考 [config.py](./src/config.py)
 
-  | **Parameter**    | **Description**                                       | **Default setting** |
-  | ---------------- | ----------------------------------------------------- | ------------------- |
-  | MILVUS_HOST      | The IP address of Milvus, you can get it by ifconfig. If running everything on one machine, most likely 127.0.0.1 | 127.0.0.1           |
-  | MILVUS_PORT      | Port of Milvus.                                       | 19530               |
-  | VECTOR_DIMENSION | Dimension of the vectors.                             | 2048                |
-  | MYSQL_HOST       | The IP address of Mysql.                              | 127.0.0.1           |
-  | MYSQL_PORT       | Port of Milvus.                                       | 3306                |
-  | DEFAULT_TABLE    | The milvus and mysql default collection name.         | audio_table          |
+  | **参数**    | **描述**                | **默认设置** |
+  | ---------------- | -------------------- | ------------------- |
+  | MILVUS_HOST      | Milvus 服务的 IP 地址 | 127.0.0.1           |
+  | MILVUS_PORT      | Milvus 服务的端口号   | 19530               |
+  | VECTOR_DIMENSION | 特征向量的维度        | 192                 |
+  | MYSQL_HOST       | Mysql 服务的 IP 地址  | 127.0.0.1           |
+  | MYSQL_PORT       | Mysql 服务的端口号    | 3306                |
+  | DEFAULT_TABLE    | 默认存储的表名        | audio_table         |
 
 - 运行程序
 
@@ -104,7 +113,13 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
   ```bash
   wget -c https://www.openslr.org/resources/82/cn-celeb_v2.tar.gz && tar -xvf cn-celeb_v2.tar.gz 
   ```
-  注：如果希望快速搭建 demo，可以采用 ./src/test_main.py:download_audio_data 内部的 20 条音频，另外后续结果展示以该集合为例
+  **注**：如果希望快速搭建 demo，可以采用 ./src/test_main.py:download_audio_data 内部的 20 条音频，另外后续结果展示以该集合为例
+
+- 准备模型（如果使用默认模型，可以跳过此步骤）
+  ```bash
+  ## 修改模型配置参数，目前 model 仅支持 ecapatdnn_voxceleb12，后续将支持多种类型
+  vim ./src/encode.py
+  ```
  
  - 脚本测试（推荐）
 
@@ -182,7 +197,7 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
   
     在浏览器中输入 127.0.0.1:8068 访问前端页面
     
-    注：如果浏览器和服务不在同一台机器上，那么 IP 需要修改成服务所在的机器 IP，并且 docker-compose.yaml 中相应的 API_URL 也要修改，然后重新执行 docker-compose.yaml 文件，使修改生效。
+    **注**：如果浏览器和服务不在同一台机器上，那么 IP 需要修改成服务所在的机器 IP，并且 docker-compose.yaml 中相应的 API_URL 也要修改，然后重新执行 docker-compose.yaml 文件，使修改生效。
 
     - 上传音频
     
@@ -220,6 +235,3 @@ ffce340b3790  minio/minio:RELEASE.2020-12-03T00-03-10Z  "/usr/bin/docker-ent…"
 | 模型 | 采样率
 | :--- | :---: 
 | ecapa_tdnn| 16000
-| panns_cnn6| 32000
-| panns_cnn10| 32000
-| panns_cnn14| 32000
diff --git a/demos/audio_searching/src/encode.py b/demos/audio_searching/src/encode.py
index 35805784..f67184c2 100644
--- a/demos/audio_searching/src/encode.py
+++ b/demos/audio_searching/src/encode.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import numpy as np
-from logs import LOGGER
 
+from logs import LOGGER
 from paddlespeech.cli import VectorExecutor
 
 vector_executor = VectorExecutor()
@@ -24,7 +24,8 @@ def get_audio_embedding(path):
     Use vpr_inference to generate embedding of audio
     """
     try:
-        embedding = vector_executor(audio_file=path)
+        embedding = vector_executor(
+            audio_file=path, model='ecapatdnn_voxceleb12')
         embedding = embedding / np.linalg.norm(embedding)
         embedding = embedding.tolist()
         return embedding

From c11a5d97c2ca10c9d548f8741fc12d44593012db Mon Sep 17 00:00:00 2001
From: Yang Zhou <goat.zhou@qq.com>
Date: Tue, 5 Apr 2022 22:08:25 +0800
Subject: [PATCH 126/126] refactor linear feature:unify vector & remove
 redundant function

---
 .../examples/feat/linear_spectrogram_main.cc  |   4 +
 .../frontend/audio/linear_spectrogram.cc      | 147 ++++++------------
 .../frontend/audio/linear_spectrogram.h       |  12 +-
 3 files changed, 58 insertions(+), 105 deletions(-)

diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc
index ca76d85c..2d75bb5d 100644
--- a/speechx/examples/feat/linear_spectrogram_main.cc
+++ b/speechx/examples/feat/linear_spectrogram_main.cc
@@ -181,6 +181,10 @@ int main(int argc, char* argv[]) {
     ppspeech::LinearSpectrogramOptions opt;
     opt.frame_opts.frame_length_ms = 20;
     opt.frame_opts.frame_shift_ms = 10;
+    opt.frame_opts.dither = 0.0;
+    opt.frame_opts.remove_dc_offset = false;
+    opt.frame_opts.window_type = "hanning";
+    opt.frame_opts.preemph_coeff = 0.0;
     LOG(INFO) << "frame length (ms): " << opt.frame_opts.frame_length_ms;
     LOG(INFO) << "frame shift (ms): " << opt.frame_opts.frame_shift_ms;
 
diff --git a/speechx/speechx/frontend/audio/linear_spectrogram.cc b/speechx/speechx/frontend/audio/linear_spectrogram.cc
index 827b8ecc..d6ae3d01 100644
--- a/speechx/speechx/frontend/audio/linear_spectrogram.cc
+++ b/speechx/speechx/frontend/audio/linear_spectrogram.cc
@@ -14,6 +14,8 @@
 
 #include "frontend/audio/linear_spectrogram.h"
 #include "kaldi/base/kaldi-math.h"
+#include "kaldi/feat/feature-common.h"
+#include "kaldi/feat/feature-functions.h"
 #include "kaldi/matrix/matrix-functions.h"
 
 namespace ppspeech {
@@ -21,30 +23,23 @@ namespace ppspeech {
 using kaldi::int32;
 using kaldi::BaseFloat;
 using kaldi::Vector;
+using kaldi::SubVector;
 using kaldi::VectorBase;
 using kaldi::Matrix;
 using std::vector;
 
 LinearSpectrogram::LinearSpectrogram(
     const LinearSpectrogramOptions& opts,
-    std::unique_ptr<FrontendInterface> base_extractor) {
-    opts_ = opts;
+    std::unique_ptr<FrontendInterface> base_extractor)
+    : opts_(opts), feature_window_funtion_(opts.frame_opts) {
     base_extractor_ = std::move(base_extractor);
     int32 window_size = opts.frame_opts.WindowSize();
     int32 window_shift = opts.frame_opts.WindowShift();
-    fft_points_ = window_size;
+    dim_ = window_size / 2 + 1;
     chunk_sample_size_ =
         static_cast<int32>(opts.streaming_chunk * opts.frame_opts.samp_freq);
-    hanning_window_.resize(window_size);
-
-    double a = M_2PI / (window_size - 1);
-    hanning_window_energy_ = 0;
-    for (int i = 0; i < window_size; ++i) {
-        hanning_window_[i] = 0.5 - 0.5 * cos(a * i);
-        hanning_window_energy_ += hanning_window_[i] * hanning_window_[i];
-    }
-
-    dim_ = fft_points_ / 2 + 1;  // the dimension is Fs/2 Hz
+    hanning_window_energy_ = kaldi::VecVec(feature_window_funtion_.window,
+                                           feature_window_funtion_.window);
 }
 
 void LinearSpectrogram::Accept(const VectorBase<BaseFloat>& inputs) {
@@ -56,99 +51,57 @@ bool LinearSpectrogram::Read(Vector<BaseFloat>* feats) {
     bool flag = base_extractor_->Read(&input_feats);
     if (flag == false || input_feats.Dim() == 0) return false;
 
-    vector<BaseFloat> input_feats_vec(input_feats.Dim());
-    std::memcpy(input_feats_vec.data(),
-                input_feats.Data(),
-                input_feats.Dim() * sizeof(BaseFloat));
-    vector<vector<BaseFloat>> result;
-    Compute(input_feats_vec, result);
-    int32 feat_size = 0;
-    if (result.size() != 0) {
-        feat_size = result.size() * result[0].size();
-    }
-    feats->Resize(feat_size);
-    // todo refactor (SimleGoat)
-    for (size_t idx = 0; idx < feat_size; ++idx) {
-        (*feats)(idx) = result[idx / dim_][idx % dim_];
-    }
-    return true;
-}
-
-void LinearSpectrogram::Hanning(vector<float>* data) const {
-    CHECK_GE(data->size(), hanning_window_.size());
-
-    for (size_t i = 0; i < hanning_window_.size(); ++i) {
-        data->at(i) *= hanning_window_[i];
-    }
-}
-
-bool LinearSpectrogram::NumpyFft(vector<BaseFloat>* v,
-                                 vector<BaseFloat>* real,
-                                 vector<BaseFloat>* img) const {
-    Vector<BaseFloat> v_tmp;
-    v_tmp.Resize(v->size());
-    std::memcpy(v_tmp.Data(), v->data(), sizeof(BaseFloat) * (v->size()));
-    RealFft(&v_tmp, true);
-    v->resize(v_tmp.Dim());
-    std::memcpy(v->data(), v_tmp.Data(), sizeof(BaseFloat) * (v->size()));
-
-    real->push_back(v->at(0));
-    img->push_back(0);
-    for (int i = 1; i < v->size() / 2; i++) {
-        real->push_back(v->at(2 * i));
-        img->push_back(v->at(2 * i + 1));
-    }
-    real->push_back(v->at(1));
-    img->push_back(0);
-
+    int32 feat_len = input_feats.Dim();
+    int32 left_len = reminded_wav_.Dim();
+    Vector<BaseFloat> waves(feat_len + left_len);
+    waves.Range(0, left_len).CopyFromVec(reminded_wav_);
+    waves.Range(left_len, feat_len).CopyFromVec(input_feats);
+    Compute(waves, feats);
+    int32 frame_shift = opts_.frame_opts.WindowShift();
+    int32 num_frames = kaldi::NumFrames(waves.Dim(), opts_.frame_opts);
+    int32 left_samples = waves.Dim() - frame_shift * num_frames;
+    reminded_wav_.Resize(left_samples);
+    reminded_wav_.CopyFromVec(
+        waves.Range(frame_shift * num_frames, left_samples));
     return true;
 }
 
 // Compute spectrogram feat
-// todo: refactor later (SmileGoat)
-bool LinearSpectrogram::Compute(const vector<float>& waves,
-                                vector<vector<float>>& feats) {
-    int num_samples = waves.size();
-    const int& frame_length = opts_.frame_opts.WindowSize();
-    const int& sample_rate = opts_.frame_opts.samp_freq;
-    const int& frame_shift = opts_.frame_opts.WindowShift();
-    const int& fft_points = fft_points_;
-    const float scale = hanning_window_energy_ * sample_rate;
+bool LinearSpectrogram::Compute(const Vector<BaseFloat>& waves,
+                                Vector<BaseFloat>* feats) {
+    int32 num_samples = waves.Dim();
+    int32 frame_length = opts_.frame_opts.WindowSize();
+    int32 sample_rate = opts_.frame_opts.samp_freq;
+    BaseFloat scale = 2.0 / (hanning_window_energy_ * sample_rate);
 
     if (num_samples < frame_length) {
         return true;
     }
 
-    int num_frames = 1 + ((num_samples - frame_length) / frame_shift);
-    feats.resize(num_frames);
-    vector<float> fft_real((fft_points_ / 2 + 1), 0);
-    vector<float> fft_img((fft_points_ / 2 + 1), 0);
-    vector<float> v(frame_length, 0);
-    vector<float> power((fft_points / 2 + 1));
-
-    for (int i = 0; i < num_frames; ++i) {
-        vector<float> data(waves.data() + i * frame_shift,
-                           waves.data() + i * frame_shift + frame_length);
-        Hanning(&data);
-        fft_img.clear();
-        fft_real.clear();
-        v.assign(data.begin(), data.end());
-        NumpyFft(&v, &fft_real, &fft_img);
-
-        feats[i].resize(fft_points / 2 + 1);  // the last dimension is Fs/2 Hz
-        for (int j = 0; j < (fft_points / 2 + 1); ++j) {
-            power[j] = fft_real[j] * fft_real[j] + fft_img[j] * fft_img[j];
-            feats[i][j] = power[j];
-
-            if (j == 0 || j == feats[0].size() - 1) {
-                feats[i][j] /= scale;
-            } else {
-                feats[i][j] *= (2.0 / scale);
-            }
-
-            // log added eps=1e-14
-            feats[i][j] = std::log(feats[i][j] + 1e-14);
-        }
+    int32 num_frames = kaldi::NumFrames(num_samples, opts_.frame_opts);
+    feats->Resize(num_frames * dim_);
+    Vector<BaseFloat> window;
+
+    for (int frame_idx = 0; frame_idx < num_frames; ++frame_idx) {
+        kaldi::ExtractWindow(0,
+                             waves,
+                             frame_idx,
+                             opts_.frame_opts,
+                             feature_window_funtion_,
+                             &window,
+                             NULL);
+
+        SubVector<BaseFloat> output_row(feats->Data() + frame_idx * dim_, dim_);
+        window.Resize(frame_length, kaldi::kCopyData);
+        RealFft(&window, true);
+        kaldi::ComputePowerSpectrum(&window);
+        SubVector<BaseFloat> power_spectrum(window, 0, dim_);
+        power_spectrum.Scale(scale);
+        power_spectrum(0) = power_spectrum(0) / 2;
+        power_spectrum(dim_ - 1) = power_spectrum(dim_ - 1) / 2;
+        power_spectrum.Add(1e-14);
+        power_spectrum.ApplyLog();
+        output_row.CopyFromVec(power_spectrum);
     }
     return true;
 }
diff --git a/speechx/speechx/frontend/audio/linear_spectrogram.h b/speechx/speechx/frontend/audio/linear_spectrogram.h
index bbf8d685..896c494d 100644
--- a/speechx/speechx/frontend/audio/linear_spectrogram.h
+++ b/speechx/speechx/frontend/audio/linear_spectrogram.h
@@ -49,19 +49,15 @@ class LinearSpectrogram : public FrontendInterface {
     virtual void Reset() { base_extractor_->Reset(); }
 
   private:
-    void Hanning(std::vector<kaldi::BaseFloat>* data) const;
-    bool Compute(const std::vector<kaldi::BaseFloat>& waves,
-                 std::vector<std::vector<kaldi::BaseFloat>>& feats);
-    bool NumpyFft(std::vector<kaldi::BaseFloat>* v,
-                  std::vector<kaldi::BaseFloat>* real,
-                  std::vector<kaldi::BaseFloat>* img) const;
+    bool Compute(const kaldi::Vector<kaldi::BaseFloat>& waves,
+                 kaldi::Vector<kaldi::BaseFloat>* feats);
 
-    kaldi::int32 fft_points_;
     size_t dim_;
-    std::vector<kaldi::BaseFloat> hanning_window_;
+    kaldi::FeatureWindowFunction feature_window_funtion_;
     kaldi::BaseFloat hanning_window_energy_;
     LinearSpectrogramOptions opts_;
     std::unique_ptr<FrontendInterface> base_extractor_;
+    kaldi::Vector<kaldi::BaseFloat> reminded_wav_;
     int chunk_sample_size_;
     DISALLOW_COPY_AND_ASSIGN(LinearSpectrogram);
 };