From e84690f6f005ae8e6b7f123bfeb2cd9d304c9838 Mon Sep 17 00:00:00 2001 From: huangyuxin Date: Mon, 15 Nov 2021 10:12:34 +0000 Subject: [PATCH 1/2] add conda install pipline --- docs/source/install.md | 10 ++++++++++ requirements.txt | 6 ++++++ setup.sh | 20 ++++++++++++++++++++ tools/extras/install_miniconda.sh | 4 ++-- 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 setup.sh diff --git a/docs/source/install.md b/docs/source/install.md index 0700a166..f937994d 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -16,6 +16,16 @@ cd DeepSpeech pip install -e . ``` +For user who only needs the basic function of paddlespeech, using conda to do installing is recommended. +```python +pushd tools +bash extras/install_miniconda.sh +popd +source ~/.bashrc +bash setup.sh +``` + + ## Setup (Other Platform) - Make sure these libraries or tools in [dependencies](./dependencies.md) installed. More information please see: `setup.py `and ` tools/Makefile`. diff --git a/requirements.txt b/requirements.txt index 2b34d36b..83b91bac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,3 +44,9 @@ visualdl==2.2.0 webrtcvad yacs yq +pypi-kenlm +GPUtil +psutil +pynvml +distro + diff --git a/setup.sh b/setup.sh new file mode 100644 index 00000000..c28b050a --- /dev/null +++ b/setup.sh @@ -0,0 +1,20 @@ +# Install conda dependencies +conda install -c conda-forge sox libsndfile swig bzip2 bottleneck gcc_linux-64=7.5.0 gxx_linux-64=7.5.0 --yes + +# Install the python lib +pip install -r requirements.txt + +# Install the auto_log +pushd tools/extras +bash install_autolog.sh +popd + +# Install the ctcdecoder +pushd paddlespeech/s2t/decoders/ctcdecoder/swig +bash -e setup.sh +popd + +# Install the python_speech_features +pushd third_party +bash -e install.sh +popd diff --git a/tools/extras/install_miniconda.sh b/tools/extras/install_miniconda.sh index 3d1909af..97829f0b 100755 --- a/tools/extras/install_miniconda.sh +++ b/tools/extras/install_miniconda.sh @@ -9,9 +9,9 @@ WGET=${WGET:-wget} if [ -d "$DOWNLOAD_DIR" ]; then cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1 else - $WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1 + $WGET https://repo.anaconda.com/miniconda/Miniconda3-py37_4.10.3-Linux-x86_64.sh || exit 1 fi -bash Miniconda3-latest-Linux-x86_64.sh -b +bash Miniconda3-py37_4.10.3-Linux-x86_64.sh -b $HOME/miniconda3/bin/python -m pip install --user tqdm $HOME/miniconda3/bin/python -m pip install --user scikit-learn From 2f92a5d9a7e384d26081cdcd6102314329ee3baf Mon Sep 17 00:00:00 2001 From: huangyuxin Date: Mon, 15 Nov 2021 12:29:48 +0000 Subject: [PATCH 2/2] add conda init, use gcc 8.4.0 --- docs/source/install.md | 8 +++++++- requirements.txt | 2 +- setup.sh | 2 +- tools/extras/install_miniconda.sh | 6 ++++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/source/install.md b/docs/source/install.md index f937994d..d68b990d 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -17,11 +17,17 @@ pip install -e . ``` For user who only needs the basic function of paddlespeech, using conda to do installing is recommended. +You can go to [minicoda](https://docs.conda.io/en/latest/miniconda.html) to select a version and install it by yourself, or you can use the scripts below to install the last miniconda version. + ```python pushd tools bash extras/install_miniconda.sh popd -source ~/.bashrc +bash +``` + +After installing the conda, run the setup.sh to complete the installing process. +```python bash setup.sh ``` diff --git a/requirements.txt b/requirements.txt index 83b91bac..8e2552e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ python-dateutil pyworld resampy==0.2.2 sacrebleu -scipy==1.2.1 +scipy sentencepiece snakeviz soundfile~=0.10 diff --git a/setup.sh b/setup.sh index c28b050a..0bfacb54 100644 --- a/setup.sh +++ b/setup.sh @@ -1,5 +1,5 @@ # Install conda dependencies -conda install -c conda-forge sox libsndfile swig bzip2 bottleneck gcc_linux-64=7.5.0 gxx_linux-64=7.5.0 --yes +conda install -c conda-forge sox libsndfile swig bzip2 bottleneck gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 --yes # Install the python lib pip install -r requirements.txt diff --git a/tools/extras/install_miniconda.sh b/tools/extras/install_miniconda.sh index 97829f0b..c6ee4b36 100755 --- a/tools/extras/install_miniconda.sh +++ b/tools/extras/install_miniconda.sh @@ -9,9 +9,11 @@ WGET=${WGET:-wget} if [ -d "$DOWNLOAD_DIR" ]; then cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1 else - $WGET https://repo.anaconda.com/miniconda/Miniconda3-py37_4.10.3-Linux-x86_64.sh || exit 1 + $WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1 fi -bash Miniconda3-py37_4.10.3-Linux-x86_64.sh -b +bash Miniconda3-latest-Linux-x86_64.sh -b + +$HOME/miniconda3/bin/conda init $HOME/miniconda3/bin/python -m pip install --user tqdm $HOME/miniconda3/bin/python -m pip install --user scikit-learn