diff --git a/.gitignore b/.gitignore index 1724bd43..8cbb734d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,8 @@ tools/sctk tools/sctk-20159b5/ tools/kaldi tools/OpenBLAS/ +tools/Miniconda3-latest-Linux-x86_64.sh +tools/activate_python.sh +tools/miniconda.sh *output/ diff --git a/docs/source/install.md b/docs/source/install.md index d276759b..0fe71b99 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -41,19 +41,19 @@ bash Then you can create an conda virtual environment using the script: ```bash -conda create -n py37 python=3.7 +conda create -y -p tools/venv python=3.7 ``` Activate the conda virtual environment: ```bash -conda activate py37 +conda activate tools/venv ``` Intall the conda dependencies ```bash -conda install -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 --yes +conda install -y -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 ``` ### Install PaddlePaddle @@ -150,11 +150,11 @@ popd # use the "bash" command to make the conda environment works bash # create an conda virtual environment -conda create -n py37 python=3.7 +conda create -y -n tools/venv python=3.7 # Activate the conda virtual environment: -conda activate py37 +conda activate tools/venv # Install the conda packags -conda install -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 --yes +conda install -y -c conda-forge sox libsndfile swig bzip2 libflac bc gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 ``` ### Install PaddlePaddle @@ -168,7 +168,7 @@ python3 -m pip install paddlepaddle-gpu==2.2.0 ### Get the Funcition for Developing PaddleSpeech ```bash -pip install -e .[develop] +pip install .[develop] ``` ### Install the Kaldi (Optional) @@ -181,11 +181,9 @@ popd ``` - - ## Setup for Other Platform - Make sure these libraries or tools in [dependencies](./dependencies.md) installed. More information please see: `setup.py `and ` tools/Makefile`. - The version of `swig` should >= 3.0 - we will do more to simplify the install process. -- Install Paddlespeech \ No newline at end of file +- Install Paddlespeech diff --git a/examples/librispeech/asr1/RESULTS.md b/examples/librispeech/asr1/RESULTS.md index a5257c9d..20cb3f8c 100644 --- a/examples/librispeech/asr1/RESULTS.md +++ b/examples/librispeech/asr1/RESULTS.md @@ -24,4 +24,4 @@ | transformer | 32.52 M | conf/transformer.yaml | spec_aug | test-clean | attention | 6.484564081827799 | 0.044355 | | transformer | 32.52 M | conf/transformer.yaml | spec_aug | test-clean | ctc_greedy_search | 6.484564081827799 | 0.050479 | | transformer | 32.52 M | conf/transformer.yaml | spec_aug | test-clean | ctc_prefix_beam_search | 6.484564081827799 | 0.049890 | -| transformer | 32.52 M | conf/transformer.yaml | spec_aug | test-clean | attention_rescoring | 6.484564081827799 | 0.039200 | \ No newline at end of file +| transformer | 32.52 M | conf/transformer.yaml | spec_aug | test-clean | attention_rescoring | 6.484564081827799 | 0.039200 | diff --git a/setup.py b/setup.py index ca5b3589..786cfc9e 100644 --- a/setup.py +++ b/setup.py @@ -130,6 +130,7 @@ def _post_install(install_lib_dir): check_call("bash -e setup.sh") print("ctcdecoder install.") + class DevelopCommand(develop): def run(self): develop.run(self) @@ -213,8 +214,7 @@ setup_info = dict( }, # Package info - packages=find_packages(exclude=('utils', 'tests', 'tests.*', 'examples*', - 'paddleaudio*', 'third_party*', 'tools*')), + packages=find_packages(include=('paddlespeech')), zip_safe=True, classifiers=[ 'Development Status :: 3 - Alpha', diff --git a/tools/Makefile b/tools/Makefile index 5b2cbe8d..285f85c8 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -8,9 +8,10 @@ CC ?= gcc # used for sph2pipe WGET ?= wget --no-check-certificate -.PHONY: all clean +.PHONY: all clean + +all: apt.done kenlm.done mfa.done sctk.done -all: apt.done kenlm.done mfa.done sclite.done virtualenv.done: test -d venv || virtualenv -p $(PYTHON) venv @@ -28,6 +29,7 @@ apt.done: echo "check_certificate = off" >> ~/.wgetrc touch apt.done + kenlm.done: # Ubuntu 16.04 透過 apt 會安裝 boost 1.58.0 # it seems that boost (1.54.0) requires higher version. After I switched to g++-5 it compiles normally. @@ -38,12 +40,6 @@ kenlm.done: cd kenlm && python3 setup.py install touch kenlm.done -sox.done: - apt install -y libvorbis-dev libmp3lame-dev libmad-ocaml-dev - test -d sox-14.4.2 || $(WGET) https://nchc.dl.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.gz - tar -xvzf sox-14.4.2.tar.gz -C . - cd sox-14.4.2 && ./configure --prefix=/usr/ && make -j4 && make install - touch sox.done mfa.done: test -d montreal-forced-aligner || $(WGET) https://paddlespeech.bj.bcebos.com/Parakeet/montreal-forced-aligner_linux.tar.gz @@ -58,48 +54,46 @@ kaldi.done: openblas.done bash extras/install_kaldi.sh touch kaldi.done -#== SCTK =============================================================================== -# SCTK official repo does not have version tags. Here's the mapping: -# # 2.4.9 = 659bc36; 2.4.10 = d914e1b; 2.4.11 = 20159b5. -SCTK_GITHASH = 20159b5 - -SCTK_CXFLAGS = -w -march=native -SCTK_MKENV = CFLAGS="$(CFLAGS) $(SCTK_CXFLAGS)" \ - CXXFLAGS="$(CXXFLAGS) -std=c++11 $(SCTK_CXFLAGS)" \ - - -# Keep the existing target 'sclite' to avoid breaking the users who might have -# scripted it in. -.PHONY: sclite.done sctk_cleaned sctk_made - -sclite.done sctk_made: sctk/.compiled - touch sclite.done - -sctk/.compiled: sctk - rm -f sctk/.compiled - $(SCTK_MKENV) $(MAKE) -C sctk config - $(SCTK_MKENV) $(MAKE) -C sctk all doc - $(MAKE) -C sctk install - touch sctk/.compiled - -# The GitHub archive unpacks into SCTK-{40-character-long-hash}/ -sctk: sctk-$(SCTK_GITHASH).tar.gz - tar zxvf sctk-$(SCTK_GITHASH).tar.gz - rm -rf sctk-$(SCTK_GITHASH) sctk - mv SCTK-$(SCTK_GITHASH)* sctk-$(SCTK_GITHASH) - ln -s sctk-$(SCTK_GITHASH) sctk - touch sctk-$(SCTK_GITHASH).tar.gz - -sctk-$(SCTK_GITHASH).tar.gz: - if [ -d '$(DOWNLOAD_DIR)' ]; then \ - cp -p '$(DOWNLOAD_DIR)/sctk-$(SCTK_GITHASH).tar.gz' .; \ - else \ - $(WGET) -nv -T 10 -t 3 -O sctk-$(SCTK_GITHASH).tar.gz \ - https://github.com/usnistgov/SCTK/archive/$(SCTK_GITHASH).tar.gz; \ - fi - -sctk_cleaned: - -for d in sctk/ sctk-*/; do \ - [ ! -f $$d/.compiled ] || $(MAKE) -C $$d clean; \ - rm -f $$d/.compiled; \ - done +sctk.done: + ./extras/install_sclite.sh + touch sctk.done + +###################### +dev: python conda_packages.done sctk.done + +# Use pip for paddle installation even if you have anaconda +ifneq ($(shell test -f ./activate_python.sh && grep 'conda activate' ./activate_python.sh),) +USE_CONDA := 1 +else +USE_CONDA := +endif + +python: activate_python.sh + +activate_python.sh: + test -f activate_python.sh || { echo "Error: Run ./setup_python.sh or ./setup_anaconda.sh"; exit 1; } + +bc.done: activate_python.sh + . ./activate_python.sh && { command -v bc || conda install -y bc -c conda-forge; } + touch bc.done +cmake.done: activate_python.sh + . ./activate_python.sh && { command -v cmake || conda install -y cmake; } + touch cmake.done +flac.done: activate_python.sh + . ./activate_python.sh && { command -v flac || conda install -y libflac -c conda-forge; } + touch flac.done +ffmpeg.done: activate_python.sh + . ./activate_python.sh && { command -v ffmpeg || conda install -y ffmpeg -c conda-forge; } + touch ffmpeg.done +sox.done: activate_python.sh + . ./activate_python.sh && { command -v sox || conda install -y sox -c conda-forge; } + touch sox.done +sndfile.done: activate_python.sh + . ./activate_python.sh && { python3 -c "from ctypes.util import find_library as F; assert F('sndfile') is not None" || conda install -y libsndfile=1.0.28 -c conda-forge; } + touch sndfile.done +ifneq ($(strip $(USE_CONDA)),) +conda_packages.done: bc.done cmake.done flac.done ffmpeg.done sox.done sndfile.done +else +conda_packages.done: +endif + touch conda_packages.done \ No newline at end of file diff --git a/tools/extras/install_miniconda.sh b/tools/extras/install_miniconda.sh index c6ee4b36..e798e22d 100755 --- a/tools/extras/install_miniconda.sh +++ b/tools/extras/install_miniconda.sh @@ -9,7 +9,7 @@ WGET=${WGET:-wget} if [ -d "$DOWNLOAD_DIR" ]; then cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1 else - $WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1 + $WGET -c https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1 fi bash Miniconda3-latest-Linux-x86_64.sh -b diff --git a/tools/setup_anaconda.sh b/tools/setup_anaconda.sh new file mode 100755 index 00000000..1ed4dd02 --- /dev/null +++ b/tools/setup_anaconda.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ -z "${PS1:-}" ]; then + PS1=__dummy__ +fi +CONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + +if [ $# -gt 4 ]; then + echo "Usage: $0 [output] [conda-env-name] [python-version>]" + exit 1; +elif [ $# -eq 3 ]; then + output_dir="$1" + name="$2" + PYTHON_VERSION="$3" +elif [ $# -eq 2 ]; then + output_dir="$1" + name="$2" + PYTHON_VERSION="" +elif [ $# -eq 1 ]; then + output_dir="$1" + name="" + PYTHON_VERSION="" +elif [ $# -eq 0 ]; then + output_dir=venv + name="" + PYTHON_VERSION="" +fi + +if [ -e activate_python.sh ]; then + echo "Warning: activate_python.sh already exists. It will be overwritten" +fi + +if [ ! -e "${output_dir}/etc/profile.d/conda.sh" ]; then + if [ ! -e miniconda.sh ]; then + wget --tries=3 "${CONDA_URL}" -O miniconda.sh + fi + + bash miniconda.sh -b -p "${output_dir}" +fi + +# shellcheck disable=SC1090 +source "${output_dir}/etc/profile.d/conda.sh" +conda deactivate + +# If the env already exists, skip recreation +if [ -n "${name}" ] && ! conda activate ${name}; then + conda create -yn "${name}" +fi +conda activate ${name} + +if [ -n "${PYTHON_VERSION}" ]; then + conda install -y conda "python=${PYTHON_VERSION}" +else + conda install -y conda +fi + +conda install -y pip setuptools + +cat << EOF > activate_python.sh +#!/usr/bin/env bash +# THIS FILE IS GENERATED BY tools/setup_anaconda.sh +if [ -z "\${PS1:-}" ]; then + PS1=__dummy__ +fi +. $(cd ${output_dir}; pwd)/etc/profile.d/conda.sh && conda deactivate && conda activate ${name} +EOF \ No newline at end of file