Merge pull request #1069 from zh794390558/install

[install] install with conda
3 years ago · a439a5b2eb
parent b5527762b5 72ef492f21
commit a439a5b2eb
7 changed files with 131 additions and 64 deletions
--- a/.gitignore
+++ b/.gitignore
@ -26,5 +26,8 @@ tools/sctk
 tools/sctk-20159b5/
 tools/kaldi
 tools/OpenBLAS/
+tools/Miniconda3-latest-Linux-x86_64.sh
+tools/activate_python.sh
+tools/miniconda.sh

 *output/
--- a/docs/source/install.md
+++ b/docs/source/install.md
@ -27,15 +27,15 @@ bash
 ```
 Then you can create an conda virtual environment using the following command:
 ```bash
-conda create -n py37 python=3.7
+conda create -y -p tools/venv python=3.7
 ```
 Activate the conda virtual environment:
 ```bash
-conda activate py37
+conda activate tools/venv
 ```
 Install  conda dependencies for `paddlespeech` :
 ```bash
-conda install -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 --yes
+conda install -y -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0
 ```
 ### Install PaddlePaddle
 For example, for CUDA 10.2, CuDNN7.5 install paddle 2.2.0:
@ -103,11 +103,11 @@ popd
 # use the "bash" command to make the conda environment works
 bash
 # create an conda virtual environment
-conda create -n py37 python=3.7
+conda create -y -n tools/venv python=3.7
 # Activate the conda virtual environment:
-conda activate py37
+conda activate tools/venv
 # Install the conda packags
-conda install -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 --yes
+conda install -y -c conda-forge sox libsndfile swig bzip2 libflac bc gcc_linux-64=8.4.0 gxx_linux-64=8.4.0
 ```
 ### Install PaddlePaddle
 For example, for CUDA 10.2, CuDNN7.5 install paddle 2.2.0:
@ -117,7 +117,7 @@ python3 -m pip install paddlepaddle-gpu==2.2.0
 ```
 ### Get the Funcition for Developing PaddleSpeech
 ```bash
-pip install -e .[develop]
+pip install .[develop]
 ```
 ### Install the Kaldi (Optional)
 ```bash
@ -126,7 +126,10 @@ bash extras/install_openblas.sh
 bash extras/install_kaldi.sh
 popd
 ```
+
+
 ## Setup for Other Platform 
 - Make sure these libraries or tools in [dependencies](./dependencies.md) installed. More information please see: `setup.py `and ` tools/Makefile`.
 - The version of `swig` should >= 3.0
 - we will simplify the install process in the future.
+
--- a/examples/librispeech/asr1/RESULTS.md
+++ b/examples/librispeech/asr1/RESULTS.md
@ -24,4 +24,4 @@
 | transformer | 32.52 M | conf/transformer.yaml | spec_aug  | test-clean | attention | 6.484564081827799 | 0.044355 |  
 | transformer | 32.52 M | conf/transformer.yaml | spec_aug  | test-clean | ctc_greedy_search | 6.484564081827799 | 0.050479 |  
 | transformer | 32.52 M | conf/transformer.yaml | spec_aug  | test-clean | ctc_prefix_beam_search | 6.484564081827799 | 0.049890 |  
-| transformer | 32.52 M | conf/transformer.yaml | spec_aug  | test-clean | attention_rescoring | 6.484564081827799 | 0.039200 |  
+| transformer | 32.52 M | conf/transformer.yaml | spec_aug  | test-clean | attention_rescoring | 6.484564081827799 | 0.039200 |  
--- a/setup.py
+++ b/setup.py
@ -130,6 +130,7 @@ def _post_install(install_lib_dir):
        check_call("bash -e setup.sh")
    print("ctcdecoder install.")

+
 class DevelopCommand(develop):
    def run(self):
        develop.run(self)
@ -213,8 +214,7 @@ setup_info = dict(
    },

    # Package info
-    packages=find_packages(exclude=('utils', 'tests', 'tests.*', 'examples*',
-                                    'paddleaudio*', 'third_party*', 'tools*')),
+    packages=find_packages(include=('paddlespeech')),
    zip_safe=True,
    classifiers=[
        'Development Status :: 3 - Alpha',
--- a/tools/Makefile
+++ b/tools/Makefile
@ -8,9 +8,10 @@ CC ?= gcc        # used for sph2pipe

 WGET ?= wget --no-check-certificate

-.PHONY: all clean
+.PHONY: all clean 
+
+all: apt.done kenlm.done mfa.done sctk.done

-all: apt.done kenlm.done mfa.done sclite.done

 virtualenv.done:
 	test -d venv || virtualenv -p $(PYTHON) venv
@ -28,6 +29,7 @@ apt.done:
 	echo "check_certificate = off" >> ~/.wgetrc
 	touch apt.done

+
 kenlm.done:
 	# Ubuntu 16.04 透過 apt 會安裝 boost 1.58.0
 	# it seems that boost (1.54.0) requires higher version. After I switched to g++-5 it compiles normally.
@ -38,12 +40,6 @@ kenlm.done:
 	cd kenlm && python3 setup.py install
 	touch kenlm.done

-sox.done:
-	apt install -y libvorbis-dev libmp3lame-dev libmad-ocaml-dev
-	test -d sox-14.4.2 || $(WGET) https://nchc.dl.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.gz
-	tar -xvzf sox-14.4.2.tar.gz -C .
-	cd sox-14.4.2 && ./configure --prefix=/usr/ && make -j4 && make install
-	touch sox.done

 mfa.done:
 	test -d montreal-forced-aligner || $(WGET) https://paddlespeech.bj.bcebos.com/Parakeet/montreal-forced-aligner_linux.tar.gz
@ -58,48 +54,46 @@ kaldi.done: openblas.done
 	bash extras/install_kaldi.sh
 	touch kaldi.done

-#== SCTK ===============================================================================
-# SCTK official repo does not have version tags. Here's the mapping:
-# # 2.4.9 = 659bc36; 2.4.10 = d914e1b; 2.4.11 = 20159b5.
-SCTK_GITHASH = 20159b5
-
-SCTK_CXFLAGS = -w -march=native
-SCTK_MKENV = CFLAGS="$(CFLAGS) $(SCTK_CXFLAGS)" \
-			              CXXFLAGS="$(CXXFLAGS) -std=c++11 $(SCTK_CXFLAGS)" \
-
-
-# Keep the existing target 'sclite' to avoid breaking the users who might have
-# scripted it in.
-.PHONY: sclite.done sctk_cleaned sctk_made
-
-sclite.done sctk_made: sctk/.compiled
-	touch sclite.done
-
-sctk/.compiled: sctk
-	rm -f sctk/.compiled
-	$(SCTK_MKENV) $(MAKE) -C sctk config
-	$(SCTK_MKENV) $(MAKE) -C sctk all doc
-	$(MAKE) -C sctk install
-	touch sctk/.compiled
-
-# The GitHub archive unpacks into SCTK-{40-character-long-hash}/
-sctk: sctk-$(SCTK_GITHASH).tar.gz
-	tar zxvf sctk-$(SCTK_GITHASH).tar.gz
-	rm -rf sctk-$(SCTK_GITHASH) sctk
-	mv SCTK-$(SCTK_GITHASH)* sctk-$(SCTK_GITHASH)
-	ln -s sctk-$(SCTK_GITHASH) sctk
-	touch sctk-$(SCTK_GITHASH).tar.gz
-
-sctk-$(SCTK_GITHASH).tar.gz:
-	if [ -d '$(DOWNLOAD_DIR)' ]; then \
-	  cp -p '$(DOWNLOAD_DIR)/sctk-$(SCTK_GITHASH).tar.gz' .; \
-	else \
-	  $(WGET) -nv -T 10 -t 3 -O sctk-$(SCTK_GITHASH).tar.gz \
-	    https://github.com/usnistgov/SCTK/archive/$(SCTK_GITHASH).tar.gz; \
-	fi
-
-sctk_cleaned:
-	-for d in sctk/ sctk-*/; do \
-	   [ ! -f $$d/.compiled ] || $(MAKE) -C $$d clean; \
-	   rm -f $$d/.compiled; \
-	done
+sctk.done:
+	./extras/install_sclite.sh
+	touch sctk.done
+
+######################
+dev: python conda_packages.done sctk.done
+
+# Use pip for paddle installation even if you have anaconda
+ifneq ($(shell test -f ./activate_python.sh && grep 'conda activate' ./activate_python.sh),)
+USE_CONDA := 1
+else
+USE_CONDA :=
+endif
+
+python: activate_python.sh 
+
+activate_python.sh:
+	test -f activate_python.sh || { echo "Error: Run ./setup_python.sh or ./setup_anaconda.sh"; exit 1; }
+
+bc.done: activate_python.sh
+	. ./activate_python.sh && { command -v bc || conda install -y bc -c conda-forge; }
+	touch bc.done
+cmake.done: activate_python.sh
+	. ./activate_python.sh && { command -v cmake || conda install -y cmake; }
+	touch cmake.done
+flac.done: activate_python.sh
+	. ./activate_python.sh && { command -v flac || conda install -y libflac -c conda-forge; }
+	touch flac.done
+ffmpeg.done: activate_python.sh
+	. ./activate_python.sh && { command -v ffmpeg || conda install -y ffmpeg -c conda-forge; }
+	touch ffmpeg.done
+sox.done: activate_python.sh
+	. ./activate_python.sh && { command -v sox || conda install -y sox -c conda-forge; }
+	touch sox.done
+sndfile.done: activate_python.sh
+	. ./activate_python.sh && { python3 -c "from ctypes.util import find_library as F; assert F('sndfile') is not None" || conda install -y libsndfile=1.0.28 -c conda-forge; }
+	touch sndfile.done
+ifneq ($(strip $(USE_CONDA)),)
+conda_packages.done: bc.done cmake.done flac.done ffmpeg.done sox.done sndfile.done
+else
+conda_packages.done:
+endif
+	touch conda_packages.done
--- a/tools/extras/install_miniconda.sh
+++ b/tools/extras/install_miniconda.sh
@ -9,7 +9,7 @@ WGET=${WGET:-wget}
 if [ -d "$DOWNLOAD_DIR" ]; then
    cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1
 else
-    $WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1
+    $WGET -c https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1
 fi
 bash Miniconda3-latest-Linux-x86_64.sh -b

--- a/tools/setup_anaconda.sh
+++ b/tools/setup_anaconda.sh
@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -z "${PS1:-}" ]; then
+    PS1=__dummy__
+fi
+CONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+
+if [ $# -gt 4 ]; then
+    echo "Usage: $0 [output] [conda-env-name] [python-version>]"
+    exit 1;
+elif [ $# -eq 3 ]; then
+    output_dir="$1"
+    name="$2"
+    PYTHON_VERSION="$3"
+elif [ $# -eq 2 ]; then
+    output_dir="$1"
+    name="$2"
+    PYTHON_VERSION=""
+elif [ $# -eq 1 ]; then
+    output_dir="$1"
+    name=""
+    PYTHON_VERSION=""
+elif [ $# -eq 0 ]; then
+    output_dir=venv
+    name=""
+    PYTHON_VERSION=""
+fi
+
+if [ -e activate_python.sh ]; then
+    echo "Warning: activate_python.sh already exists. It will be overwritten"
+fi
+
+if [ ! -e "${output_dir}/etc/profile.d/conda.sh" ]; then
+    if [ ! -e miniconda.sh ]; then
+        wget --tries=3 "${CONDA_URL}" -O miniconda.sh
+    fi
+
+    bash miniconda.sh -b -p "${output_dir}"
+fi
+
+# shellcheck disable=SC1090
+source "${output_dir}/etc/profile.d/conda.sh"
+conda deactivate
+
+# If the env already exists, skip recreation
+if [ -n "${name}" ] && ! conda activate ${name}; then
+    conda create -yn "${name}"
+fi
+conda activate ${name}
+
+if [ -n "${PYTHON_VERSION}" ]; then
+    conda install -y conda "python=${PYTHON_VERSION}"
+else
+    conda install -y conda
+fi
+
+conda install -y pip setuptools
+
+cat << EOF > activate_python.sh
+#!/usr/bin/env bash
+# THIS FILE IS GENERATED BY tools/setup_anaconda.sh
+if [ -z "\${PS1:-}" ]; then
+    PS1=__dummy__
+fi
+. $(cd ${output_dir}; pwd)/etc/profile.d/conda.sh && conda deactivate && conda activate ${name}
+EOF