Merge pull request #1069 from zh794390558/install

[install] install with conda

.gitignore

@@ -26,5 +26,8 @@ tools/sctk
 tools/sctk-20159b5/
 tools/kaldi
 tools/OpenBLAS/
+tools/Miniconda3-latest-Linux-x86_64.sh
+tools/activate_python.sh
+tools/miniconda.sh
 
 *output/

docs/source/install.md

@@ -27,15 +27,15 @@ bash
 ```
 Then you can create an conda virtual environment using the following command:
 ```bash
-conda create -n py37 python=3.7
+conda create -y -p tools/venv python=3.7
 ```
 Activate the conda virtual environment:
 ```bash
-conda activate py37
+conda activate tools/venv
 ```
 Install  conda dependencies for `paddlespeech` :
 ```bash
-conda install -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 --yes
+conda install -y -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0
 ```
 ### Install PaddlePaddle
 For example, for CUDA 10.2, CuDNN7.5 install paddle 2.2.0:
@@ -103,11 +103,11 @@ popd
 # use the "bash" command to make the conda environment works
 bash
 # create an conda virtual environment
-conda create -n py37 python=3.7
+conda create -y -n tools/venv python=3.7
 # Activate the conda virtual environment:
-conda activate py37
+conda activate tools/venv
 # Install the conda packags
-conda install -c conda-forge sox libsndfile swig bzip2 gcc_linux-64=8.4.0 gxx_linux-64=8.4.0 --yes
+conda install -y -c conda-forge sox libsndfile swig bzip2 libflac bc gcc_linux-64=8.4.0 gxx_linux-64=8.4.0
 ```
 ### Install PaddlePaddle
 For example, for CUDA 10.2, CuDNN7.5 install paddle 2.2.0:
@@ -117,7 +117,7 @@ python3 -m pip install paddlepaddle-gpu==2.2.0
 ```
 ### Get the Funcition for Developing PaddleSpeech
 ```bash
-pip install -e .[develop]
+pip install .[develop]
 ```
 ### Install the Kaldi (Optional)
 ```bash
@@ -126,7 +126,10 @@ bash extras/install_openblas.sh
 bash extras/install_kaldi.sh
 popd
 ```
+
+
 ## Setup for Other Platform 
 - Make sure these libraries or tools in [dependencies](./dependencies.md) installed. More information please see: `setup.py `and ` tools/Makefile`.
 - The version of `swig` should >= 3.0
 - we will simplify the install process in the future.
+

setup.py

@@ -130,6 +130,7 @@ def _post_install(install_lib_dir):
         check_call("bash -e setup.sh")
     print("ctcdecoder install.")
 
+
 class DevelopCommand(develop):
     def run(self):
         develop.run(self)
@@ -213,8 +214,7 @@ setup_info = dict(
     },
 
     # Package info
-    packages=find_packages(exclude=('utils', 'tests', 'tests.*', 'examples*',
-                                    'paddleaudio*', 'third_party*', 'tools*')),
+    packages=find_packages(include=('paddlespeech')),
     zip_safe=True,
     classifiers=[
         'Development Status :: 3 - Alpha',

tools/Makefile

@@ -10,7 +10,8 @@ WGET ?= wget --no-check-certificate
 
 .PHONY: all clean 
 
-all: apt.done kenlm.done mfa.done sclite.done
+all: apt.done kenlm.done mfa.done sctk.done
+
 
 virtualenv.done:
 	test -d venv || virtualenv -p $(PYTHON) venv
@@ -28,6 +29,7 @@ apt.done:
 	echo "check_certificate = off" >> ~/.wgetrc
 	touch apt.done
 
+
 kenlm.done:
 	# Ubuntu 16.04 透過 apt 會安裝 boost 1.58.0
 	# it seems that boost (1.54.0) requires higher version. After I switched to g++-5 it compiles normally.
@@ -38,12 +40,6 @@ kenlm.done:
 	cd kenlm && python3 setup.py install
 	touch kenlm.done
 
-sox.done:
-	apt install -y libvorbis-dev libmp3lame-dev libmad-ocaml-dev
-	test -d sox-14.4.2 || $(WGET) https://nchc.dl.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.gz
-	tar -xvzf sox-14.4.2.tar.gz -C .
-	cd sox-14.4.2 && ./configure --prefix=/usr/ && make -j4 && make install
-	touch sox.done
 
 mfa.done:
 	test -d montreal-forced-aligner || $(WGET) https://paddlespeech.bj.bcebos.com/Parakeet/montreal-forced-aligner_linux.tar.gz
@@ -58,48 +54,46 @@ kaldi.done: openblas.done
 	bash extras/install_kaldi.sh
 	touch kaldi.done
 
-#== SCTK ===============================================================================
-# SCTK official repo does not have version tags. Here's the mapping:
-# # 2.4.9 = 659bc36; 2.4.10 = d914e1b; 2.4.11 = 20159b5.
-SCTK_GITHASH = 20159b5
-
-SCTK_CXFLAGS = -w -march=native
-SCTK_MKENV = CFLAGS="$(CFLAGS) $(SCTK_CXFLAGS)" \
-			              CXXFLAGS="$(CXXFLAGS) -std=c++11 $(SCTK_CXFLAGS)" \
-
-
-# Keep the existing target 'sclite' to avoid breaking the users who might have
-# scripted it in.
-.PHONY: sclite.done sctk_cleaned sctk_made
-
-sclite.done sctk_made: sctk/.compiled
-	touch sclite.done
-
-sctk/.compiled: sctk
-	rm -f sctk/.compiled
-	$(SCTK_MKENV) $(MAKE) -C sctk config
-	$(SCTK_MKENV) $(MAKE) -C sctk all doc
-	$(MAKE) -C sctk install
-	touch sctk/.compiled
-
-# The GitHub archive unpacks into SCTK-{40-character-long-hash}/
-sctk: sctk-$(SCTK_GITHASH).tar.gz
-	tar zxvf sctk-$(SCTK_GITHASH).tar.gz
-	rm -rf sctk-$(SCTK_GITHASH) sctk
-	mv SCTK-$(SCTK_GITHASH)* sctk-$(SCTK_GITHASH)
-	ln -s sctk-$(SCTK_GITHASH) sctk
-	touch sctk-$(SCTK_GITHASH).tar.gz
-
-sctk-$(SCTK_GITHASH).tar.gz:
-	if [ -d '$(DOWNLOAD_DIR)' ]; then \
-	  cp -p '$(DOWNLOAD_DIR)/sctk-$(SCTK_GITHASH).tar.gz' .; \
-	else \
-	  $(WGET) -nv -T 10 -t 3 -O sctk-$(SCTK_GITHASH).tar.gz \
-	    https://github.com/usnistgov/SCTK/archive/$(SCTK_GITHASH).tar.gz; \
-	fi
-
-sctk_cleaned:
-	-for d in sctk/ sctk-*/; do \
-	   [ ! -f $$d/.compiled ] || $(MAKE) -C $$d clean; \
-	   rm -f $$d/.compiled; \
-	done
+sctk.done:
+	./extras/install_sclite.sh
+	touch sctk.done
+
+######################
+dev: python conda_packages.done sctk.done
+
+# Use pip for paddle installation even if you have anaconda
+ifneq ($(shell test -f ./activate_python.sh && grep 'conda activate' ./activate_python.sh),)
+USE_CONDA := 1
+else
+USE_CONDA :=
+endif
+
+python: activate_python.sh 
+
+activate_python.sh:
+	test -f activate_python.sh || { echo "Error: Run ./setup_python.sh or ./setup_anaconda.sh"; exit 1; }
+
+bc.done: activate_python.sh
+	. ./activate_python.sh && { command -v bc || conda install -y bc -c conda-forge; }
+	touch bc.done
+cmake.done: activate_python.sh
+	. ./activate_python.sh && { command -v cmake || conda install -y cmake; }
+	touch cmake.done
+flac.done: activate_python.sh
+	. ./activate_python.sh && { command -v flac || conda install -y libflac -c conda-forge; }
+	touch flac.done
+ffmpeg.done: activate_python.sh
+	. ./activate_python.sh && { command -v ffmpeg || conda install -y ffmpeg -c conda-forge; }
+	touch ffmpeg.done
+sox.done: activate_python.sh
+	. ./activate_python.sh && { command -v sox || conda install -y sox -c conda-forge; }
+	touch sox.done
+sndfile.done: activate_python.sh
+	. ./activate_python.sh && { python3 -c "from ctypes.util import find_library as F; assert F('sndfile') is not None" || conda install -y libsndfile=1.0.28 -c conda-forge; }
+	touch sndfile.done
+ifneq ($(strip $(USE_CONDA)),)
+conda_packages.done: bc.done cmake.done flac.done ffmpeg.done sox.done sndfile.done
+else
+conda_packages.done:
+endif
+	touch conda_packages.done

tools/extras/install_miniconda.sh

@@ -9,7 +9,7 @@ WGET=${WGET:-wget}
 if [ -d "$DOWNLOAD_DIR" ]; then
     cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1
 else
-    $WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1
+    $WGET -c https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1
 fi
 bash Miniconda3-latest-Linux-x86_64.sh -b
 

tools/setup_anaconda.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -z "${PS1:-}" ]; then
+    PS1=__dummy__
+fi
+CONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+
+if [ $# -gt 4 ]; then
+    echo "Usage: $0 [output] [conda-env-name] [python-version>]"
+    exit 1;
+elif [ $# -eq 3 ]; then
+    output_dir="$1"
+    name="$2"
+    PYTHON_VERSION="$3"
+elif [ $# -eq 2 ]; then
+    output_dir="$1"
+    name="$2"
+    PYTHON_VERSION=""
+elif [ $# -eq 1 ]; then
+    output_dir="$1"
+    name=""
+    PYTHON_VERSION=""
+elif [ $# -eq 0 ]; then
+    output_dir=venv
+    name=""
+    PYTHON_VERSION=""
+fi
+
+if [ -e activate_python.sh ]; then
+    echo "Warning: activate_python.sh already exists. It will be overwritten"
+fi
+
+if [ ! -e "${output_dir}/etc/profile.d/conda.sh" ]; then
+    if [ ! -e miniconda.sh ]; then
+        wget --tries=3 "${CONDA_URL}" -O miniconda.sh
+    fi
+
+    bash miniconda.sh -b -p "${output_dir}"
+fi
+
+# shellcheck disable=SC1090
+source "${output_dir}/etc/profile.d/conda.sh"
+conda deactivate
+
+# If the env already exists, skip recreation
+if [ -n "${name}" ] && ! conda activate ${name}; then
+    conda create -yn "${name}"
+fi
+conda activate ${name}
+
+if [ -n "${PYTHON_VERSION}" ]; then
+    conda install -y conda "python=${PYTHON_VERSION}"
+else
+    conda install -y conda
+fi
+
+conda install -y pip setuptools
+
+cat << EOF > activate_python.sh
+#!/usr/bin/env bash
+# THIS FILE IS GENERATED BY tools/setup_anaconda.sh
+if [ -z "\${PS1:-}" ]; then
+    PS1=__dummy__
+fi
+. $(cd ${output_dir}; pwd)/etc/profile.d/conda.sh && conda deactivate && conda activate ${name}
+EOF