From ccfecd17b2baacd05a9c6b93181017569f10d46a Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Fri, 2 Jul 2021 09:26:02 +0000 Subject: [PATCH 1/4] mfa and kaldi deps install --- tools/install/install_gcc.sh | 15 ++++++++++++ tools/install/install_kaldi.sh | 34 +++++++++++++++++++++++++++ tools/install/install_openblas.sh | 39 +++++++++++++++++++++++++++++++ tools/install/ngram.sh | 17 ++++++++++++++ tools/install/openfst.sh | 21 +++++++++++++++++ tools/install/pynini.sh | 13 +++++++++++ 6 files changed, 139 insertions(+) create mode 100755 tools/install/install_gcc.sh create mode 100755 tools/install/install_kaldi.sh create mode 100755 tools/install/install_openblas.sh create mode 100755 tools/install/ngram.sh create mode 100755 tools/install/openfst.sh create mode 100755 tools/install/pynini.sh diff --git a/tools/install/install_gcc.sh b/tools/install/install_gcc.sh new file mode 100755 index 00000000..6eda8ea7 --- /dev/null +++ b/tools/install/install_gcc.sh @@ -0,0 +1,15 @@ +set -e +set -x + +# gcc +apt update -y +apt install build-essential -y +apt install software-properties-common -y +add-apt-repository ppa:ubuntu-toolchain-r/test +apt install gcc-8 g++-8 -y +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 +update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 80 +update-alternatives --config gcc + +# gfortran +apt-get install gfortran-8 diff --git a/tools/install/install_kaldi.sh b/tools/install/install_kaldi.sh new file mode 100755 index 00000000..545fbe4e --- /dev/null +++ b/tools/install/install_kaldi.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Installation script for Kaldi +# +set -e + +apt-get install subversion -y + +KALDI_GIT="--depth 1 -b master https://github.com/kaldi-asr/kaldi.git" + +KALDI_DIR="$PWD/kaldi" + +if [ ! -d "$KALDI_DIR" ]; then + git clone $KALDI_GIT $KALDI_DIR +else + echo "$KALDI_DIR already exists!" +fi + +cd "$KALDI_DIR/tools" +git pull + +# Prevent kaldi from switching default python version +mkdir -p "python" +touch "python/.use_default_python" + +./extras/check_dependencies.sh + +make -j4 + +cd ../src +./configure --shared --use-cuda=no --static-math +make clean -j && make depend -j && make -j4 + +echo "Done installing Kaldi." diff --git a/tools/install/install_openblas.sh b/tools/install/install_openblas.sh new file mode 100755 index 00000000..b1e4d3da --- /dev/null +++ b/tools/install/install_openblas.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +OPENBLAS_VERSION=0.3.13 + +WGET=${WGET:-wget} + +set -e + +if ! command -v gfortran 2>/dev/null; then + echo "$0: gfortran is not installed. Please install it, e.g. by:" + echo " apt-get install gfortran" + echo "(if on Debian or Ubuntu), or:" + echo " yum install gcc-gfortran" + echo "(if on RedHat/CentOS). On a Mac, if brew is installed, it's:" + echo " brew install gfortran" + exit 1 +fi + + +tarball=OpenBLAS-$OPENBLAS_VERSION.tar.gz + +rm -rf xianyi-OpenBLAS-* OpenBLAS OpenBLAS-*.tar.gz + +if [ -d "$DOWNLOAD_DIR" ]; then + cp -p "$DOWNLOAD_DIR/$tarball" . +else + url=$($WGET -qO- "https://api.github.com/repos/xianyi/OpenBLAS/releases/tags/v${OPENBLAS_VERSION}" | python -c 'import sys,json;print(json.load(sys.stdin)["tarball_url"])') + test -n "$url" + $WGET -t3 -nv -O $tarball "$url" +fi + +tar xzf $tarball +mv xianyi-OpenBLAS-* OpenBLAS + +make PREFIX=$(pwd)/OpenBLAS/install USE_LOCKING=1 USE_THREAD=0 -C OpenBLAS all install +if [ $? -eq 0 ]; then + echo "OpenBLAS is installed successfully." + rm $tarball +fi diff --git a/tools/install/ngram.sh b/tools/install/ngram.sh new file mode 100755 index 00000000..304bb93d --- /dev/null +++ b/tools/install/ngram.sh @@ -0,0 +1,17 @@ +set -e +set -x + +# need support c++17, so need gcc >= 8 +# openfst +ngram=ngram-1.3.13 +shared=true + +test -e ${ngram}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/NGramDownload/${ngram}.tar.gz +test -d ${ngram} || tar -xvf ${ngram}.tar.gz && chown -R root:root ${ngram} + +if [ $shared == true ];then + pushd ${ngram} && ./configure --enable-shared && popd +else + pushd ${ngram} && ./configure --enable-static && popd +fi +pushd ${ngram} && make -j && make install && popd diff --git a/tools/install/openfst.sh b/tools/install/openfst.sh new file mode 100755 index 00000000..8c6a4574 --- /dev/null +++ b/tools/install/openfst.sh @@ -0,0 +1,21 @@ +set -e +set -x + +# need support c++17, so need gcc >= 8 +# openfst +openfst=openfst-1.8.1 +shared=true + +test -e ${openfst}.tar.gz || wget http://www.openfst.org/twiki/pub/FST/FstDownload/${openfst}.tar.gz +test -d ${openfst} || tar -xvf ${openfst}.tar.gz && chown -R root:root ${openfst} + +wfst_so_path=$(python3 -c 'import sysconfig; import os; from pathlib import Path; site = sysconfig.get_paths()["purelib"]; site=Path(site); suffix = ("/usr/local/lib",) + site.parts[-2:]; print(os.path.join(*suffix));') + +if [ $shared == true ];then + pushd ${openfst} && ./configure --enable-shared --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/output && popd +else + pushd ${openfst} && ./configure --enable-static --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/output && popd +fi +pushd ${openfst} && make -j && make install && popd + +cp ${wfst_so_path}/pywrapfst.* $(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') diff --git a/tools/install/pynini.sh b/tools/install/pynini.sh new file mode 100755 index 00000000..a6f86f55 --- /dev/null +++ b/tools/install/pynini.sh @@ -0,0 +1,13 @@ +set -e +set -x + +pynini=pynini-2.1.4 + +test -e ${pynini}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/PyniniDownload/${pynini}.tar.gz +test -d ${pynini} || tar -xvf ${pynini}.tar.gz && chown -R root:root ${pynini} + +#wfst_so_path=$(python3 -c 'import sysconfig; import os; from pathlib import Path; site = sysconfig.get_paths()["purelib"]; site=Path(site); suffix = ("/usr/local/lib",) + site.parts[-2:]; print(os.path.join(*suffix));') + +pushd ${pynini} && python setup.py install && popd + +#cp ${wfst_so_path}/pywrapfst.* $(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') From f6fb815364d3e8a39688f72ad7d5fef624be011a Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Fri, 2 Jul 2021 10:46:21 +0000 Subject: [PATCH 2/4] mfa, kaldi install scripts --- tools/install/README.md | 11 + tools/install/install_mfa.sh | 13 + tools/install/install_miniconda.sh | 19 ++ tools/install/install_mkl.sh | 240 ++++++++++++++++++ tools/install/{ngram.sh => install_ngram.sh} | 0 .../{openfst.sh => install_openfst.sh} | 0 .../install/{pynini.sh => install_pynini.sh} | 0 7 files changed, 283 insertions(+) create mode 100644 tools/install/README.md create mode 100755 tools/install/install_mfa.sh create mode 100755 tools/install/install_miniconda.sh create mode 100755 tools/install/install_mkl.sh rename tools/install/{ngram.sh => install_ngram.sh} (100%) rename tools/install/{openfst.sh => install_openfst.sh} (100%) rename tools/install/{pynini.sh => install_pynini.sh} (100%) diff --git a/tools/install/README.md b/tools/install/README.md new file mode 100644 index 00000000..19c06a13 --- /dev/null +++ b/tools/install/README.md @@ -0,0 +1,11 @@ +1. kaldi + +deps gcc, mkl or openblas + +2. OpenFST/ngram/pynini + +deps gcc + +3. MFA + +deps kaldi diff --git a/tools/install/install_mfa.sh b/tools/install/install_mfa.sh new file mode 100755 index 00000000..fdcdaa94 --- /dev/null +++ b/tools/install/install_mfa.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +test -d Montreal-Forced-Aligner || git clone https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner.git + +pushd Montreal-Forced-Aligner && python setup.py install + +test -d kaldi || { echo "need install kaldi first"; exit 1;} + +mfa thirdparty kaldi $PWD/kaldi + +mfa thirdparty validate + +echo "install mfa pass." diff --git a/tools/install/install_miniconda.sh b/tools/install/install_miniconda.sh new file mode 100755 index 00000000..3d1909af --- /dev/null +++ b/tools/install/install_miniconda.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +WGET=${WGET:-wget} + +# The script automatically choose default settings of miniconda for installation +# Miniconda will be installed in the HOME directory. ($HOME/miniconda3). +# Also don't make miniconda's python as default. + +if [ -d "$DOWNLOAD_DIR" ]; then + cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1 +else + $WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1 +fi +bash Miniconda3-latest-Linux-x86_64.sh -b + +$HOME/miniconda3/bin/python -m pip install --user tqdm +$HOME/miniconda3/bin/python -m pip install --user scikit-learn +$HOME/miniconda3/bin/python -m pip install --user librosa +$HOME/miniconda3/bin/python -m pip install --user h5py diff --git a/tools/install/install_mkl.sh b/tools/install/install_mkl.sh new file mode 100755 index 00000000..84fdc9c3 --- /dev/null +++ b/tools/install/install_mkl.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash + +# Intel MKL is now freely available even for commercial use. This script +# attempts to install the MKL package automatically from Intel's repository. +# +# For manual repository setup instructions, see: +# https://software.intel.com/articles/installing-intel-free-libs-and-python-yum-repo +# https://software.intel.com/articles/installing-intel-free-libs-and-python-apt-repo +# +# For other package managers, or non-Linux platforms, see: +# https://software.intel.com/mkl/choose-download + +set -o pipefail + +default_package=intel-mkl-64bit-2020.0-088 + +yum_repo='https://yum.repos.intel.com/mkl/setup/intel-mkl.repo' +apt_repo='https://apt.repos.intel.com/mkl' +intel_key_url='https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB' + +Usage () { + cat >&2 <] + Checks if MKL is present on the system, and/or attempts to install it. + If is not provided, ${default_package} will be installed. + Intel packages are installed under the /opt/intel directory. You should be root + to install MKL into this directory; run this script using the sudo command. + Options: + -s - Skip check for MKL being already present. + -p -- Force type of package management. Use only + if automatic detection fails, as instructed. + -h - Show this message. + Environment: + CC The C compiler to use for MKL check. If not set, uses 'cc'. + EOF + exit 2 + } + + Fatal () { echo "$0: $@"; exit 1; } + + Have () { type -t "$1" >/dev/null; } + + # Option values. + skip_cc= + distro= + + while getopts ":hksp:" opt; do + case ${opt} in + h) Usage ;; + s) skip_cc=yes ;; + p) case $OPTARG in + suse|redhat|debian|fedora|arch) distro=$OPTARG ;; + *) Fatal "invalid value -p '${OPTARG}'. " \ + "Allowed: 'suse', 'redhat', 'debian', 'fedora', or 'arch'." + esac ;; + \?) echo >&2 "$0: invalid option -${OPTARG}."; Usage ;; + esac +done +shift $((OPTIND-1)) + +orig_arg_package=${1-''} +package=${1:-$default_package} + +# Check that we are actually on Linux, otherwise give a helpful reference. +[[ $(uname) == Linux ]] || Fatal "\ + This script can be used on Linux only, and your system is $(uname). +Installer packages for Mac and Windows are available for download from Intel: +https://software.intel.com/mkl/choose-download" + +# Test if MKL is already installed on the system. +if [[ ! $skip_cc ]]; then + : ${CC:=cc} + Have "$CC" || Fatal "\ + C compiler $CC not found. + You can skip the check for MKL presence by invoking this script with the '-s' + option to this script, but you will need a functional compiler anyway, so we + recommend that you install it first." + + mkl_version=$($CC -E -I /opt/intel/mkl/include - <<< \ + '#include + __INTEL_MKL__.__INTEL_MKL_MINOR__.__INTEL_MKL_UPDATE__' 2>/dev/null | + tail -n 1 ) || mkl_version= + mkl_version=${mkl_version// /} + [[ $mkl_version ]] && Fatal "\ + MKL version $mkl_version is already installed. + You can skip the check for MKL presence by invoking this script with the '-s' + option and proceed with automated installation, but we highly discourage + this. This script will register Intel repositories with your system, and it + seems that they have been already registered, or MKL has been installed some + other way. + You should use your package manager to check which MKL package is already + installed. Note that Intel packages register the latest installed version of + the library as the default. If your installed version is older than + $package, it makes sense to upgrade." +fi +# Try to determine which package manager the distro uses, unless overridden. +if [[ ! $distro ]]; then + dist_vars=$(cat /etc/os-release 2>/dev/null) + eval "$dist_vars" + for rune in $CPE_NAME $ID $ID_LIKE; do + case "$rune" in + cpe:/o:fedoraproject:fedora:2[01]) distro=redhat; break;; # Use yum. + rhel|centos) distro=redhat; break;; + redhat|suse|fedora|debian|arch) distro=$rune; break;; + esac + done + # Certain old distributions do not have /etc/os-release. We are unlikely to + # encounter these in the wild, but just in case. + # NOTE: Do not try to guess Fedora specifically here! Fedora 20 and below + # detect as redhat, and this is good, because they use yum by default. + [[ ! $distro && -f /etc/redhat-release ]] && distro=redhat + [[ ! $distro && -f /etc/SuSE-release ]] && distro=suse + [[ ! $distro && -f /etc/debian_release ]] && distro=debian + [[ ! $distro && -f /etc/arch-release ]] && distro=arch + [[ ! $distro ]] && Fatal "\ + Unable to determine package management style. + Invoke this script with the option '-p