From f6fb815364d3e8a39688f72ad7d5fef624be011a Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Fri, 2 Jul 2021 10:46:21 +0000 Subject: [PATCH] mfa, kaldi install scripts --- tools/install/README.md | 11 + tools/install/install_mfa.sh | 13 + tools/install/install_miniconda.sh | 19 ++ tools/install/install_mkl.sh | 240 ++++++++++++++++++ tools/install/{ngram.sh => install_ngram.sh} | 0 .../{openfst.sh => install_openfst.sh} | 0 .../install/{pynini.sh => install_pynini.sh} | 0 7 files changed, 283 insertions(+) create mode 100644 tools/install/README.md create mode 100755 tools/install/install_mfa.sh create mode 100755 tools/install/install_miniconda.sh create mode 100755 tools/install/install_mkl.sh rename tools/install/{ngram.sh => install_ngram.sh} (100%) rename tools/install/{openfst.sh => install_openfst.sh} (100%) rename tools/install/{pynini.sh => install_pynini.sh} (100%) diff --git a/tools/install/README.md b/tools/install/README.md new file mode 100644 index 00000000..19c06a13 --- /dev/null +++ b/tools/install/README.md @@ -0,0 +1,11 @@ +1. kaldi + +deps gcc, mkl or openblas + +2. OpenFST/ngram/pynini + +deps gcc + +3. MFA + +deps kaldi diff --git a/tools/install/install_mfa.sh b/tools/install/install_mfa.sh new file mode 100755 index 00000000..fdcdaa94 --- /dev/null +++ b/tools/install/install_mfa.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +test -d Montreal-Forced-Aligner || git clone https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner.git + +pushd Montreal-Forced-Aligner && python setup.py install + +test -d kaldi || { echo "need install kaldi first"; exit 1;} + +mfa thirdparty kaldi $PWD/kaldi + +mfa thirdparty validate + +echo "install mfa pass." diff --git a/tools/install/install_miniconda.sh b/tools/install/install_miniconda.sh new file mode 100755 index 00000000..3d1909af --- /dev/null +++ b/tools/install/install_miniconda.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +WGET=${WGET:-wget} + +# The script automatically choose default settings of miniconda for installation +# Miniconda will be installed in the HOME directory. ($HOME/miniconda3). +# Also don't make miniconda's python as default. + +if [ -d "$DOWNLOAD_DIR" ]; then + cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1 +else + $WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1 +fi +bash Miniconda3-latest-Linux-x86_64.sh -b + +$HOME/miniconda3/bin/python -m pip install --user tqdm +$HOME/miniconda3/bin/python -m pip install --user scikit-learn +$HOME/miniconda3/bin/python -m pip install --user librosa +$HOME/miniconda3/bin/python -m pip install --user h5py diff --git a/tools/install/install_mkl.sh b/tools/install/install_mkl.sh new file mode 100755 index 00000000..84fdc9c3 --- /dev/null +++ b/tools/install/install_mkl.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash + +# Intel MKL is now freely available even for commercial use. This script +# attempts to install the MKL package automatically from Intel's repository. +# +# For manual repository setup instructions, see: +# https://software.intel.com/articles/installing-intel-free-libs-and-python-yum-repo +# https://software.intel.com/articles/installing-intel-free-libs-and-python-apt-repo +# +# For other package managers, or non-Linux platforms, see: +# https://software.intel.com/mkl/choose-download + +set -o pipefail + +default_package=intel-mkl-64bit-2020.0-088 + +yum_repo='https://yum.repos.intel.com/mkl/setup/intel-mkl.repo' +apt_repo='https://apt.repos.intel.com/mkl' +intel_key_url='https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB' + +Usage () { + cat >&2 <] + Checks if MKL is present on the system, and/or attempts to install it. + If is not provided, ${default_package} will be installed. + Intel packages are installed under the /opt/intel directory. You should be root + to install MKL into this directory; run this script using the sudo command. + Options: + -s - Skip check for MKL being already present. + -p -- Force type of package management. Use only + if automatic detection fails, as instructed. + -h - Show this message. + Environment: + CC The C compiler to use for MKL check. If not set, uses 'cc'. + EOF + exit 2 + } + + Fatal () { echo "$0: $@"; exit 1; } + + Have () { type -t "$1" >/dev/null; } + + # Option values. + skip_cc= + distro= + + while getopts ":hksp:" opt; do + case ${opt} in + h) Usage ;; + s) skip_cc=yes ;; + p) case $OPTARG in + suse|redhat|debian|fedora|arch) distro=$OPTARG ;; + *) Fatal "invalid value -p '${OPTARG}'. " \ + "Allowed: 'suse', 'redhat', 'debian', 'fedora', or 'arch'." + esac ;; + \?) echo >&2 "$0: invalid option -${OPTARG}."; Usage ;; + esac +done +shift $((OPTIND-1)) + +orig_arg_package=${1-''} +package=${1:-$default_package} + +# Check that we are actually on Linux, otherwise give a helpful reference. +[[ $(uname) == Linux ]] || Fatal "\ + This script can be used on Linux only, and your system is $(uname). +Installer packages for Mac and Windows are available for download from Intel: +https://software.intel.com/mkl/choose-download" + +# Test if MKL is already installed on the system. +if [[ ! $skip_cc ]]; then + : ${CC:=cc} + Have "$CC" || Fatal "\ + C compiler $CC not found. + You can skip the check for MKL presence by invoking this script with the '-s' + option to this script, but you will need a functional compiler anyway, so we + recommend that you install it first." + + mkl_version=$($CC -E -I /opt/intel/mkl/include - <<< \ + '#include + __INTEL_MKL__.__INTEL_MKL_MINOR__.__INTEL_MKL_UPDATE__' 2>/dev/null | + tail -n 1 ) || mkl_version= + mkl_version=${mkl_version// /} + [[ $mkl_version ]] && Fatal "\ + MKL version $mkl_version is already installed. + You can skip the check for MKL presence by invoking this script with the '-s' + option and proceed with automated installation, but we highly discourage + this. This script will register Intel repositories with your system, and it + seems that they have been already registered, or MKL has been installed some + other way. + You should use your package manager to check which MKL package is already + installed. Note that Intel packages register the latest installed version of + the library as the default. If your installed version is older than + $package, it makes sense to upgrade." +fi +# Try to determine which package manager the distro uses, unless overridden. +if [[ ! $distro ]]; then + dist_vars=$(cat /etc/os-release 2>/dev/null) + eval "$dist_vars" + for rune in $CPE_NAME $ID $ID_LIKE; do + case "$rune" in + cpe:/o:fedoraproject:fedora:2[01]) distro=redhat; break;; # Use yum. + rhel|centos) distro=redhat; break;; + redhat|suse|fedora|debian|arch) distro=$rune; break;; + esac + done + # Certain old distributions do not have /etc/os-release. We are unlikely to + # encounter these in the wild, but just in case. + # NOTE: Do not try to guess Fedora specifically here! Fedora 20 and below + # detect as redhat, and this is good, because they use yum by default. + [[ ! $distro && -f /etc/redhat-release ]] && distro=redhat + [[ ! $distro && -f /etc/SuSE-release ]] && distro=suse + [[ ! $distro && -f /etc/debian_release ]] && distro=debian + [[ ! $distro && -f /etc/arch-release ]] && distro=arch + [[ ! $distro ]] && Fatal "\ + Unable to determine package management style. + Invoke this script with the option '-p