Hui Zhang
3 years ago
committed by
GitHub
commit
6c16754365
@ -0,0 +1,11 @@
|
||||
1. kaldi
|
||||
|
||||
deps gcc, mkl or openblas
|
||||
|
||||
2. OpenFST/ngram/pynini
|
||||
|
||||
deps gcc
|
||||
|
||||
3. MFA
|
||||
|
||||
deps kaldi
|
||||
@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# gcc
|
||||
apt update -y
|
||||
apt install build-essential -y
|
||||
apt install software-properties-common -y
|
||||
add-apt-repository ppa:ubuntu-toolchain-r/test
|
||||
apt install gcc-8 g++-8 -y
|
||||
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80
|
||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 80
|
||||
update-alternatives --config gcc
|
||||
|
||||
# gfortran
|
||||
apt-get install gfortran-8
|
||||
@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Installation script for Kaldi
|
||||
#
|
||||
set -e
|
||||
|
||||
apt-get install subversion -y
|
||||
|
||||
KALDI_GIT="--depth 1 -b master https://github.com/kaldi-asr/kaldi.git"
|
||||
|
||||
KALDI_DIR="$PWD/kaldi"
|
||||
|
||||
if [ ! -d "$KALDI_DIR" ]; then
|
||||
git clone $KALDI_GIT $KALDI_DIR
|
||||
else
|
||||
echo "$KALDI_DIR already exists!"
|
||||
fi
|
||||
|
||||
cd "$KALDI_DIR/tools"
|
||||
git pull
|
||||
|
||||
# Prevent kaldi from switching default python version
|
||||
mkdir -p "python"
|
||||
touch "python/.use_default_python"
|
||||
|
||||
./extras/check_dependencies.sh
|
||||
|
||||
make -j4
|
||||
|
||||
pushd ../src
|
||||
./configure --shared --use-cuda=no --static-math --mathlib=OPENBLAS --openblas-root=${KALDI_DIR}/../OpenBLAS/install
|
||||
make clean -j && make depend -j && make -j4
|
||||
popd
|
||||
|
||||
echo "Done installing Kaldi."
|
||||
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
# install openblas, kaldi before
|
||||
|
||||
test -d Montreal-Forced-Aligner || git clone https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner.git
|
||||
|
||||
pushd Montreal-Forced-Aligner && git checkout v2.0.0a7 && python setup.py install
|
||||
|
||||
test -d kaldi || { echo "need install kaldi first"; exit 1;}
|
||||
|
||||
mfa thirdparty kaldi $PWD/kaldi
|
||||
|
||||
mfa thirdparty validate
|
||||
|
||||
echo "install mfa pass."
|
||||
@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
WGET=${WGET:-wget}
|
||||
|
||||
# The script automatically choose default settings of miniconda for installation
|
||||
# Miniconda will be installed in the HOME directory. ($HOME/miniconda3).
|
||||
# Also don't make miniconda's python as default.
|
||||
|
||||
if [ -d "$DOWNLOAD_DIR" ]; then
|
||||
cp -p "$DOWNLOAD_DIR/Miniconda3-latest-Linux-x86_64.sh" . || exit 1
|
||||
else
|
||||
$WGET https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh || exit 1
|
||||
fi
|
||||
bash Miniconda3-latest-Linux-x86_64.sh -b
|
||||
|
||||
$HOME/miniconda3/bin/python -m pip install --user tqdm
|
||||
$HOME/miniconda3/bin/python -m pip install --user scikit-learn
|
||||
$HOME/miniconda3/bin/python -m pip install --user librosa
|
||||
$HOME/miniconda3/bin/python -m pip install --user h5py
|
||||
@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Intel MKL is now freely available even for commercial use. This script
|
||||
# attempts to install the MKL package automatically from Intel's repository.
|
||||
#
|
||||
# For manual repository setup instructions, see:
|
||||
# https://software.intel.com/articles/installing-intel-free-libs-and-python-yum-repo
|
||||
# https://software.intel.com/articles/installing-intel-free-libs-and-python-apt-repo
|
||||
#
|
||||
# For other package managers, or non-Linux platforms, see:
|
||||
# https://software.intel.com/mkl/choose-download
|
||||
|
||||
set -o pipefail
|
||||
|
||||
default_package=intel-mkl-64bit-2020.0-088
|
||||
|
||||
yum_repo='https://yum.repos.intel.com/mkl/setup/intel-mkl.repo'
|
||||
apt_repo='https://apt.repos.intel.com/mkl'
|
||||
intel_key_url='https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB'
|
||||
|
||||
Usage () {
|
||||
cat >&2 <<EOF
|
||||
Usage: $0 [-s] [<MKL-package>]
|
||||
|
||||
Checks if MKL is present on the system, and/or attempts to install it.
|
||||
|
||||
If <MKL-package> is not provided, ${default_package} will be installed.
|
||||
|
||||
Intel packages are installed under the /opt/intel directory. You should be root
|
||||
to install MKL into this directory; run this script using the sudo command.
|
||||
|
||||
Options:
|
||||
-s - Skip check for MKL being already present.
|
||||
-p <suse|redhat|debian|fedora|arch> -- Force type of package management. Use only
|
||||
if automatic detection fails, as instructed.
|
||||
-h - Show this message.
|
||||
|
||||
Environment:
|
||||
CC The C compiler to use for MKL check. If not set, uses 'cc'.
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
Fatal () { echo "$0: $@"; exit 1; }
|
||||
|
||||
Have () { type -t "$1" >/dev/null; }
|
||||
|
||||
# Option values.
|
||||
skip_cc=
|
||||
distro=
|
||||
|
||||
while getopts ":hksp:" opt; do
|
||||
case ${opt} in
|
||||
h) Usage ;;
|
||||
s) skip_cc=yes ;;
|
||||
p) case $OPTARG in
|
||||
suse|redhat|debian|fedora|arch) distro=$OPTARG ;;
|
||||
*) Fatal "invalid value -p '${OPTARG}'. " \
|
||||
"Allowed: 'suse', 'redhat', 'debian', 'fedora', or 'arch'."
|
||||
esac ;;
|
||||
\?) echo >&2 "$0: invalid option -${OPTARG}."; Usage ;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND-1))
|
||||
|
||||
orig_arg_package=${1-''}
|
||||
package=${1:-$default_package}
|
||||
|
||||
# Check that we are actually on Linux, otherwise give a helpful reference.
|
||||
[[ $(uname) == Linux ]] || Fatal "\
|
||||
This script can be used on Linux only, and your system is $(uname).
|
||||
|
||||
Installer packages for Mac and Windows are available for download from Intel:
|
||||
https://software.intel.com/mkl/choose-download"
|
||||
|
||||
# Test if MKL is already installed on the system.
|
||||
if [[ ! $skip_cc ]]; then
|
||||
: ${CC:=cc}
|
||||
Have "$CC" || Fatal "\
|
||||
C compiler $CC not found.
|
||||
|
||||
You can skip the check for MKL presence by invoking this script with the '-s'
|
||||
option to this script, but you will need a functional compiler anyway, so we
|
||||
recommend that you install it first."
|
||||
|
||||
mkl_version=$($CC -E -I /opt/intel/mkl/include - <<< \
|
||||
'#include <mkl_version.h>
|
||||
__INTEL_MKL__.__INTEL_MKL_MINOR__.__INTEL_MKL_UPDATE__' 2>/dev/null |
|
||||
tail -n 1 ) || mkl_version=
|
||||
mkl_version=${mkl_version// /}
|
||||
|
||||
[[ $mkl_version ]] && Fatal "\
|
||||
MKL version $mkl_version is already installed.
|
||||
|
||||
You can skip the check for MKL presence by invoking this script with the '-s'
|
||||
option and proceed with automated installation, but we highly discourage
|
||||
this. This script will register Intel repositories with your system, and it
|
||||
seems that they have been already registered, or MKL has been installed some
|
||||
other way.
|
||||
|
||||
You should use your package manager to check which MKL package is already
|
||||
installed. Note that Intel packages register the latest installed version of
|
||||
the library as the default. If your installed version is older than
|
||||
$package, it makes sense to upgrade."
|
||||
fi
|
||||
|
||||
# Try to determine which package manager the distro uses, unless overridden.
|
||||
if [[ ! $distro ]]; then
|
||||
dist_vars=$(cat /etc/os-release 2>/dev/null)
|
||||
eval "$dist_vars"
|
||||
for rune in $CPE_NAME $ID $ID_LIKE; do
|
||||
case "$rune" in
|
||||
cpe:/o:fedoraproject:fedora:2[01]) distro=redhat; break;; # Use yum.
|
||||
rhel|centos) distro=redhat; break;;
|
||||
redhat|suse|fedora|debian|arch) distro=$rune; break;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Certain old distributions do not have /etc/os-release. We are unlikely to
|
||||
# encounter these in the wild, but just in case.
|
||||
# NOTE: Do not try to guess Fedora specifically here! Fedora 20 and below
|
||||
# detect as redhat, and this is good, because they use yum by default.
|
||||
[[ ! $distro && -f /etc/redhat-release ]] && distro=redhat
|
||||
[[ ! $distro && -f /etc/SuSE-release ]] && distro=suse
|
||||
[[ ! $distro && -f /etc/debian_release ]] && distro=debian
|
||||
[[ ! $distro && -f /etc/arch-release ]] && distro=arch
|
||||
|
||||
[[ ! $distro ]] && Fatal "\
|
||||
Unable to determine package management style.
|
||||
|
||||
Invoke this script with the option '-p <style>', where <style> can be:
|
||||
redhat -- RedHat-like, uses yum and rpm for package management.
|
||||
fedora -- Fedora 22+, also RedHat-like, but uses dnf instead of yum.
|
||||
suse -- SUSE-like, uses zypper and rpm.
|
||||
debian -- Debian-like, uses apt and dpkg.
|
||||
arch -- Archlinux, uses pacman.
|
||||
|
||||
We do not currently support other package management systems. Check the Intel's
|
||||
documentation at https://software.intel.com/mkl/choose-download for other
|
||||
install options."
|
||||
|
||||
echo >&2 "$0: Your system is using ${distro}-style package management."
|
||||
fi
|
||||
|
||||
# Check for root.
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo >&2 "$0: You must be root to install MKL.
|
||||
|
||||
Restart this script using the 'sudo' command, as:
|
||||
|
||||
sudo $0 -sp $distro $package
|
||||
|
||||
We recommend adding the '-sp $distro' options to skip the MKL and distro
|
||||
detection, since this has already been done. This minimizes the number of
|
||||
programs invoked with the root privileges to keep your system safe from
|
||||
unexpected or erroneous changes. Also, if you are setting the CC environment
|
||||
variable, sudo might not allow it to propagate to the command that it invokes."
|
||||
|
||||
if [ -t 0 ]; then
|
||||
echo; read -ep "Run the above sudo command now? [Y/n]:"
|
||||
case $REPLY in
|
||||
''|[Yy]*) set -x; exec sudo "$0" -sp "$distro" "$package"
|
||||
esac
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The install variants, each in a function to simplify error reporting.
|
||||
# Each one invokes a subshell with a 'set -x' to to show system-modifying
|
||||
# commands it runs. The subshells simply limit the scope of this diagnostics
|
||||
# and avoid creating noise (if we were using 'set +x', it would be printed).
|
||||
Install_redhat () {
|
||||
# yum-utils contains yum-config-manager, in case the user does not have it.
|
||||
( set -x
|
||||
rpm --import $intel_key_url
|
||||
yum -y install yum-utils &&
|
||||
yum-config-manager --add-repo "$yum_repo" &&
|
||||
yum -y install "$package" )
|
||||
}
|
||||
|
||||
Install_fedora () {
|
||||
( set -x
|
||||
rpm --import $intel_key_url
|
||||
dnf -y install 'dnf-command(config-manager)' &&
|
||||
dnf config-manager --add-repo "$yum_repo" &&
|
||||
dnf -y install "$package" )
|
||||
}
|
||||
|
||||
Install_suse () {
|
||||
# zypper bug until libzypp-17.6.4: '--gpg-auto-import-keys' is ignored.
|
||||
# See https://github.com/openSUSE/zypper/issues/144#issuecomment-418685933
|
||||
# We must disable gpg checks with '--no-gpg-checks'. I won't bend backwards
|
||||
# as far as check the installed .so version...
|
||||
( set -x
|
||||
rpm --import $intel_key_url
|
||||
zypper addrepo "$yum_repo" &&
|
||||
zypper --gpg-auto-import-keys --no-gpg-checks \
|
||||
--non-interactive install "$package" )
|
||||
}
|
||||
|
||||
Install_debian () {
|
||||
local keyring='/usr/share/keyrings/intel-sw-products.gpg' \
|
||||
sources_d='/etc/apt/sources.list.d' \
|
||||
trusted_d='/etc/apt/trusted.gpg.d' \
|
||||
apt_maj= apt_min= apt_ver=
|
||||
|
||||
# apt before 1.2 does not understand the signed-by option, and always
|
||||
# look for the keyring in their trusted.gpg.d directory. This is not
|
||||
# considered a good security practice any more. If apt is old, add a link
|
||||
# to the keyring file and remind the user to delete it when apt is upgraded.
|
||||
IFS=' .' builtin read _ apt_maj apt_min _ < <(apt-get --version)
|
||||
apt_ver=$(builtin printf '%03d%03d' $apt_maj $apt_min)
|
||||
|
||||
# Get alternative location of /etc/apt/sources.list.d, if so configured.
|
||||
eval $(apt-config shell sources_d Dir::Etc::sourceparts/f \
|
||||
trusted_d Dir::Etc::trustedparts/f)
|
||||
|
||||
# apt is much more involved to configure than other package managers, as fas
|
||||
# as third-party security keys go.
|
||||
( set -x;
|
||||
apt-get update &&
|
||||
apt-get install -y wget apt-transport-https ca-certificates gnupg &&
|
||||
wget -qO- $intel_key_url | apt-key --keyring $keyring add - &&
|
||||
echo "deb [signed-by=${keyring}] $apt_repo all main" \
|
||||
> "$sources_d/intel-mkl.list" ) || return 1
|
||||
|
||||
if [[ $apt_ver < '001002' ]]; then
|
||||
( set -x; ln -s "$keyring" "${trusted_d}/" ) || return 1
|
||||
fi
|
||||
|
||||
( set +x
|
||||
apt-get update &&
|
||||
apt-get install -y "$package" ) || return 1
|
||||
|
||||
# Print the message after the large install, so the user may notice. I hope...
|
||||
if [[ $apt_ver < '001002' ]]; then
|
||||
echo >&2 "$0: Your apt-get version is earlier than 1.2.
|
||||
|
||||
This version does not understand individual repositories signing keys, and
|
||||
trusts all keys in $trusted_d. We have created a link
|
||||
$trusted_d/$(basename $keyring) pointing to the file
|
||||
$keyring. If/when you upgrade your system to
|
||||
a higher version of apt, removing this link will help make it more secure.
|
||||
|
||||
This is not considered a severe security issue, but separating keyrings is the
|
||||
current recommended security practice."
|
||||
fi
|
||||
}
|
||||
|
||||
Install_arch () {
|
||||
( set -x
|
||||
echo y | pacman -Syu intel-mkl && # In pacman we don't specify the version
|
||||
pacman -Q --info intel-mkl | grep -v None
|
||||
)
|
||||
}
|
||||
|
||||
# Register MKL .so libraries with the ld.so.
|
||||
ConfigLdSo() {
|
||||
[ -d /etc/ld.so.conf.d ] || return 0
|
||||
type -t ldconfig >/dev/null || return 0
|
||||
echo >&2 "$0: Configuring ld runtime bindings"
|
||||
( set -x;
|
||||
echo >/etc/ld.so.conf.d/intel-mkl.conf "\
|
||||
/opt/intel/lib/intel64
|
||||
/opt/intel/mkl/lib/intel64"
|
||||
ldconfig )
|
||||
}
|
||||
|
||||
# Invoke installation.
|
||||
if Install_${distro} && ConfigLdSo; then
|
||||
echo >&2 "$0: MKL package $package was successfully installed"
|
||||
else
|
||||
Fatal "MKL package $package installation FAILED.
|
||||
|
||||
Please open an issue with us at https://github.com/kaldi-asr/kaldi/ if you
|
||||
believe this is a bug."
|
||||
fi
|
||||
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
|
||||
# need support c++17, so need gcc >= 8
|
||||
# openfst
|
||||
ngram=ngram-1.3.13
|
||||
openfst=openfst-1.8.1
|
||||
shared=true
|
||||
|
||||
export CPLUS_INCLUDE_PATH=$PWD/${openfst}/install/include/:$CPLUS_INCLUDE_PATH
|
||||
export LD_LIBRARY_PATH=$PWD/${openfst}/install/lib/:$LD_LIBRARY_PATH
|
||||
|
||||
test -e ${ngram}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/NGramDownload/${ngram}.tar.gz
|
||||
test -d ${ngram} || tar -xvf ${ngram}.tar.gz && chown -R root:root ${ngram}
|
||||
|
||||
if [ $shared == true ];then
|
||||
pushd ${ngram} && ./configure --enable-shared && popd
|
||||
else
|
||||
pushd ${ngram} && ./configure --enable-static && popd
|
||||
fi
|
||||
pushd ${ngram} && make -j && make install && popd
|
||||
@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
OPENBLAS_VERSION=0.3.13
|
||||
|
||||
WGET=${WGET:-wget}
|
||||
|
||||
set -e
|
||||
|
||||
if ! command -v gfortran 2>/dev/null; then
|
||||
echo "$0: gfortran is not installed. Please install it, e.g. by:"
|
||||
echo " apt-get install gfortran"
|
||||
echo "(if on Debian or Ubuntu), or:"
|
||||
echo " yum install gcc-gfortran"
|
||||
echo "(if on RedHat/CentOS). On a Mac, if brew is installed, it's:"
|
||||
echo " brew install gfortran"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
tarball=OpenBLAS-$OPENBLAS_VERSION.tar.gz
|
||||
|
||||
rm -rf xianyi-OpenBLAS-* OpenBLAS OpenBLAS-*.tar.gz
|
||||
|
||||
if [ -d "$DOWNLOAD_DIR" ]; then
|
||||
cp -p "$DOWNLOAD_DIR/$tarball" .
|
||||
else
|
||||
url=$($WGET -qO- "https://api.github.com/repos/xianyi/OpenBLAS/releases/tags/v${OPENBLAS_VERSION}" | python -c 'import sys,json;print(json.load(sys.stdin)["tarball_url"])')
|
||||
test -n "$url"
|
||||
$WGET -t3 -nv -O $tarball "$url"
|
||||
fi
|
||||
|
||||
tar xzf $tarball
|
||||
mv xianyi-OpenBLAS-* OpenBLAS
|
||||
|
||||
make PREFIX=$(pwd)/OpenBLAS/install USE_LOCKING=1 USE_THREAD=0 -C OpenBLAS all install
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "OpenBLAS is installed successfully."
|
||||
rm $tarball
|
||||
fi
|
||||
@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# need support c++17, so need gcc >= 8
|
||||
# openfst
|
||||
openfst=openfst-1.8.1
|
||||
shared=true
|
||||
|
||||
test -e ${openfst}.tar.gz || wget http://www.openfst.org/twiki/pub/FST/FstDownload/${openfst}.tar.gz
|
||||
test -d ${openfst} || tar -xvf ${openfst}.tar.gz && chown -R root:root ${openfst}
|
||||
|
||||
|
||||
if [ $shared == true ];then
|
||||
pushd ${openfst} && ./configure --enable-shared --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/install && popd
|
||||
else
|
||||
pushd ${openfst} && ./configure --enable-static --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/install && popd
|
||||
fi
|
||||
pushd ${openfst} && make -j && make install && popd
|
||||
|
||||
|
||||
suffix_path=$(python3 -c 'import sysconfig; import os; from pathlib import Path; site = sysconfig.get_paths()["purelib"]; site=Path(site); pwd = os.getcwd(); suffix = site.parts[-2:]; print(os.path.join(*suffix));')
|
||||
wfst_so_path=${PWD}/${openfst}/install/lib/${suffix_path}
|
||||
cp ${wfst_so_path}/pywrapfst.* $(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')
|
||||
@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
pynini=pynini-2.1.4
|
||||
openfst=openfst-1.8.1
|
||||
|
||||
LIBRARY_PATH=$PWD/${openfst}/install/lib
|
||||
|
||||
test -e ${pynini}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/PyniniDownload/${pynini}.tar.gz
|
||||
test -d ${pynini} || tar -xvf ${pynini}.tar.gz && chown -R root:root ${pynini}
|
||||
|
||||
pushd ${pynini} && LIBRARY_PATH=$LIBRARY_PATH python setup.py install && popd
|
||||
Loading…
Reference in new issue