fix install scripts

pull/696/head
Hui Zhang 4 years ago
parent a002846fbd
commit 3a21277798

@ -1,3 +1,5 @@
#!/bin/bash
set -e set -e
set -x set -x

@ -27,8 +27,9 @@ touch "python/.use_default_python"
make -j4 make -j4
cd ../src pushd ../src
./configure --shared --use-cuda=no --static-math ./configure --shared --use-cuda=no --static-math --mathlib=OPENBLAS --openblas-root=${KALDI_DIR}/../OpenBLAS/install
make clean -j && make depend -j && make -j4 make clean -j && make depend -j && make -j4
popd
echo "Done installing Kaldi." echo "Done installing Kaldi."

@ -1,8 +1,10 @@
#!/bin/bash #!/bin/bash
# install openblas, kaldi before
test -d Montreal-Forced-Aligner || git clone https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner.git test -d Montreal-Forced-Aligner || git clone https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner.git
pushd Montreal-Forced-Aligner && python setup.py install pushd Montreal-Forced-Aligner && git checkout v2.0.0a7 && python setup.py install
test -d kaldi || { echo "need install kaldi first"; exit 1;} test -d kaldi || { echo "need install kaldi first"; exit 1;}

@ -19,42 +19,47 @@ apt_repo='https://apt.repos.intel.com/mkl'
intel_key_url='https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB' intel_key_url='https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB'
Usage () { Usage () {
cat >&2 <<EOF cat >&2 <<EOF
Usage: $0 [-s] [<MKL-package>] Usage: $0 [-s] [<MKL-package>]
Checks if MKL is present on the system, and/or attempts to install it.
If <MKL-package> is not provided, ${default_package} will be installed. Checks if MKL is present on the system, and/or attempts to install it.
Intel packages are installed under the /opt/intel directory. You should be root
to install MKL into this directory; run this script using the sudo command. If <MKL-package> is not provided, ${default_package} will be installed.
Options:
-s - Skip check for MKL being already present. Intel packages are installed under the /opt/intel directory. You should be root
-p <suse|redhat|debian|fedora|arch> -- Force type of package management. Use only to install MKL into this directory; run this script using the sudo command.
if automatic detection fails, as instructed.
-h - Show this message. Options:
Environment: -s - Skip check for MKL being already present.
CC The C compiler to use for MKL check. If not set, uses 'cc'. -p <suse|redhat|debian|fedora|arch> -- Force type of package management. Use only
EOF if automatic detection fails, as instructed.
exit 2 -h - Show this message.
}
Environment:
Fatal () { echo "$0: $@"; exit 1; } CC The C compiler to use for MKL check. If not set, uses 'cc'.
EOF
Have () { type -t "$1" >/dev/null; } exit 2
}
# Option values.
skip_cc= Fatal () { echo "$0: $@"; exit 1; }
distro=
Have () { type -t "$1" >/dev/null; }
while getopts ":hksp:" opt; do
case ${opt} in # Option values.
h) Usage ;; skip_cc=
s) skip_cc=yes ;; distro=
p) case $OPTARG in
suse|redhat|debian|fedora|arch) distro=$OPTARG ;; while getopts ":hksp:" opt; do
*) Fatal "invalid value -p '${OPTARG}'. " \ case ${opt} in
"Allowed: 'suse', 'redhat', 'debian', 'fedora', or 'arch'." h) Usage ;;
esac ;; s) skip_cc=yes ;;
\?) echo >&2 "$0: invalid option -${OPTARG}."; Usage ;; p) case $OPTARG in
esac suse|redhat|debian|fedora|arch) distro=$OPTARG ;;
*) Fatal "invalid value -p '${OPTARG}'. " \
"Allowed: 'suse', 'redhat', 'debian', 'fedora', or 'arch'."
esac ;;
\?) echo >&2 "$0: invalid option -${OPTARG}."; Usage ;;
esac
done done
shift $((OPTIND-1)) shift $((OPTIND-1))
@ -63,178 +68,210 @@ package=${1:-$default_package}
# Check that we are actually on Linux, otherwise give a helpful reference. # Check that we are actually on Linux, otherwise give a helpful reference.
[[ $(uname) == Linux ]] || Fatal "\ [[ $(uname) == Linux ]] || Fatal "\
This script can be used on Linux only, and your system is $(uname). This script can be used on Linux only, and your system is $(uname).
Installer packages for Mac and Windows are available for download from Intel: Installer packages for Mac and Windows are available for download from Intel:
https://software.intel.com/mkl/choose-download" https://software.intel.com/mkl/choose-download"
# Test if MKL is already installed on the system. # Test if MKL is already installed on the system.
if [[ ! $skip_cc ]]; then if [[ ! $skip_cc ]]; then
: ${CC:=cc} : ${CC:=cc}
Have "$CC" || Fatal "\ Have "$CC" || Fatal "\
C compiler $CC not found. C compiler $CC not found.
You can skip the check for MKL presence by invoking this script with the '-s'
option to this script, but you will need a functional compiler anyway, so we You can skip the check for MKL presence by invoking this script with the '-s'
recommend that you install it first." option to this script, but you will need a functional compiler anyway, so we
recommend that you install it first."
mkl_version=$($CC -E -I /opt/intel/mkl/include - <<< \
'#include <mkl_version.h> mkl_version=$($CC -E -I /opt/intel/mkl/include - <<< \
__INTEL_MKL__.__INTEL_MKL_MINOR__.__INTEL_MKL_UPDATE__' 2>/dev/null | '#include <mkl_version.h>
tail -n 1 ) || mkl_version= __INTEL_MKL__.__INTEL_MKL_MINOR__.__INTEL_MKL_UPDATE__' 2>/dev/null |
mkl_version=${mkl_version// /} tail -n 1 ) || mkl_version=
[[ $mkl_version ]] && Fatal "\ mkl_version=${mkl_version// /}
MKL version $mkl_version is already installed.
You can skip the check for MKL presence by invoking this script with the '-s' [[ $mkl_version ]] && Fatal "\
option and proceed with automated installation, but we highly discourage MKL version $mkl_version is already installed.
this. This script will register Intel repositories with your system, and it
seems that they have been already registered, or MKL has been installed some You can skip the check for MKL presence by invoking this script with the '-s'
other way. option and proceed with automated installation, but we highly discourage
You should use your package manager to check which MKL package is already this. This script will register Intel repositories with your system, and it
installed. Note that Intel packages register the latest installed version of seems that they have been already registered, or MKL has been installed some
the library as the default. If your installed version is older than other way.
$package, it makes sense to upgrade."
You should use your package manager to check which MKL package is already
installed. Note that Intel packages register the latest installed version of
the library as the default. If your installed version is older than
$package, it makes sense to upgrade."
fi fi
# Try to determine which package manager the distro uses, unless overridden. # Try to determine which package manager the distro uses, unless overridden.
if [[ ! $distro ]]; then if [[ ! $distro ]]; then
dist_vars=$(cat /etc/os-release 2>/dev/null) dist_vars=$(cat /etc/os-release 2>/dev/null)
eval "$dist_vars" eval "$dist_vars"
for rune in $CPE_NAME $ID $ID_LIKE; do for rune in $CPE_NAME $ID $ID_LIKE; do
case "$rune" in case "$rune" in
cpe:/o:fedoraproject:fedora:2[01]) distro=redhat; break;; # Use yum. cpe:/o:fedoraproject:fedora:2[01]) distro=redhat; break;; # Use yum.
rhel|centos) distro=redhat; break;; rhel|centos) distro=redhat; break;;
redhat|suse|fedora|debian|arch) distro=$rune; break;; redhat|suse|fedora|debian|arch) distro=$rune; break;;
esac esac
done done
# Certain old distributions do not have /etc/os-release. We are unlikely to
# encounter these in the wild, but just in case. # Certain old distributions do not have /etc/os-release. We are unlikely to
# NOTE: Do not try to guess Fedora specifically here! Fedora 20 and below # encounter these in the wild, but just in case.
# detect as redhat, and this is good, because they use yum by default. # NOTE: Do not try to guess Fedora specifically here! Fedora 20 and below
[[ ! $distro && -f /etc/redhat-release ]] && distro=redhat # detect as redhat, and this is good, because they use yum by default.
[[ ! $distro && -f /etc/SuSE-release ]] && distro=suse [[ ! $distro && -f /etc/redhat-release ]] && distro=redhat
[[ ! $distro && -f /etc/debian_release ]] && distro=debian [[ ! $distro && -f /etc/SuSE-release ]] && distro=suse
[[ ! $distro && -f /etc/arch-release ]] && distro=arch [[ ! $distro && -f /etc/debian_release ]] && distro=debian
[[ ! $distro ]] && Fatal "\ [[ ! $distro && -f /etc/arch-release ]] && distro=arch
Unable to determine package management style.
Invoke this script with the option '-p <style>', where <style> can be: [[ ! $distro ]] && Fatal "\
redhat -- RedHat-like, uses yum and rpm for package management. Unable to determine package management style.
fedora -- Fedora 22+, also RedHat-like, but uses dnf instead of yum.
suse -- SUSE-like, uses zypper and rpm. Invoke this script with the option '-p <style>', where <style> can be:
debian -- Debian-like, uses apt and dpkg. redhat -- RedHat-like, uses yum and rpm for package management.
arch -- Archlinux, uses pacman. fedora -- Fedora 22+, also RedHat-like, but uses dnf instead of yum.
We do not currently support other package management systems. Check the Intel's suse -- SUSE-like, uses zypper and rpm.
documentation at https://software.intel.com/mkl/choose-download for other debian -- Debian-like, uses apt and dpkg.
install options." arch -- Archlinux, uses pacman.
echo >&2 "$0: Your system is using ${distro}-style package management."
We do not currently support other package management systems. Check the Intel's
documentation at https://software.intel.com/mkl/choose-download for other
install options."
echo >&2 "$0: Your system is using ${distro}-style package management."
fi fi
# Check for root. # Check for root.
if [[ "$(id -u)" -ne 0 ]]; then if [[ "$(id -u)" -ne 0 ]]; then
echo >&2 "$0: You must be root to install MKL. echo >&2 "$0: You must be root to install MKL.
Restart this script using the 'sudo' command, as:
sudo $0 -sp $distro $package Restart this script using the 'sudo' command, as:
We recommend adding the '-sp $distro' options to skip the MKL and distro
detection, since this has already been done. This minimizes the number of sudo $0 -sp $distro $package
programs invoked with the root privileges to keep your system safe from
unexpected or erroneous changes. Also, if you are setting the CC environment We recommend adding the '-sp $distro' options to skip the MKL and distro
variable, sudo might not allow it to propagate to the command that it invokes." detection, since this has already been done. This minimizes the number of
if [ -t 0 ]; then programs invoked with the root privileges to keep your system safe from
echo; read -ep "Run the above sudo command now? [Y/n]:" unexpected or erroneous changes. Also, if you are setting the CC environment
case $REPLY in variable, sudo might not allow it to propagate to the command that it invokes."
''|[Yy]*) set -x; exec sudo "$0" -sp "$distro" "$package"
esac if [ -t 0 ]; then
fi echo; read -ep "Run the above sudo command now? [Y/n]:"
exit 0 case $REPLY in
''|[Yy]*) set -x; exec sudo "$0" -sp "$distro" "$package"
esac
fi
exit 0
fi fi
# The install variants, each in a function to simplify error reporting. # The install variants, each in a function to simplify error reporting.
# Each one invokes a subshell with a 'set -x' to to show system-modifying # Each one invokes a subshell with a 'set -x' to to show system-modifying
# commands it runs. The subshells simply limit the scope of this diagnostics # commands it runs. The subshells simply limit the scope of this diagnostics
# and avoid creating noise (if we were using 'set +x', it would be printed). # and avoid creating noise (if we were using 'set +x', it would be printed).
Install_redhat () { Install_redhat () {
# yum-utils contains yum-config-manager, in case the user does not have it. # yum-utils contains yum-config-manager, in case the user does not have it.
( set -x ( set -x
rpm --import $intel_key_url rpm --import $intel_key_url
yum -y install yum-utils && yum -y install yum-utils &&
yum-config-manager --add-repo "$yum_repo" && yum-config-manager --add-repo "$yum_repo" &&
yum -y install "$package" ) yum -y install "$package" )
} }
Install_fedora () { Install_fedora () {
( set -x ( set -x
rpm --import $intel_key_url rpm --import $intel_key_url
dnf -y install 'dnf-command(config-manager)' && dnf -y install 'dnf-command(config-manager)' &&
dnf config-manager --add-repo "$yum_repo" && dnf config-manager --add-repo "$yum_repo" &&
dnf -y install "$package" ) dnf -y install "$package" )
} }
Install_suse () { Install_suse () {
# zypper bug until libzypp-17.6.4: '--gpg-auto-import-keys' is ignored. # zypper bug until libzypp-17.6.4: '--gpg-auto-import-keys' is ignored.
# See https://github.com/openSUSE/zypper/issues/144#issuecomment-418685933 # See https://github.com/openSUSE/zypper/issues/144#issuecomment-418685933
# We must disable gpg checks with '--no-gpg-checks'. I won't bend backwards # We must disable gpg checks with '--no-gpg-checks'. I won't bend backwards
# as far as check the installed .so version... # as far as check the installed .so version...
( set -x ( set -x
rpm --import $intel_key_url rpm --import $intel_key_url
zypper addrepo "$yum_repo" && zypper addrepo "$yum_repo" &&
zypper --gpg-auto-import-keys --no-gpg-checks \ zypper --gpg-auto-import-keys --no-gpg-checks \
--non-interactive install "$package" ) --non-interactive install "$package" )
} }
Install_debian () { Install_debian () {
local keyring='/usr/share/keyrings/intel-sw-products.gpg' \ local keyring='/usr/share/keyrings/intel-sw-products.gpg' \
sources_d='/etc/apt/sources.list.d' \ sources_d='/etc/apt/sources.list.d' \
trusted_d='/etc/apt/trusted.gpg.d' \ trusted_d='/etc/apt/trusted.gpg.d' \
apt_maj= apt_min= apt_ver= apt_maj= apt_min= apt_ver=
# apt before 1.2 does not understand the signed-by option, and always
# look for the keyring in their trusted.gpg.d directory. This is not # apt before 1.2 does not understand the signed-by option, and always
# considered a good security practice any more. If apt is old, add a link # look for the keyring in their trusted.gpg.d directory. This is not
# to the keyring file and remind the user to delete it when apt is upgraded. # considered a good security practice any more. If apt is old, add a link
IFS=' .' builtin read _ apt_maj apt_min _ < <(apt-get --version) # to the keyring file and remind the user to delete it when apt is upgraded.
apt_ver=$(builtin printf '%03d%03d' $apt_maj $apt_min) IFS=' .' builtin read _ apt_maj apt_min _ < <(apt-get --version)
# Get alternative location of /etc/apt/sources.list.d, if so configured. apt_ver=$(builtin printf '%03d%03d' $apt_maj $apt_min)
eval $(apt-config shell sources_d Dir::Etc::sourceparts/f \
trusted_d Dir::Etc::trustedparts/f) # Get alternative location of /etc/apt/sources.list.d, if so configured.
# apt is much more involved to configure than other package managers, as fas eval $(apt-config shell sources_d Dir::Etc::sourceparts/f \
# as third-party security keys go. trusted_d Dir::Etc::trustedparts/f)
( set -x;
# apt is much more involved to configure than other package managers, as fas
# as third-party security keys go.
( set -x;
apt-get update && apt-get update &&
apt-get install -y wget apt-transport-https ca-certificates gnupg && apt-get install -y wget apt-transport-https ca-certificates gnupg &&
wget -qO- $intel_key_url | apt-key --keyring $keyring add - && wget -qO- $intel_key_url | apt-key --keyring $keyring add - &&
echo "deb [signed-by=${keyring}] $apt_repo all main" \ echo "deb [signed-by=${keyring}] $apt_repo all main" \
> "$sources_d/intel-mkl.list" ) || return 1 > "$sources_d/intel-mkl.list" ) || return 1
if [[ $apt_ver < '001002' ]]; then
( set -x; ln -s "$keyring" "${trusted_d}/" ) || return 1 if [[ $apt_ver < '001002' ]]; then
fi ( set -x; ln -s "$keyring" "${trusted_d}/" ) || return 1
( set +x fi
( set +x
apt-get update && apt-get update &&
apt-get install -y "$package" ) || return 1 apt-get install -y "$package" ) || return 1
# Print the message after the large install, so the user may notice. I hope...
if [[ $apt_ver < '001002' ]]; then # Print the message after the large install, so the user may notice. I hope...
echo >&2 "$0: Your apt-get version is earlier than 1.2. if [[ $apt_ver < '001002' ]]; then
This version does not understand individual repositories signing keys, and echo >&2 "$0: Your apt-get version is earlier than 1.2.
trusts all keys in $trusted_d. We have created a link
$trusted_d/$(basename $keyring) pointing to the file This version does not understand individual repositories signing keys, and
$keyring. If/when you upgrade your system to trusts all keys in $trusted_d. We have created a link
a higher version of apt, removing this link will help make it more secure. $trusted_d/$(basename $keyring) pointing to the file
This is not considered a severe security issue, but separating keyrings is the $keyring. If/when you upgrade your system to
current recommended security practice." a higher version of apt, removing this link will help make it more secure.
fi
This is not considered a severe security issue, but separating keyrings is the
current recommended security practice."
fi
} }
Install_arch () { Install_arch () {
( set -x ( set -x
echo y | pacman -Syu intel-mkl && # In pacman we don't specify the version echo y | pacman -Syu intel-mkl && # In pacman we don't specify the version
pacman -Q --info intel-mkl | grep -v None pacman -Q --info intel-mkl | grep -v None
) )
} }
# Register MKL .so libraries with the ld.so. # Register MKL .so libraries with the ld.so.
ConfigLdSo() { ConfigLdSo() {
[ -d /etc/ld.so.conf.d ] || return 0 [ -d /etc/ld.so.conf.d ] || return 0
type -t ldconfig >/dev/null || return 0 type -t ldconfig >/dev/null || return 0
echo >&2 "$0: Configuring ld runtime bindings" echo >&2 "$0: Configuring ld runtime bindings"
( set -x; ( set -x;
echo >/etc/ld.so.conf.d/intel-mkl.conf "\ echo >/etc/ld.so.conf.d/intel-mkl.conf "\
/opt/intel/lib/intel64 /opt/intel/lib/intel64
/opt/intel/mkl/lib/intel64" /opt/intel/mkl/lib/intel64"
ldconfig ) ldconfig )
} }
# Invoke installation. # Invoke installation.
if Install_${distro} && ConfigLdSo; then if Install_${distro} && ConfigLdSo; then
echo >&2 "$0: MKL package $package was successfully installed" echo >&2 "$0: MKL package $package was successfully installed"
else else
Fatal "MKL package $package installation FAILED. Fatal "MKL package $package installation FAILED.
Please open an issue with us at https://github.com/kaldi-asr/kaldi/ if you
believe this is a bug." Please open an issue with us at https://github.com/kaldi-asr/kaldi/ if you
believe this is a bug."
fi fi

@ -1,11 +1,18 @@
#!/bin/bash
set -e set -e
set -x set -x
# need support c++17, so need gcc >= 8 # need support c++17, so need gcc >= 8
# openfst # openfst
ngram=ngram-1.3.13 ngram=ngram-1.3.13
openfst=openfst-1.8.1
shared=true shared=true
export CPLUS_INCLUDE_PATH=$PWD/${openfst}/install/include/:$CPLUS_INCLUDE_PATH
export LD_LIBRARY_PATH=$PWD/${openfst}/install/lib/:$LD_LIBRARY_PATH
test -e ${ngram}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/NGramDownload/${ngram}.tar.gz test -e ${ngram}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/NGramDownload/${ngram}.tar.gz
test -d ${ngram} || tar -xvf ${ngram}.tar.gz && chown -R root:root ${ngram} test -d ${ngram} || tar -xvf ${ngram}.tar.gz && chown -R root:root ${ngram}

@ -1,3 +1,5 @@
#!/bin/bash
set -e set -e
set -x set -x
@ -9,13 +11,15 @@ shared=true
test -e ${openfst}.tar.gz || wget http://www.openfst.org/twiki/pub/FST/FstDownload/${openfst}.tar.gz test -e ${openfst}.tar.gz || wget http://www.openfst.org/twiki/pub/FST/FstDownload/${openfst}.tar.gz
test -d ${openfst} || tar -xvf ${openfst}.tar.gz && chown -R root:root ${openfst} test -d ${openfst} || tar -xvf ${openfst}.tar.gz && chown -R root:root ${openfst}
wfst_so_path=$(python3 -c 'import sysconfig; import os; from pathlib import Path; site = sysconfig.get_paths()["purelib"]; site=Path(site); suffix = ("/usr/local/lib",) + site.parts[-2:]; print(os.path.join(*suffix));')
if [ $shared == true ];then if [ $shared == true ];then
pushd ${openfst} && ./configure --enable-shared --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/output && popd pushd ${openfst} && ./configure --enable-shared --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/install && popd
else else
pushd ${openfst} && ./configure --enable-static --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/output && popd pushd ${openfst} && ./configure --enable-static --enable-compact-fsts --enable-compress --enable-const-fsts --enable-far --enable-linear-fsts --enable-lookahead-fsts --enable-mpdt --enable-ngram-fsts --enable-pdt --enable-python --enable-special --enable-bin --enable-grm --prefix ${PWD}/install && popd
fi fi
pushd ${openfst} && make -j && make install && popd pushd ${openfst} && make -j && make install && popd
suffix_path=$(python3 -c 'import sysconfig; import os; from pathlib import Path; site = sysconfig.get_paths()["purelib"]; site=Path(site); pwd = os.getcwd(); suffix = site.parts[-2:]; print(os.path.join(*suffix));')
wfst_so_path=${PWD}/${openfst}/install/lib/${suffix_path}
cp ${wfst_so_path}/pywrapfst.* $(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') cp ${wfst_so_path}/pywrapfst.* $(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')

@ -1,13 +1,14 @@
#!/bin/bash
set -e set -e
set -x set -x
pynini=pynini-2.1.4 pynini=pynini-2.1.4
openfst=openfst-1.8.1
LIBRARY_PATH=$PWD/${openfst}/install/lib
test -e ${pynini}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/PyniniDownload/${pynini}.tar.gz test -e ${pynini}.tar.gz || wget http://www.openfst.org/twiki/pub/GRM/PyniniDownload/${pynini}.tar.gz
test -d ${pynini} || tar -xvf ${pynini}.tar.gz && chown -R root:root ${pynini} test -d ${pynini} || tar -xvf ${pynini}.tar.gz && chown -R root:root ${pynini}
#wfst_so_path=$(python3 -c 'import sysconfig; import os; from pathlib import Path; site = sysconfig.get_paths()["purelib"]; site=Path(site); suffix = ("/usr/local/lib",) + site.parts[-2:]; print(os.path.join(*suffix));') pushd ${pynini} && LIBRARY_PATH=$LIBRARY_PATH python setup.py install && popd
pushd ${pynini} && python setup.py install && popd
#cp ${wfst_so_path}/pywrapfst.* $(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')

Loading…
Cancel
Save