From cd3c57d563da6aed8f9f1a77ce436cc531e24149 Mon Sep 17 00:00:00 2001 From: SmileGoat Date: Thu, 3 Mar 2022 20:20:53 +0800 Subject: [PATCH 1/2] add build.sh & openblas --- speechx/CMakeLists.txt | 75 +++++++++------------ speechx/build.sh | 23 +++++++ speechx/speechx/CMakeLists.txt | 2 +- speechx/speechx/kaldi/matrix/CMakeLists.txt | 2 +- speechx/speechx/kaldi/matrix/kaldi-blas.h | 6 +- 5 files changed, 58 insertions(+), 50 deletions(-) create mode 100644 speechx/build.sh diff --git a/speechx/CMakeLists.txt b/speechx/CMakeLists.txt index e54ba3a4..84c3b161 100644 --- a/speechx/CMakeLists.txt +++ b/speechx/CMakeLists.txt @@ -49,24 +49,7 @@ include_directories(${absl_SOURCE_DIR}) #) #FetchContent_MakeAvailable(libsndfile) -#boost set(boost_SOURCE_DIR ${fc_patch}/boost-src) -set(boost_PREFIX_DIR ${fc_patch}/boost-subbuild/boost-prefix) -include(ExternalProject) -ExternalProject_Add( - boost - URL https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz - URL_HASH SHA256=aeb26f80e80945e82ee93e5939baebdca47b9dee80a07d3144be1e1a6a66dd6a - SOURCE_DIR ${boost_SOURCE_DIR} - PREFIX ${boost_PREFIX_DIR} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ./bootstrap.sh - BUILD_COMMAND ./b2 - INSTALL_COMMAND "" -) -link_directories(${boost_SOURCE_DIR}/stage/lib) -include_directories(${boost_SOURCE_DIR}) - set(BOOST_ROOT ${boost_SOURCE_DIR}) include(FetchContent) FetchContent_Declare( @@ -75,10 +58,9 @@ FetchContent_Declare( GIT_TAG "df2d717e95183f79a90b2fa6e4307083a351ca6a" ) FetchContent_MakeAvailable(kenlm) -add_dependencies(kenlm boost) include_directories(${kenlm_SOURCE_DIR}) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -pthread -fPIC -O0 -Wall -g -ggdb") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -pthread -fPIC -O0 -Wall -g") # gflags FetchContent_Declare( gflags @@ -98,11 +80,11 @@ FetchContent_MakeAvailable(glog) include_directories(${glog_BINARY_DIR} ${glog_SOURCE_DIR}/src) # gtest -FetchContent_Declare(googletest - URL https://github.com/google/googletest/archive/release-1.10.0.zip - URL_HASH SHA256=94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91 -) -FetchContent_MakeAvailable(googletest) +#FetchContent_Declare(googletest + #URL https://github.com/google/googletest/archive/release-1.10.0.zip + #URL_HASH SHA256=94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91 +#) +#FetchContent_MakeAvailable(googletest) # openfst set(openfst_SOURCE_DIR ${fc_patch}/openfst-src) @@ -111,7 +93,7 @@ set(openfst_PREFIX_DIR ${fc_patch}/openfst-subbuild/openfst-populate-prefix) ExternalProject_Add(openfst URL https://github.com/mjansche/openfst/archive/refs/tags/1.7.2.zip URL_HASH SHA256=ffc56931025579a8af3515741c0f3b0fc3a854c023421472c07ca0c6389c75e6 -# PREFIX ${openfst_PREFIX_DIR} + #PREFIX ${openfst_PREFIX_DIR} SOURCE_DIR ${openfst_SOURCE_DIR} BINARY_DIR ${openfst_BINARY_DIR} CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure --prefix=${openfst_PREFIX_DIR} @@ -142,14 +124,10 @@ set(PADDLE_LIB ${fc_patch}/paddle-lib) include_directories("${PADDLE_LIB}/paddle/include") set(PADDLE_LIB_THIRD_PARTY_PATH "${PADDLE_LIB}/third_party/install/") include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/include") -#include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/include") -#include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/include") include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/include") include_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/include") link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}protobuf/lib") -#link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}glog/lib") -#link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}gflags/lib") link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}xxhash/lib") link_directories("${PADDLE_LIB_THIRD_PARTY_PATH}cryptopp/lib") link_directories("${PADDLE_LIB}/paddle/lib") @@ -171,23 +149,30 @@ set(DEPS ${DEPS} glog gflags protobuf xxhash cryptopp ${EXTERNAL_LIB}) -add_subdirectory(speechx) - #openblas -#set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS) -#set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) -#ExternalProject_Add( -# OpenBLAS -# GIT_REPOSITORY https://github.com/xianyi/OpenBLAS -# GIT_TAG v0.3.13 -# GIT_SHALLOW TRUE -# GIT_PROGRESS TRUE -# CONFIGURE_COMMAND "" -# BUILD_IN_SOURCE TRUE -# BUILD_COMMAND make USE_LOCKING=1 USE_THREAD=0 -# INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install -# UPDATE_DISCONNECTED TRUE -#) +set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS) +set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) +set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix) +ExternalProject_Add( + OpenBLAS + GIT_REPOSITORY https://github.com/xianyi/OpenBLAS + GIT_TAG 1ef97c470ccf23d06d3bb0597c52df4a2e9093b9 + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE + PREFIX ${OpenBLAS_PREFIX} + SOURCE_DIR ${OpenBLAS_SOURCE_DIR} + BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND make + INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install + UPDATE_DISCONNECTED TRUE +) +link_directories(${OpenBLAS_INSTALL_PREFIX}/lib) +include_directories(${OpenBLAS_INSTALL_PREFIX}/include) + +link_directories(/usr/local/gcc-8.2/lib64) + +add_subdirectory(speechx) ############################################################################### # Add local library diff --git a/speechx/build.sh b/speechx/build.sh new file mode 100644 index 00000000..582f9b44 --- /dev/null +++ b/speechx/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +boost_SOURCE_DIR=$PWD/fc_patch/boost-src +if [ ! -d ${boost_SOURCE_DIR} ]; then + wget https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz + tar xzfv boost_1_75_0.tar.gz + mkdir -p $PWD/fc_patch + mv boost_1_75_0 ${boost_SOURCE_DIR} + cd ${boost_SOURCE_DIR} + bash ./bootstrap.sh + ./b2 + cd - + echo -e "\n" +fi + +mkdir build +cd build + +cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR} + +make + +cd - diff --git a/speechx/speechx/CMakeLists.txt b/speechx/speechx/CMakeLists.txt index 55a782ac..4a296ec8 100644 --- a/speechx/speechx/CMakeLists.txt +++ b/speechx/speechx/CMakeLists.txt @@ -33,7 +33,7 @@ ${CMAKE_CURRENT_SOURCE_DIR}/decoder add_subdirectory(decoder) add_executable(mfcc-test codelab/feat_test/feature-mfcc-test.cc) -target_link_libraries(mfcc-test kaldi-mfcc ${MATH_LIB}) +target_link_libraries(mfcc-test kaldi-mfcc) add_executable(linear_spectrogram_main codelab/feat_test/linear_spectrogram_main.cc) target_link_libraries(linear_spectrogram_main frontend kaldi-util kaldi-feat-common gflags glog) diff --git a/speechx/speechx/kaldi/matrix/CMakeLists.txt b/speechx/speechx/kaldi/matrix/CMakeLists.txt index 64f44864..a4dbde2e 100644 --- a/speechx/speechx/kaldi/matrix/CMakeLists.txt +++ b/speechx/speechx/kaldi/matrix/CMakeLists.txt @@ -13,4 +13,4 @@ srfft.cc tp-matrix.cc ) -target_link_libraries(kaldi-matrix gfortran kaldi-base ${MATH_LIB}) +target_link_libraries(kaldi-matrix gfortran kaldi-base libopenblas.a) diff --git a/speechx/speechx/kaldi/matrix/kaldi-blas.h b/speechx/speechx/kaldi/matrix/kaldi-blas.h index cf4ad9df..143781c8 100644 --- a/speechx/speechx/kaldi/matrix/kaldi-blas.h +++ b/speechx/speechx/kaldi/matrix/kaldi-blas.h @@ -42,7 +42,7 @@ -#define HAVE_MKL +#define HAVE_OPENBLAS #if (defined(HAVE_CLAPACK) && (defined(HAVE_ATLAS) || defined(HAVE_MKL))) \ || (defined(HAVE_ATLAS) && defined(HAVE_MKL)) @@ -96,8 +96,8 @@ #elif defined(HAVE_OPENBLAS) // getting cblas.h and lapacke.h from /. // putting in "" not <> to search -I before system libraries. - #include "third_party/openblas/cblas.h" - #include "third_party/openblas/lapacke.h" + #include "cblas.h" + #include "lapacke.h" #undef I #undef complex // get rid of macros from f2c.h -- these are dangerous. From a89032d21b3c59aab1f7f8a238976a7cae928a26 Mon Sep 17 00:00:00 2001 From: SmileGoat Date: Fri, 4 Mar 2022 10:56:31 +0800 Subject: [PATCH 2/2] add comment --- speechx/CMakeLists.txt | 36 +++++++++++++++++++----------------- speechx/build.sh | 4 ++++ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/speechx/CMakeLists.txt b/speechx/CMakeLists.txt index 84c3b161..af6582cd 100644 --- a/speechx/CMakeLists.txt +++ b/speechx/CMakeLists.txt @@ -150,26 +150,28 @@ set(DEPS ${DEPS} ${EXTERNAL_LIB}) #openblas -set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS) -set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) -set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix) -ExternalProject_Add( - OpenBLAS - GIT_REPOSITORY https://github.com/xianyi/OpenBLAS - GIT_TAG 1ef97c470ccf23d06d3bb0597c52df4a2e9093b9 - GIT_SHALLOW TRUE - GIT_PROGRESS TRUE - PREFIX ${OpenBLAS_PREFIX} - SOURCE_DIR ${OpenBLAS_SOURCE_DIR} - BUILD_IN_SOURCE TRUE - CONFIGURE_COMMAND "" - BUILD_COMMAND make - INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install - UPDATE_DISCONNECTED TRUE -) +set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/openblas-install) +#set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS) +#set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) +#set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix) +#ExternalProject_Add( + #OpenBLAS + #GIT_REPOSITORY https://github.com/xianyi/OpenBLAS + #GIT_TAG 1ef97c470ccf23d06d3bb0597c52df4a2e9093b9 + #GIT_SHALLOW TRUE + #GIT_PROGRESS TRUE + #PREFIX ${OpenBLAS_PREFIX} + #SOURCE_DIR ${OpenBLAS_SOURCE_DIR} + #BUILD_IN_SOURCE TRUE + #CONFIGURE_COMMAND "" + #BUILD_COMMAND make + #INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install + #UPDATE_DISCONNECTED TRUE +#) link_directories(${OpenBLAS_INSTALL_PREFIX}/lib) include_directories(${OpenBLAS_INSTALL_PREFIX}/include) +# gfortan dir in the docker. link_directories(/usr/local/gcc-8.2/lib64) add_subdirectory(speechx) diff --git a/speechx/build.sh b/speechx/build.sh index 582f9b44..effdacd0 100644 --- a/speechx/build.sh +++ b/speechx/build.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +# the build script had verified in the paddlepaddle docker image. +# please follow the instruction below to install PaddlePaddle image. +# https://www.paddlepaddle.org.cn/documentation/docs/zh/install/docker/linux-docker.html + boost_SOURCE_DIR=$PWD/fc_patch/boost-src if [ ! -d ${boost_SOURCE_DIR} ]; then wget https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz