From 04eb8a285d033def56fa15cc0fbd6c406dc935a2 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Mon, 7 Mar 2022 07:01:38 +0000 Subject: [PATCH 1/3] openblas cmake --- speechx/CMakeLists.txt | 30 ++++----------- speechx/build.sh | 5 ++- speechx/cmake/external/absl.cmake | 5 +++ speechx/cmake/external/gtest.cmake | 1 + speechx/cmake/external/openblas.cmake | 53 +++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 25 deletions(-) create mode 100644 speechx/cmake/external/openblas.cmake diff --git a/speechx/CMakeLists.txt b/speechx/CMakeLists.txt index 494b08d0..34b90116 100644 --- a/speechx/CMakeLists.txt +++ b/speechx/CMakeLists.txt @@ -26,6 +26,9 @@ set(FETCHCONTENT_BASE_DIR ${fc_patch}) # compiler option set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14 -pthread -fPIC -O0 -Wall -g") +SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} --std=c++14 -pthread -fPIC -O0 -Wall -g -ggdb") +SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} --std=c++14 -pthread -fPIC -O3 -Wall") + ############################################################################### # Option Configurations ############################################################################### @@ -134,29 +137,10 @@ set(DEPS ${DEPS} ${EXTERNAL_LIB}) #openblas -set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/openblas-install) -#set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS) -#set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) -#set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix) -#ExternalProject_Add( - #OpenBLAS - #GIT_REPOSITORY https://github.com/xianyi/OpenBLAS - #GIT_TAG 1ef97c470ccf23d06d3bb0597c52df4a2e9093b9 - #GIT_SHALLOW TRUE - #GIT_PROGRESS TRUE - #PREFIX ${OpenBLAS_PREFIX} - #SOURCE_DIR ${OpenBLAS_SOURCE_DIR} - #BUILD_IN_SOURCE TRUE - #CONFIGURE_COMMAND "" - #BUILD_COMMAND make - #INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install - #UPDATE_DISCONNECTED TRUE -#) -link_directories(${OpenBLAS_INSTALL_PREFIX}/lib) -include_directories(${OpenBLAS_INSTALL_PREFIX}/include) - -# gfortan dir in the docker. -link_directories(/usr/local/gcc-8.2/lib64) +include(openblas) + +# openblas need, gfortan dir in the docker. +#link_directories(/usr/local/gcc-8.2/lib64) ############################################################################### # Add local library diff --git a/speechx/build.sh b/speechx/build.sh index a5c93267..15caa18a 100755 --- a/speechx/build.sh +++ b/speechx/build.sh @@ -16,11 +16,12 @@ if [ ! -d ${boost_SOURCE_DIR} ]; then wget -c https://boostorg.jfrog.io/artifact echo -e "\n" fi +rm -rf build mkdir -p build cd build -cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR} --target clean +cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR} -make +make -j cd - diff --git a/speechx/cmake/external/absl.cmake b/speechx/cmake/external/absl.cmake index b22e56d8..2c5e5af5 100644 --- a/speechx/cmake/external/absl.cmake +++ b/speechx/cmake/external/absl.cmake @@ -1,5 +1,10 @@ include(FetchContent) + +set(BUILD_SHARED_LIBS OFF) # up to you +set(BUILD_TESTING OFF) # to disable abseil test, or gtest will fail. +set(ABSL_ENABLE_INSTALL ON) # now you can enable install rules even in subproject... + FetchContent_Declare( absl GIT_REPOSITORY "https://github.com/abseil/abseil-cpp.git" diff --git a/speechx/cmake/external/gtest.cmake b/speechx/cmake/external/gtest.cmake index da6f7cc5..7fe397fc 100644 --- a/speechx/cmake/external/gtest.cmake +++ b/speechx/cmake/external/gtest.cmake @@ -5,4 +5,5 @@ FetchContent_Declare( URL_HASH SHA256=94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91 ) FetchContent_MakeAvailable(gtest) + include_directories(${gtest_BINARY_DIR} ${gtest_SOURCE_DIR}/src) \ No newline at end of file diff --git a/speechx/cmake/external/openblas.cmake b/speechx/cmake/external/openblas.cmake new file mode 100644 index 00000000..141860fe --- /dev/null +++ b/speechx/cmake/external/openblas.cmake @@ -0,0 +1,53 @@ +include(FetchContent) + +# set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/openblas-install) +# set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS) + +set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) +set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix) + +# ExternalProject_Add( +# OpenBLAS +# GIT_REPOSITORY https://github.com/xianyi/OpenBLAS +# GIT_TAG 1ef97c470ccf23d06d3bb0597c52df4a2e9093b9 +# GIT_SHALLOW TRUE +# GIT_PROGRESS TRUE +# PREFIX ${OpenBLAS_PREFIX} +# SOURCE_DIR ${OpenBLAS_SOURCE_DIR} +# BUILD_IN_SOURCE TRUE +# CONFIGURE_COMMAND "" +# BUILD_COMMAND make +# INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install +# UPDATE_DISCONNECTED TRUE +# ) + +# ###################################################################################################################### +# OPENBLAS https://github.com/lattice/quda/blob/develop/CMakeLists.txt#L575 +# ###################################################################################################################### +enable_language(Fortran) +#TODO: switch to CPM +include(GNUInstallDirs) +ExternalProject_Add( + OPENBLAS + GIT_REPOSITORY https://github.com/xianyi/OpenBLAS.git + GIT_TAG v0.3.10 + GIT_SHALLOW YES + PREFIX ${OpenBLAS_PREFIX} + SOURCE_DIR ${OpenBLAS_SOURCE_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= + CMAKE_GENERATOR "Unix Makefiles") +# https://cmake.org/cmake/help/latest/module/ExternalProject.html?highlight=externalproject_get_property#external-project-definition +ExternalProject_Get_Property(OPENBLAS INSTALL_DIR) +set(OpenBLAS_INSTALL_PREFIX ${INSTALL_DIR}) +add_library(openblas STATIC IMPORTED) +add_dependencies(openblas OPENBLAS) +set_target_properties(openblas PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES Fortran) +# ${CMAKE_INSTALL_LIBDIR} lib +set_target_properties(openblas PROPERTIES IMPORTED_LOCATION ${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libopenblas.a) + + +# https://cmake.org/cmake/help/latest/command/install.html?highlight=cmake_install_libdir#installing-targets +# ${CMAKE_INSTALL_LIBDIR} lib +# ${CMAKE_INSTALL_INCLUDEDIR} include +link_directories(${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}) +include_directories(${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) \ No newline at end of file From 36d16fffcdad186a8db15e9523471ea848093d54 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Mon, 7 Mar 2022 07:17:13 +0000 Subject: [PATCH 2/3] libsndfile from repo; rm useless openblas --- speechx/CMakeLists.txt | 8 ++++---- speechx/build.sh | 2 +- speechx/cmake/EnableCMP0048.cmake | 2 +- speechx/cmake/external/libsndfile.cmake | 6 +++--- speechx/cmake/external/openblas.cmake | 18 ------------------ 5 files changed, 9 insertions(+), 27 deletions(-) diff --git a/speechx/CMakeLists.txt b/speechx/CMakeLists.txt index 34b90116..dcb13a71 100644 --- a/speechx/CMakeLists.txt +++ b/speechx/CMakeLists.txt @@ -72,6 +72,8 @@ find_package(Eigen3 REQUIRED) include(kenlm) add_dependencies(kenlm eigen boost) +#openblas +include(openblas) # openfst set(openfst_SOURCE_DIR ${fc_patch}/openfst-src) @@ -94,6 +96,8 @@ add_dependencies(openfst gflags glog) link_directories(${openfst_PREFIX_DIR}/lib) include_directories(${openfst_PREFIX_DIR}/include) + + # paddle lib set(paddle_SOURCE_DIR ${fc_patch}/paddle-lib) set(paddle_PREFIX_DIR ${fc_patch}/paddle-lib-prefix) @@ -136,11 +140,7 @@ set(DEPS ${DEPS} glog gflags protobuf xxhash cryptopp ${EXTERNAL_LIB}) -#openblas -include(openblas) -# openblas need, gfortan dir in the docker. -#link_directories(/usr/local/gcc-8.2/lib64) ############################################################################### # Add local library diff --git a/speechx/build.sh b/speechx/build.sh index 15caa18a..3c63928e 100755 --- a/speechx/build.sh +++ b/speechx/build.sh @@ -22,6 +22,6 @@ cd build cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR} -make -j +make -j1 cd - diff --git a/speechx/cmake/EnableCMP0048.cmake b/speechx/cmake/EnableCMP0048.cmake index f9afa013..1b59188f 100644 --- a/speechx/cmake/EnableCMP0048.cmake +++ b/speechx/cmake/EnableCMP0048.cmake @@ -1 +1 @@ -cmake_policy(SET CMP0048 NEW) +cmake_policy(SET CMP0048 NEW) \ No newline at end of file diff --git a/speechx/cmake/external/libsndfile.cmake b/speechx/cmake/external/libsndfile.cmake index fc5e0a8a..52d64bac 100644 --- a/speechx/cmake/external/libsndfile.cmake +++ b/speechx/cmake/external/libsndfile.cmake @@ -11,8 +11,8 @@ else() set(FETCHCONTENT_SOURCE_DIR_LIBSNDFILE "") endif() -set(LIBSNDFILE_GIT_REPO "https://github.com/erikd/libsndfile" CACHE STRING "libsndfile git repository url" FORCE) -set(LIBSNDFILE_GIT_TAG c11deaa04ec84161996824061f6d705970972e2e CACHE STRING "libsndfile git tag" FORCE) +set(LIBSNDFILE_GIT_REPO "https://github.com/libsndfile/libsndfile.git" CACHE STRING "libsndfile git repository url" FORCE) +set(LIBSNDFILE_GIT_TAG 1.0.31 CACHE STRING "libsndfile git tag" FORCE) FetchContent_Declare(libsndfile GIT_REPOSITORY ${LIBSNDFILE_GIT_REPO} @@ -48,7 +48,7 @@ function(libsndfile_build) # finally we include libsndfile itself add_subdirectory(${libsndfile_SOURCE_DIR} ${libsndfile_BINARY_DIR} EXCLUDE_FROM_ALL) # copying .hh for c++ support - file(COPY "${libsndfile_SOURCE_DIR}/src/sndfile.hh" DESTINATION ${LIBSNDFILE_INCLUDE_DIR}) + #file(COPY "${libsndfile_SOURCE_DIR}/src/sndfile.hh" DESTINATION ${LIBSNDFILE_INCLUDE_DIR}) endfunction() libsndfile_build() diff --git a/speechx/cmake/external/openblas.cmake b/speechx/cmake/external/openblas.cmake index 141860fe..14e17195 100644 --- a/speechx/cmake/external/openblas.cmake +++ b/speechx/cmake/external/openblas.cmake @@ -1,26 +1,8 @@ include(FetchContent) -# set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/openblas-install) -# set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS) - set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix) -# ExternalProject_Add( -# OpenBLAS -# GIT_REPOSITORY https://github.com/xianyi/OpenBLAS -# GIT_TAG 1ef97c470ccf23d06d3bb0597c52df4a2e9093b9 -# GIT_SHALLOW TRUE -# GIT_PROGRESS TRUE -# PREFIX ${OpenBLAS_PREFIX} -# SOURCE_DIR ${OpenBLAS_SOURCE_DIR} -# BUILD_IN_SOURCE TRUE -# CONFIGURE_COMMAND "" -# BUILD_COMMAND make -# INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install -# UPDATE_DISCONNECTED TRUE -# ) - # ###################################################################################################################### # OPENBLAS https://github.com/lattice/quda/blob/develop/CMakeLists.txt#L575 # ###################################################################################################################### From 3887a9b2a593ef1f8ab5ccb10fde48de97c896d5 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Mon, 7 Mar 2022 08:23:39 +0000 Subject: [PATCH 3/3] openfst.cmake --- speechx/CMakeLists.txt | 25 ++++--------------------- speechx/build.sh | 1 + speechx/cmake/external/boost.cmake | 27 +++++++++++++++++++++++++++ speechx/cmake/external/glog.cmake | 2 -- speechx/cmake/external/kenlm.cmake | 4 ---- speechx/cmake/external/openfst.cmake | 19 +++++++++++++++++++ 6 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 speechx/cmake/external/boost.cmake create mode 100644 speechx/cmake/external/openfst.cmake diff --git a/speechx/CMakeLists.txt b/speechx/CMakeLists.txt index dcb13a71..083e180d 100644 --- a/speechx/CMakeLists.txt +++ b/speechx/CMakeLists.txt @@ -24,8 +24,8 @@ get_filename_component(fc_patch "fc_patch" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR set(FETCHCONTENT_BASE_DIR ${fc_patch}) # compiler option +# Keep the same with openfst, -fPIC or -fpic set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14 -pthread -fPIC -O0 -Wall -g") - SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} --std=c++14 -pthread -fPIC -O0 -Wall -g -ggdb") SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} --std=c++14 -pthread -fPIC -O3 -Wall") @@ -60,9 +60,10 @@ include(absl) include(libsndfile) # boost +# include(boost) # not work set(boost_SOURCE_DIR ${fc_patch}/boost-src) set(BOOST_ROOT ${boost_SOURCE_DIR}) -#find_package(boost REQUIRED PATHS ${BOOST_ROOT}) +# #find_package(boost REQUIRED PATHS ${BOOST_ROOT}) # Eigen include(eigen) @@ -76,26 +77,8 @@ add_dependencies(kenlm eigen boost) include(openblas) # openfst -set(openfst_SOURCE_DIR ${fc_patch}/openfst-src) -set(openfst_BINARY_DIR ${fc_patch}/openfst-build) -set(openfst_PREFIX_DIR ${fc_patch}/openfst-subbuild/openfst-populate-prefix) -ExternalProject_Add(openfst - URL https://github.com/mjansche/openfst/archive/refs/tags/1.7.2.zip - URL_HASH SHA256=ffc56931025579a8af3515741c0f3b0fc3a854c023421472c07ca0c6389c75e6 - #PREFIX ${openfst_PREFIX_DIR} - SOURCE_DIR ${openfst_SOURCE_DIR} - BINARY_DIR ${openfst_BINARY_DIR} - CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure --prefix=${openfst_PREFIX_DIR} - "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR}" - "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}" - "LIBS=-lgflags_nothreads -lglog -lpthread" - COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR} - BUILD_COMMAND make -j 4 -) +include(openfst) add_dependencies(openfst gflags glog) -link_directories(${openfst_PREFIX_DIR}/lib) -include_directories(${openfst_PREFIX_DIR}/include) - # paddle lib diff --git a/speechx/build.sh b/speechx/build.sh index 3c63928e..b3406387 100755 --- a/speechx/build.sh +++ b/speechx/build.sh @@ -21,6 +21,7 @@ mkdir -p build cd build cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR} +#cmake .. make -j1 diff --git a/speechx/cmake/external/boost.cmake b/speechx/cmake/external/boost.cmake new file mode 100644 index 00000000..6bc97aad --- /dev/null +++ b/speechx/cmake/external/boost.cmake @@ -0,0 +1,27 @@ +include(FetchContent) +set(Boost_DEBUG ON) + +set(Boost_PREFIX_DIR ${fc_patch}/boost) +set(Boost_SOURCE_DIR ${fc_patch}/boost-src) + +FetchContent_Declare( + Boost + URL https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz + URL_HASH SHA256=aeb26f80e80945e82ee93e5939baebdca47b9dee80a07d3144be1e1a6a66dd6a + PREFIX ${Boost_PREFIX_DIR} + SOURCE_DIR ${Boost_SOURCE_DIR} +) + +execute_process(COMMAND bootstrap.sh WORKING_DIRECTORY ${Boost_SOURCE_DIR}) +execute_process(COMMAND b2 WORKING_DIRECTORY ${Boost_SOURCE_DIR}) + +FetchContent_MakeAvailable(Boost) + +message(STATUS "boost src dir: ${Boost_SOURCE_DIR}") +message(STATUS "boost inc dir: ${Boost_INCLUDE_DIR}") +message(STATUS "boost bin dir: ${Boost_BINARY_DIR}") + +set(BOOST_ROOT ${Boost_SOURCE_DIR}) +message(STATUS "boost root dir: ${BOOST_ROOT}") + +include_directories(${Boost_SOURCE_DIR}) \ No newline at end of file diff --git a/speechx/cmake/external/glog.cmake b/speechx/cmake/external/glog.cmake index 7bde9652..dcfd86c3 100644 --- a/speechx/cmake/external/glog.cmake +++ b/speechx/cmake/external/glog.cmake @@ -1,10 +1,8 @@ include(FetchContent) - FetchContent_Declare( glog URL https://github.com/google/glog/archive/v0.4.0.zip URL_HASH SHA256=9e1b54eb2782f53cd8af107ecf08d2ab64b8d0dc2b7f5594472f3bd63ca85cdc ) FetchContent_MakeAvailable(glog) - include_directories(${glog_BINARY_DIR} ${glog_SOURCE_DIR}/src) diff --git a/speechx/cmake/external/kenlm.cmake b/speechx/cmake/external/kenlm.cmake index 61f9714b..17c76c3f 100644 --- a/speechx/cmake/external/kenlm.cmake +++ b/speechx/cmake/external/kenlm.cmake @@ -1,14 +1,10 @@ include(FetchContent) - FetchContent_Declare( kenlm GIT_REPOSITORY "https://github.com/kpu/kenlm.git" GIT_TAG "df2d717e95183f79a90b2fa6e4307083a351ca6a" ) - # https://github.com/kpu/kenlm/blob/master/cmake/modules/FindEigen3.cmake set(EIGEN3_INCLUDE_DIR ${Eigen3_SOURCE_DIR}) - FetchContent_MakeAvailable(kenlm) - include_directories(${kenlm_SOURCE_DIR}) \ No newline at end of file diff --git a/speechx/cmake/external/openfst.cmake b/speechx/cmake/external/openfst.cmake new file mode 100644 index 00000000..07abb18e --- /dev/null +++ b/speechx/cmake/external/openfst.cmake @@ -0,0 +1,19 @@ +include(FetchContent) +set(openfst_SOURCE_DIR ${fc_patch}/openfst-src) +set(openfst_BINARY_DIR ${fc_patch}/openfst-build) + +ExternalProject_Add(openfst + URL https://github.com/mjansche/openfst/archive/refs/tags/1.7.2.zip + URL_HASH SHA256=ffc56931025579a8af3515741c0f3b0fc3a854c023421472c07ca0c6389c75e6 +# #PREFIX ${openfst_PREFIX_DIR} +# SOURCE_DIR ${openfst_SOURCE_DIR} +# BINARY_DIR ${openfst_BINARY_DIR} + CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure --prefix=${openfst_PREFIX_DIR} + "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR}" + "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}" + "LIBS=-lgflags_nothreads -lglog -lpthread" + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR} + BUILD_COMMAND make -j 4 +) +link_directories(${openfst_PREFIX_DIR}/lib) +include_directories(${openfst_PREFIX_DIR}/include)