From 52477a1030e5e326e66df96f3108810107c21ee4 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 29 Jun 2022 05:08:02 +0000 Subject: [PATCH] sox can build --- CMakeLists.txt | 23 ++++- .../cmake => cmake}/FindGFortranLibs.cmake | 0 .../cmake => cmake}/external/openblas.cmake | 1 + .../cmake => cmake}/external/pybind.cmake | 2 + cmake/summary.cmake | 9 +- paddlespeech/audio/CMakeLists.txt | 37 -------- paddlespeech/audio/third_party/CMakeLists.txt | 15 ++- .../audio/third_party/kaldi/CMakeLists.txt | 93 +++++++++++++------ tools/setup_helpers/extension.py | 4 +- 9 files changed, 103 insertions(+), 81 deletions(-) rename {paddlespeech/audio/cmake => cmake}/FindGFortranLibs.cmake (100%) rename {paddlespeech/audio/cmake => cmake}/external/openblas.cmake (98%) rename {paddlespeech/audio/cmake => cmake}/external/pybind.cmake (92%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8778776b..2912b6132 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,8 +13,10 @@ if(NOT CMAKE_VERSION VERSION_LESS 3.15.0) cmake_policy(SET CMP0092 NEW) endif() + project(paddlespeech) + # check and set CMAKE_CXX_STANDARD string(FIND "${CMAKE_CXX_FLAGS}" "-std=c++" env_cxx_standard) if(env_cxx_standard GREATER -1) @@ -27,23 +29,38 @@ endif() set(CMAKE_CXX_STANDARD 14) set(CMAKE_C_STANDARD 11) - set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_VERBOSE_MAKEFILE ON) + # Options option(BUILD_SOX "Build libsox statically" ON) option(BUILD_MAD "Enable libmad" ON) +option(BUILD_KALDI "Build kaldi statically" ON) option(BUILD_PADDLEAUDIO_PYTHON_EXTENSION "Build Python extension" ON) -set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake") +# cmake +set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${PROJECT_SOURCE_DIR}/cmake;${PROJECT_SOURCE_DIR}/cmake/external") + + +# fc_patch dir +set(FETCHCONTENT_QUIET off) +get_filename_component(fc_patch "fc_patch" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") +set(FETCHCONTENT_BASE_DIR ${fc_patch}) + + +include(openblas) +# include(pybind) + +# packages find_package(Python3 COMPONENTS Interpreter Development) find_package(pybind11 CONFIG) -message(STATUS "Python_INCLUDE_DIR=" ${Python_INCLUDE_DIR}) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -fPIC -O0 -Wall -g") add_subdirectory(paddlespeech/audio) # Summary diff --git a/paddlespeech/audio/cmake/FindGFortranLibs.cmake b/cmake/FindGFortranLibs.cmake similarity index 100% rename from paddlespeech/audio/cmake/FindGFortranLibs.cmake rename to cmake/FindGFortranLibs.cmake diff --git a/paddlespeech/audio/cmake/external/openblas.cmake b/cmake/external/openblas.cmake similarity index 98% rename from paddlespeech/audio/cmake/external/openblas.cmake rename to cmake/external/openblas.cmake index 0c204f3a2..a108f49a2 100644 --- a/paddlespeech/audio/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -1,4 +1,5 @@ include(FetchContent) +include(ExternalProject) set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src) set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix) diff --git a/paddlespeech/audio/cmake/external/pybind.cmake b/cmake/external/pybind.cmake similarity index 92% rename from paddlespeech/audio/cmake/external/pybind.cmake rename to cmake/external/pybind.cmake index 2d74e61eb..941918970 100644 --- a/paddlespeech/audio/cmake/external/pybind.cmake +++ b/cmake/external/pybind.cmake @@ -1,4 +1,6 @@ include(FetchContent) +include(ExternalProject) + FetchContent_Declare( pybind URL https://github.com/pybind/pybind11/archive/refs/tags/v2.9.0.zip diff --git a/cmake/summary.cmake b/cmake/summary.cmake index 78b6988e4..f1b5d3c5e 100644 --- a/cmake/summary.cmake +++ b/cmake/summary.cmake @@ -33,9 +33,8 @@ function (onnx_print_configuration_summary) message(STATUS " Protobuf includes : ${PROTOBUF_INCLUDE_DIRS}") message(STATUS " Protobuf libraries : ${PROTOBUF_LIBRARIES}") message(STATUS " BUILD_ONNX_PYTHON : ${BUILD_ONNX_PYTHON}") - if (${BUILD_ONNX_PYTHON}) - message(STATUS " Python version : ${PY_VERSION}") - message(STATUS " Python executable : ${PYTHON_EXECUTABLE}") - message(STATUS " Python includes : ${PYTHON_INCLUDE_DIR}") - endif() + message(STATUS " Python version : ${Python_VERSION}") + message(STATUS " Python executable : ${Python_EXECUTABLE}") + message(STATUS " Python includes : ${Python_INCLUDE_DIRS}") + endfunction() \ No newline at end of file diff --git a/paddlespeech/audio/CMakeLists.txt b/paddlespeech/audio/CMakeLists.txt index 8566227a8..c6b43c780 100644 --- a/paddlespeech/audio/CMakeLists.txt +++ b/paddlespeech/audio/CMakeLists.txt @@ -1,40 +1,3 @@ -cmake_minimum_required(VERSION 3.16 FATAL_ERROR) - -project(paddleaudio VERSION 0.1) - -string(FIND "${CMAKE_CXX_FLAGS}" "-std=c++" env_cxx_standard) - -# cmake dir -set(paddleaudio_cmake_dir ${PROJECT_SOURCE_DIR}/cmake) - -# Modules -list(APPEND CMAKE_MODULE_PATH ${paddleaudio_cmake_dir}/external) -list(APPEND CMAKE_MODULE_PATH ${paddleaudio_cmake_dir}) -include(FetchContent) -include(ExternalProject) - -# fc_patch dir -set(FETCHCONTENT_QUIET off) -get_filename_component(fc_patch "fc_patch" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") -set(FETCHCONTENT_BASE_DIR ${fc_patch}) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -fPIC -O0 -Wall -g") -set(CMAKE_CXX_STANDARD 14) -set(CMAKE_C_STANDARD 11) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -option(BUILD_SOX "Build libsox statically" ON) - - -# checkout the thirdparty/kaldi/base/kaldi-types.h -# compile kaldi without openfst -add_definitions("-DCOMPILE_WITHOUT_OPENFST") - -include(openblas) -# include(pybind) - -include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/third_party/kaldi) -include_directories(/usr/include/python3.7m) add_subdirectory(third_party) add_subdirectory(src) diff --git a/paddlespeech/audio/third_party/CMakeLists.txt b/paddlespeech/audio/third_party/CMakeLists.txt index fc9f332f4..8dc43c36f 100644 --- a/paddlespeech/audio/third_party/CMakeLists.txt +++ b/paddlespeech/audio/third_party/CMakeLists.txt @@ -1,11 +1,16 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") +################################################################################ +# sox +################################################################################ if (BUILD_SOX) add_subdirectory(sox) endif() -include_directories( -${CMAKE_CURRENT_SOURCE_DIR} -${CMAKE_CURRENT_SOURCE_DIR}/kaldi -) -add_subdirectory(kaldi) +################################################################################ +# kaldi +################################################################################ +if (BUILD_KALDI) + add_subdirectory(kaldi) + message(STATUS "Build Kaldi") +endif() \ No newline at end of file diff --git a/paddlespeech/audio/third_party/kaldi/CMakeLists.txt b/paddlespeech/audio/third_party/kaldi/CMakeLists.txt index 66346a268..2310fb8c6 100644 --- a/paddlespeech/audio/third_party/kaldi/CMakeLists.txt +++ b/paddlespeech/audio/third_party/kaldi/CMakeLists.txt @@ -1,14 +1,58 @@ -project(kaldi) +# checkout the thirdparty/kaldi/base/kaldi-types.h +# compile kaldi without openfst +add_definitions("-DCOMPILE_WITHOUT_OPENFST") -add_library(kaldi-base +# include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/kaldi) +# include_directories(/usr/include/python3.7m) + +set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +# function (define_library name source include_dirs link_libraries compile_defs) +# add_library(${name} INTERFACE ${source}) +# target_include_directories(${name} INTERFACE ${include_dirs}) +# target_link_libraries(${name} INTERFACE ${link_libraries}) +# target_compile_definitions(${name} INTERFACE ${compile_defs}) +# set_target_properties(${name} PROPERTIES PREFIX "") +# if (MSVC) +# set_target_properties(${name} PROPERTIES SUFFIX ".pyd") +# endif(MSVC) +# install( +# TARGETS ${name} +# LIBRARY DESTINATION lib +# RUNTIME DESTINATION lib # For Windows +# ) +# endfunction() + +# kaldi-base +add_library(kaldi-base INTERFACE base/io-funcs.cc base/kaldi-error.cc base/kaldi-math.cc base/kaldi-utils.cc base/timer.cc ) +target_include_directories(kaldi-base INTERFACE ${INSTALL_DIR}/base) +target_compile_definitions(kaldi-base INTERFACE "-DCOMPILE_WITHOUT_OPENFST") + +# kaldi-matrix +add_library(kaldi-matrix INTERFACE + matrix/compressed-matrix.cc + matrix/kaldi-matrix.cc + matrix/kaldi-vector.cc + matrix/matrix-functions.cc + matrix/optimization.cc + matrix/packed-matrix.cc + matrix/qr.cc + matrix/sparse-matrix.cc + matrix/sp-matrix.cc + matrix/srfft.cc + matrix/tp-matrix.cc +) +target_include_directories(kaldi-matrix INTERFACE ${INSTALL_DIR}/matrix) +target_link_libraries(kaldi-matrix INTERFACE gfortran kaldi-base libopenblas.a) -add_library(kaldi-util +# kaldi-util +add_library(kaldi-util INTERFACE util/kaldi-holder.cc util/kaldi-io.cc util/kaldi-semaphore.cc @@ -19,19 +63,11 @@ add_library(kaldi-util util/simple-options.cc util/text-utils.cc ) -target_link_libraries(kaldi-util PUBLIC kaldi-base kaldi-matrix) - -add_library(kaldi-mfcc - feat/feature-mfcc.cc -) -target_link_libraries(kaldi-mfcc PUBLIC kaldi-feat-common) +target_include_directories(kaldi-util INTERFACE ${INSTALL_DIR}/util) +target_link_libraries(kaldi-util INTERFACE kaldi-base kaldi-matrix) -add_library(kaldi-fbank - feat/feature-fbank.cc -) -target_link_libraries(kaldi-fbank PUBLIC kaldi-feat-common) - -add_library(kaldi-feat-common +# kaldi-feat-common +add_library(kaldi-feat-common INTERFACE feat/wave-reader.cc feat/signal.cc feat/feature-functions.cc @@ -40,20 +76,19 @@ add_library(kaldi-feat-common feat/mel-computations.cc feat/cmvn.cc ) -target_link_libraries(kaldi-feat-common PUBLIC kaldi-base kaldi-matrix kaldi-util) +target_include_directories(kaldi-feat-common INTERFACE ${INSTALL_DIR}/feat) +target_link_libraries(kaldi-feat-common INTERFACE kaldi-base kaldi-matrix kaldi-util) -add_library(kaldi-matrix - matrix/compressed-matrix.cc - matrix/kaldi-matrix.cc - matrix/kaldi-vector.cc - matrix/matrix-functions.cc - matrix/optimization.cc - matrix/packed-matrix.cc - matrix/qr.cc - matrix/sparse-matrix.cc - matrix/sp-matrix.cc - matrix/srfft.cc - matrix/tp-matrix.cc +# kaldi-mfcc +add_library(kaldi-mfcc INTERFACE + feat/feature-mfcc.cc ) -target_link_libraries(kaldi-matrix gfortran kaldi-base libopenblas.a) +target_include_directories(kaldi-mfcc INTERFACE ${INSTALL_DIR}/feat) +target_link_libraries(kaldi-mfcc INTERFACE kaldi-feat-common) +# kaldi-fbank +add_library(kaldi-fbank INTERFACE + feat/feature-fbank.cc +) +target_include_directories(kaldi-fbank INTERFACE ${INSTALL_DIR}/feat) +target_link_libraries(kaldi-fbank INTERFACE kaldi-feat-common) \ No newline at end of file diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index ae84f09f4..e8b48ec60 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -35,7 +35,7 @@ def _get_build(var, default=False): _BUILD_SOX = False if platform.system() == "Windows" else _get_build( "BUILD_SOX", True) _BUILD_MAD = _get_build("BUILD_MAD", False) -# _BUILD_KALDI = False if platform.system() == "Windows" else _get_build("BUILD_KALDI", True) +_BUILD_KALDI = False if platform.system() == "Windows" else _get_build("BUILD_KALDI", True) # _BUILD_RNNT = _get_build("BUILD_RNNT", True) # _BUILD_CTC_DECODER = False if platform.system() == "Windows" else _get_build("BUILD_CTC_DECODER", True) # _USE_FFMPEG = _get_build("USE_FFMPEG", False) @@ -88,7 +88,7 @@ class CMakeBuild(build_ext): # f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", f"-DCMAKE_INSTALL_PREFIX={extdir}", "-DCMAKE_VERBOSE_MAKEFILE=ON", - f"-DPython_INCLUDE_DIR={distutils.sysconfig.get_python_inc()}", + f"-DPython_INCLUDE_DIRS={distutils.sysconfig.get_python_inc()}", f"-DBUILD_SOX:BOOL={'ON' if _BUILD_SOX else 'OFF'}", f"-DBUILD_MAD:BOOL={'ON' if _BUILD_MAD else 'OFF'}", # f"-DBUILD_KALDI:BOOL={'ON' if _BUILD_KALDI else 'OFF'}",