mv kaldi feature to paddlespeech/audio

audio/CMakeLists.txt

@@ -25,19 +25,4 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 option(BUILD_SOX "Build libsox statically" ON)
 
-
-# checkout the thirdparty/kaldi/base/kaldi-types.h
-# compile kaldi without openfst
-add_definitions("-DCOMPILE_WITHOUT_OPENFST")
-
-#include(openblas)
-include(pybind)
-set(fc_patch /paddle/mnt/zhouyang/wks2/PaddleSpeech/paddleaudio/fc_patch)
-set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/openblas-install)
-link_directories(${OpenBLAS_INSTALL_PREFIX}/lib)
-include_directories(${OpenBLAS_INSTALL_PREFIX}/include)
-
-include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/third_party/kaldi)
-include_directories(/usr/include/python3.7m)
-add_subdirectory(third_party)
 add_subdirectory(audio/csrc)

audio/audio/csrc/CMakeLists.txt

@@ -32,5 +32,3 @@ if(BUILD_SOX)
     "${LINK_LIBRARIES}"
     )
 endif()
-
-add_subdirectory(pybind/kaldi_frontend)

paddlespeech/audio/.gitignore

@@ -0,0 +1 @@
+fc_patch/

paddlespeech/audio/CMakeLists.txt

@@ -0,0 +1,39 @@
+cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
+
+project(paddleaudio VERSION 0.1)
+
+string(FIND "${CMAKE_CXX_FLAGS}" "-std=c++" env_cxx_standard)
+
+# cmake dir
+set(paddleaudio_cmake_dir ${PROJECT_SOURCE_DIR}/cmake)
+
+# Modules
+list(APPEND CMAKE_MODULE_PATH ${paddleaudio_cmake_dir}/external)
+list(APPEND CMAKE_MODULE_PATH ${paddleaudio_cmake_dir})
+include(FetchContent)
+include(ExternalProject)
+
+# fc_patch dir
+set(FETCHCONTENT_QUIET off)
+get_filename_component(fc_patch "fc_patch" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}")
+set(FETCHCONTENT_BASE_DIR ${fc_patch})
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -fPIC -O0 -Wall -g")
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+option(BUILD_SOX "Build libsox statically" ON)
+
+
+# checkout the thirdparty/kaldi/base/kaldi-types.h
+# compile kaldi without openfst
+add_definitions("-DCOMPILE_WITHOUT_OPENFST")
+
+include(openblas)
+include(pybind)
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/third_party/kaldi)
+include_directories(/usr/include/python3.7m)
+add_subdirectory(third_party)
+add_subdirectory(csrc)

paddlespeech/audio/cmake/FindGFortranLibs.cmake

@@ -0,0 +1,145 @@
+#.rst:
+# FindGFortranLibs
+# --------
+#  https://github.com/Argonne-National-Laboratory/PIPS/blob/master/cmake/Modules/FindGFortranLibs.cmake
+#  https://enccs.github.io/cmake-workshop/cxx-fortran/
+#
+# Find gcc Fortran compiler & library paths
+#
+# The module defines the following variables:
+#
+# ::
+#
+#
+#   GFORTRANLIBS_FOUND - true if system has gfortran
+#   LIBGFORTRAN_LIBRARIES - path to libgfortran
+#   LIBQUADMATH_LIBRARIES - path to libquadmath
+#   GFORTRAN_LIBARIES_DIR - directory containing libgfortran, libquadmath
+#   GFORTRAN_INCLUDE_DIR - directory containing gfortran/gcc headers
+#   LIBGOMP_LIBRARIES - path to libgomp
+#   LIBGOMP_INCLUDE_DIR - directory containing omp.h header
+#   GFORTRAN_VERSION_STRING - version of gfortran found
+#
+set(CMAKE_REQUIRED_QUIET ${LIBIOMP_FIND_QUIETLY})
+
+if(NOT CMAKE_REQUIRED_QUIET)
+  message(STATUS "Looking for gfortran related libraries...")
+endif()
+
+enable_language(Fortran)
+if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+
+  # Basically, call "gfortran -v" to dump compiler info to the string
+  # GFORTRAN_VERBOSE_STR, which will be used to get necessary paths
+  message(STATUS "Extracting library and header information by calling 'gfortran -v'...")
+  execute_process(COMMAND "${CMAKE_Fortran_COMPILER}" "-v" ERROR_VARIABLE
+    GFORTRAN_VERBOSE_STR RESULT_VARIABLE FLAG)
+
+  # For debugging
+  message(STATUS "'gfortran -v' returned:")
+  message(STATUS "${GFORTRAN_VERBOSE_STR}")
+
+  # Detect gfortran version
+  string(REGEX MATCH "gcc version [^\t\n ]+" GFORTRAN_VER_STR "${GFORTRAN_VERBOSE_STR}")
+  string(REGEX REPLACE "gcc version ([^\t\n ]+)" "\\1" GFORTRAN_VERSION_STRING "${GFORTRAN_VER_STR}")
+  message(STATUS "Detected gfortran version ${GFORTRAN_VERSION_STRING}")
+  unset(GFORTRAN_VER_STR)
+
+  set(MATCH_REGEX "[^\t\n ]+[\t\n ]+")
+  set(REPLACE_REGEX "([^\t\n ]+)")
+
+  # Find architecture for compiler
+  string(REGEX MATCH "Target: [^\t\n ]+"
+    GFORTRAN_ARCH_STR "${GFORTRAN_VERBOSE_STR}")
+  message(STATUS "Architecture string: ${GFORTRAN_ARCH_STR}")
+  string(REGEX REPLACE "Target: ([^\t\n ]+)" "\\1"
+    GFORTRAN_ARCH "${GFORTRAN_ARCH_STR}")
+  message(STATUS "Detected gfortran architecture: ${GFORTRAN_ARCH}")
+  unset(GFORTRAN_ARCH_STR)
+
+  # Find install prefix, if it exists; if not, use default
+  string(REGEX MATCH  "--prefix=[^\t\n ]+[\t\n ]+"
+    GFORTRAN_PREFIX_STR "${GFORTRAN_VERBOSE_STR}")
+  if(NOT GFORTRAN_PREFIX_STR)
+    message(STATUS "Detected default gfortran prefix")
+    set(GFORTRAN_PREFIX_DIR "/usr/local") # default prefix for gcc install
+  else()
+    string(REGEX REPLACE "--prefix=([^\t\n ]+)" "\\1"
+      GFORTRAN_PREFIX_DIR "${GFORTRAN_PREFIX_STR}")
+  endif()
+  message(STATUS "Detected gfortran prefix: ${GFORTRAN_PREFIX_DIR}")
+  unset(GFORTRAN_PREFIX_STR)
+
+  # Find install exec-prefix, if it exists; if not, use default
+  string(REGEX MATCH "--exec-prefix=[^\t\n ]+[\t\n ]+" "\\1"
+    GFORTRAN_EXEC_PREFIX_STR "${GFORTRAN_VERBOSE_STR}")
+  if(NOT GFORTRAN_EXEC_PREFIX_STR)
+    message(STATUS "Detected default gfortran exec-prefix")
+    set(GFORTRAN_EXEC_PREFIX_DIR "${GFORTRAN_PREFIX_DIR}")
+  else()
+    string(REGEX REPLACE "--exec-prefix=([^\t\n ]+)" "\\1"
+      GFORTRAN_EXEC_PREFIX_DIR "${GFORTRAN_EXEC_PREFIX_STR}")
+  endif()
+  message(STATUS "Detected gfortran exec-prefix: ${GFORTRAN_EXEC_PREFIX_DIR}")
+  UNSET(GFORTRAN_EXEC_PREFIX_STR)
+
+  # Find library directory and include directory, if library directory specified
+  string(REGEX MATCH "--libdir=[^\t\n ]+"
+    GFORTRAN_LIB_DIR_STR "${GFORTRAN_VERBOSE_STR}")
+  if(NOT GFORTRAN_LIB_DIR_STR)
+    message(STATUS "Found --libdir flag -- not found")
+    message(STATUS "Using default gfortran library & include directory paths")
+    set(GFORTRAN_LIBRARIES_DIR
+      "${GFORTRAN_EXEC_PREFIX_DIR}/lib/gcc/${GFORTRAN_ARCH}/${GFORTRAN_VERSION_STRING}")
+    string(CONCAT GFORTRAN_INCLUDE_DIR "${GFORTRAN_LIBRARIES_DIR}" "/include")
+  else()
+    message(STATUS "Found --libdir flag -- yes")
+    string(REGEX REPLACE "--libdir=([^\t\n ]+)" "\\1"
+      GFORTRAN_LIBRARIES_DIR "${GFORTRAN_LIB_DIR_STR}")
+    string(CONCAT GFORTRAN_INCLUDE_DIR "${GFORTRAN_LIBRARIES_DIR}" "/gcc/" "${GFORTRAN_ARCH}" "/" "${GFORTRAN_VERSION_STRING}" "/include")
+  endif()
+  message(STATUS "gfortran libraries path: ${GFORTRAN_LIBRARIES_DIR}")
+  message(STATUS "gfortran include path dir: ${GFORTRAN_INCLUDE_DIR}")
+  unset(GFORTRAN_LIB_DIR_STR)
+
+  # There are lots of other build options for gcc & gfortran. For now, the
+  # options implemented above should cover a lot of common use cases.
+
+  # Clean up be deleting the output string from "gfortran -v"
+  unset(GFORTRAN_VERBOSE_STR)
+
+  # Find paths for libgfortran, libquadmath, libgomp
+  # libgomp needed for OpenMP support without Clang
+  find_library(LIBGFORTRAN_LIBRARIES NAMES gfortran libgfortran
+    HINTS ${GFORTRAN_LIBRARIES_DIR})
+  find_library(LIBQUADMATH_LIBRARIES NAMES quadmath libquadmath
+    HINTS ${GFORTRAN_LIBRARIES_DIR})
+  find_library(LIBGOMP_LIBRARIES NAMES gomp libgomp
+    HINTS ${GFORTRAN_LIBRARIES_DIR})
+
+  # Find OpenMP headers
+  find_path(LIBGOMP_INCLUDE_DIR NAMES omp.h HINTS ${GFORTRAN_INCLUDE_DIR})
+
+else()
+  message(STATUS "CMAKE_Fortran_COMPILER_ID does not match 'GNU'!")
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+# Required: libgfortran, libquadmath, path for gfortran libraries
+# Optional: libgomp, path for OpenMP headers, path for gcc/gfortran headers
+find_package_handle_standard_args(GFortranLibs
+  REQUIRED_VARS LIBGFORTRAN_LIBRARIES LIBQUADMATH_LIBRARIES GFORTRAN_LIBRARIES_DIR
+  VERSION_VAR GFORTRAN_VERSION_STRING)
+
+if(GFORTRANLIBS_FOUND)
+  message(STATUS "Looking for gfortran libraries -- found")
+  message(STATUS "gfortran version: ${GFORTRAN_VERSION_STRING}")
+else()
+  message(STATUS "Looking for gfortran libraries -- not found")
+endif()
+
+mark_as_advanced(LIBGFORTRAN_LIBRARIES LIBQUADMATH_LIBRARIES
+  LIBGOMP_LIBRARIES LIBGOMP_INCLUDE_DIR
+  GFORTRAN_LIBRARIES_DIR GFORTRAN_INCLUDE_DIR)
+# FindGFortranLIBS.cmake ends here

paddlespeech/audio/cmake/external/openblas.cmake

@@ -0,0 +1,58 @@
+include(FetchContent)
+
+set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src)
+set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix)
+
+# ######################################################################################################################
+# OPENBLAS  https://github.com/lattice/quda/blob/develop/CMakeLists.txt#L575
+# ######################################################################################################################
+enable_language(Fortran)
+
+include(FortranCInterface)
+
+# # Clang doesn't have a Fortran compiler in its suite (yet),
+# # so detect libraries for gfortran; we need equivalents to
+# # libgfortran and libquadmath, which are implicitly
+# # linked by flags in CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES
+# include(FindGFortranLibs REQUIRED)
+# # Add directory containing libgfortran and libquadmath to
+# # linker. Should also contain libgomp, if not using
+# # Intel OpenMP runtime
+# link_directories(${GFORTRAN_LIBRARIES_DIR})
+# # gfortan dir in the docker.
+# link_directories(/usr/local/gcc-8.2/lib64)
+# # if you are working with C and Fortran
+# FortranCInterface_VERIFY()
+
+# # if you are working with C++ and Fortran
+# FortranCInterface_VERIFY(CXX)
+
+
+#TODO: switch to CPM
+include(GNUInstallDirs)
+ExternalProject_Add(
+    OPENBLAS
+    GIT_REPOSITORY https://github.com/xianyi/OpenBLAS.git
+    GIT_TAG v0.3.10
+    GIT_SHALLOW YES
+    PREFIX ${OpenBLAS_PREFIX}
+    SOURCE_DIR  ${OpenBLAS_SOURCE_DIR}
+    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> 
+    CMAKE_GENERATOR "Unix Makefiles")
+
+
+# https://cmake.org/cmake/help/latest/module/ExternalProject.html?highlight=externalproject_get_property#external-project-definition
+ExternalProject_Get_Property(OPENBLAS INSTALL_DIR)
+set(OpenBLAS_INSTALL_PREFIX ${INSTALL_DIR})
+add_library(openblas STATIC IMPORTED)
+add_dependencies(openblas OPENBLAS)
+set_target_properties(openblas PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES Fortran)
+# ${CMAKE_INSTALL_LIBDIR}  lib
+set_target_properties(openblas PROPERTIES IMPORTED_LOCATION ${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libopenblas.a)
+
+
+# https://cmake.org/cmake/help/latest/command/install.html?highlight=cmake_install_libdir#installing-targets
+# ${CMAKE_INSTALL_LIBDIR}  lib
+# ${CMAKE_INSTALL_INCLUDEDIR}  include
+link_directories(${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
+include_directories(${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/openblas)

paddlespeech/audio/cmake/external/pybind.cmake

@@ -0,0 +1,9 @@
+include(FetchContent)
+FetchContent_Declare(
+  pybind
+  URL      https://github.com/pybind/pybind11/archive/refs/tags/v2.9.0.zip 
+  URL_HASH SHA256=1c6e0141f7092867c5bf388bc3acdb2689ed49f59c3977651394c6c87ae88232
+)
+FetchContent_MakeAvailable(pybind)
+include_directories(${pybind_SOURCE_DIR}/include)
+

paddlespeech/audio/csrc/CMakeLists.txt

@@ -32,3 +32,5 @@ if(BUILD_SOX)
     "${LINK_LIBRARIES}"
     )
 endif()
+
+add_subdirectory(pybind/kaldi_frontend)

paddlespeech/audio/csrc/pybind/kaldi_frontend/cmk.bak

@@ -0,0 +1,20 @@
+
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
+
+include_directories(
+${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+add_library(kaldi_feature
+  kaldi_feature.cc
+  kaldi_feature_wrapper.cc
+)
+target_link_libraries(kaldi_feature kaldi-fbank)
+
+pybind11_add_module(kaldi_featurepy kaldi_feature.cc kaldi_feature_wrapper.cc)
+target_link_libraries(kaldi_featurepy PRIVATE kaldi_feature)
+
+set(bin_name kaldi_feature_main)
+add_executable(${bin_name} ${CMAKE_CURRENT_SOURCE_DIR}/${bin_name}.cc)
+target_include_directories(${bin_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(${bin_name} PUBLIC kaldi_feature python3.7m)

paddlespeech/audio/csrc/pybind/kaldi_frontend/kaldi_feature_main.cc.bak

@@ -0,0 +1,27 @@
+#include <iostream>
+#include "kaldi_feature_wrapper.h"
+#include <vector>
+#include "wav.h"
+#include <pybind11/pybind11.h>
+#include <pybind11/numpy.h>
+
+using namespace std; 
+namespace py=pybind11;
+
+int main() {
+   kaldi::Vector<kaldi::BaseFloat> wav(raw_wav.size());
+   kaldi::Vector<kaldi::BaseFloat> result;
+   for (int idx = 0; idx < raw_wav.size(); ++idx) {
+     wav(idx) = raw_wav[idx];
+   }
+   kaldi::FbankOptions opts;
+   paddleaudio::Fbank fbank(opts);
+   fbank.ComputeFeature(wav, &result);
+   for (int idx = 0; idx < 10; ++idx) {
+      ////double val = *((double*)result.data(0, idx)); 
+      float val = result(idx);
+      cout << val << " ";
+   }
+   cout << endl;
+   fbank.Reset();
+}

paddlespeech/audio/csrc/pybind/kaldi_frontend/kaldi_feature_wrapper.cc.bak

@@ -0,0 +1,65 @@
+#include "kaldi_feature_wrapper.h"
+
+namespace paddleaudio {
+
+KaldiFeatureWrapper* KaldiFeatureWrapper::GetInstance() {
+    static KaldiFeatureWrapper instance;
+    return &instance;
+}
+
+bool KaldiFeatureWrapper::InitFbank(kaldi::FbankOptions opts) {
+  fbank_.reset(new Fbank(opts));
+  return true;
+}
+
+py::array_t<double> KaldiFeatureWrapper::ComputeFbank(const py::array_t<double> wav) {
+  /*
+    py::buffer_info info = wav.request();
+    kaldi::Vector<kaldi::BaseFloat> input_wav;
+    kaldi::Vector<kaldi::BaseFloat> feats;
+    memcpy(input_wav.Data(), (double*)info.ptr, wav.nbytes());
+    fbank_->ComputeFeature(input_wav, &feats);
+    auto result = py::array_t<double>(feats.Dim());
+    py::buffer_info xs = result.request();
+    memcpy((double*)xs.ptr, feats.Data(), feats.Dim()*sizeof(kaldi::BaseFloat));
+    return result.reshape({ feats.Dim() / Dim(), Dim()});
+    */
+  py::buffer_info info = wav.request();
+  auto result = py::array_t<double>(info.size);
+  py::buffer_info result_info = result.request();
+
+  double* req_ptr = (double*)info.ptr;
+  double* res_ptr = (double*)result_info.ptr;
+  std::cout << info.itemsize << std::endl;
+  std::cout << info.format<< std::endl;
+  std::cout << info.ndim<< std::endl;
+  std::cout << "shape" << std::endl;
+  for (int i = 0; i < info.shape.size(); ++i) {
+    std::cout << info.shape[i]<< std::endl;
+  }
+
+  std::cout << "strides" << std::endl;
+  for (int i = 0; i < info.strides.size(); ++i) {
+    std::cout << info.strides[i]<< std::endl;
+  }
+
+  std::cout << "result shape" << std::endl;
+  for (int i = 0; i < result_info.shape.size(); ++i) {
+    std::cout << result_info.shape[i]<< std::endl;
+  }
+
+  int max_idx = 1;
+  for (auto dim : info.shape) {
+    max_idx *= dim;
+  }
+
+  for (int i = 0; i < max_idx; i++) {
+    *res_ptr = (*req_ptr) * (*req_ptr);
+    res_ptr++;
+    req_ptr++;
+  }
+    return  result;
+}
+
+    
+} // namespace paddleaudio

paddlespeech/audio/third_party/CMakeLists.txt

@@ -6,3 +6,9 @@ if (BUILD_SOX)
   target_include_directories(libsox INTERFACE ${SOX_INCLUDE_DIR})
   target_link_libraries(libsox INTERFACE ${SOX_LIBRARIES})
 endif()
+
+include_directories(
+${CMAKE_CURRENT_SOURCE_DIR}
+${CMAKE_CURRENT_SOURCE_DIR}/kaldi
+)
+add_subdirectory(kaldi)

paddlespeech/audio/third_party/kaldi/CMakeLists.txt

@@ -0,0 +1,60 @@
+project(kaldi)
+
+
+add_library(kaldi-base  
+  base/io-funcs.cc
+  base/kaldi-error.cc
+  base/kaldi-math.cc
+  base/kaldi-utils.cc
+  base/timer.cc
+)
+
+add_library(kaldi-util
+  util/kaldi-holder.cc
+  util/kaldi-io.cc
+  util/kaldi-semaphore.cc
+  util/kaldi-table.cc
+  util/kaldi-thread.cc
+  util/parse-options.cc
+  util/simple-io-funcs.cc
+  util/simple-options.cc
+  util/text-utils.cc
+)
+target_link_libraries(kaldi-util PUBLIC kaldi-base kaldi-matrix)
+
+add_library(kaldi-mfcc
+  feat/feature-mfcc.cc
+)
+target_link_libraries(kaldi-mfcc PUBLIC kaldi-feat-common)
+
+add_library(kaldi-fbank
+  feat/feature-fbank.cc
+)
+target_link_libraries(kaldi-fbank PUBLIC kaldi-feat-common)
+
+add_library(kaldi-feat-common
+  feat/wave-reader.cc
+  feat/signal.cc
+  feat/feature-functions.cc
+  feat/feature-window.cc
+  feat/resample.cc
+  feat/mel-computations.cc
+  feat/cmvn.cc
+)
+target_link_libraries(kaldi-feat-common PUBLIC kaldi-base kaldi-matrix kaldi-util)
+
+add_library(kaldi-matrix
+  matrix/compressed-matrix.cc
+  matrix/kaldi-matrix.cc
+  matrix/kaldi-vector.cc
+  matrix/matrix-functions.cc
+  matrix/optimization.cc
+  matrix/packed-matrix.cc
+  matrix/qr.cc
+  matrix/sparse-matrix.cc
+  matrix/sp-matrix.cc
+  matrix/srfft.cc
+  matrix/tp-matrix.cc
+)
+target_link_libraries(kaldi-matrix gfortran kaldi-base libopenblas.a)
+

paddlespeech/audio/third_party/kaldi/base

@@ -0,0 +1 @@
+../../../../speechx/speechx/kaldi/base

paddlespeech/audio/third_party/kaldi/feat

@@ -0,0 +1 @@
+../../../../speechx/speechx/kaldi/feat

paddlespeech/audio/third_party/kaldi/matrix

@@ -0,0 +1 @@
+../../../../speechx/speechx/kaldi/matrix

paddlespeech/audio/third_party/kaldi/util

@@ -0,0 +1 @@
+../../../../speechx/speechx/kaldi/util