diff --git a/runtime/engine/CMakeLists.txt b/runtime/engine/CMakeLists.txt index 1032222c3..d64df6489 100644 --- a/runtime/engine/CMakeLists.txt +++ b/runtime/engine/CMakeLists.txt @@ -12,7 +12,7 @@ if(WITH_ASR) endif() if(WITH_CLS) - add_subdirectory(cls) + add_subdirectory(audio_classification) endif() if(WITH_VAD) diff --git a/runtime/engine/cls/CMakeLists.txt b/runtime/engine/audio_classification/CMakeLists.txt similarity index 100% rename from runtime/engine/cls/CMakeLists.txt rename to runtime/engine/audio_classification/CMakeLists.txt diff --git a/runtime/engine/cls/nnet/CMakeLists.txt b/runtime/engine/audio_classification/nnet/CMakeLists.txt similarity index 100% rename from runtime/engine/cls/nnet/CMakeLists.txt rename to runtime/engine/audio_classification/nnet/CMakeLists.txt diff --git a/runtime/engine/cls/nnet/panns_interface.cc b/runtime/engine/audio_classification/nnet/panns_interface.cc similarity index 95% rename from runtime/engine/cls/nnet/panns_interface.cc rename to runtime/engine/audio_classification/nnet/panns_interface.cc index cfff3f92e..d8b6a8b61 100644 --- a/runtime/engine/cls/nnet/panns_interface.cc +++ b/runtime/engine/audio_classification/nnet/panns_interface.cc @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cls/nnet/panns_interface.h" +#include "audio_classification/nnet/panns_interface.h" -#include "cls/nnet/panns_nnet.h" +#include "audio_classification/nnet/panns_nnet.h" #include "common/base/config.h" namespace ppspeech { diff --git a/runtime/engine/cls/nnet/panns_interface.h b/runtime/engine/audio_classification/nnet/panns_interface.h similarity index 100% rename from runtime/engine/cls/nnet/panns_interface.h rename to runtime/engine/audio_classification/nnet/panns_interface.h diff --git a/runtime/engine/cls/nnet/panns_nnet.cc b/runtime/engine/audio_classification/nnet/panns_nnet.cc similarity index 98% rename from runtime/engine/cls/nnet/panns_nnet.cc rename to runtime/engine/audio_classification/nnet/panns_nnet.cc index 6ab7f7bb6..37ba74f9d 100644 --- a/runtime/engine/cls/nnet/panns_nnet.cc +++ b/runtime/engine/audio_classification/nnet/panns_nnet.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "cls/nnet/panns_nnet.h" +#include "audio_classification/nnet/panns_nnet.h" #ifdef WITH_PROFILING #include "kaldi/base/timer.h" #endif diff --git a/runtime/engine/cls/nnet/panns_nnet.h b/runtime/engine/audio_classification/nnet/panns_nnet.h similarity index 100% rename from runtime/engine/cls/nnet/panns_nnet.h rename to runtime/engine/audio_classification/nnet/panns_nnet.h diff --git a/runtime/engine/cls/nnet/panns_nnet_main.cc b/runtime/engine/audio_classification/nnet/panns_nnet_main.cc similarity index 97% rename from runtime/engine/cls/nnet/panns_nnet_main.cc rename to runtime/engine/audio_classification/nnet/panns_nnet_main.cc index 20f3312c9..b47753f0b 100644 --- a/runtime/engine/cls/nnet/panns_nnet_main.cc +++ b/runtime/engine/audio_classification/nnet/panns_nnet_main.cc @@ -17,7 +17,7 @@ #include "gflags/gflags.h" #include "glog/logging.h" -#include "cls/nnet/panns_interface.h" +#include "audio_classification/nnet/panns_interface.h" DEFINE_string(conf_path, "", "config path"); DEFINE_string(scp_path, "", "wav scp path"); diff --git a/runtime/examples/audio_classification/README.md b/runtime/examples/audio_classification/README.md new file mode 100644 index 000000000..6d7a37423 --- /dev/null +++ b/runtime/examples/audio_classification/README.md @@ -0,0 +1,100 @@ +# audio classification + +This directory provieds audio classification on CPU + +## conf +config is the input of engine + + [CONF] + wav_normal=true + wav_normal_type=linear + wav_norm_mul_factor=1.0 + model_path=./inference.onnx + param_path= + dict_path=./label_list + num_cpu_thread=1 + samp_freq=32000 + frame_length_ms=32 + frame_shift_ms=10 + num_bins=64 + low_freq=50 + high_freq=14000 + dither=0.0 +## label_list +model output label + + Dog + Rooster + Pig + Cow + Frog + Cat + Hen + Insects (flying) + Sheep + Crow + Rain + Sea waves + Crackling fire + ..... +## scp && test.wav +scp is the input of engine and each line in scp is wav +## execute +../../build/Linux/x86_64/engine/audio_classification/nnet/panns_nnet_main --conf_path=./conf --scp_path=./scp --topk=1 +usage: panns_nnet_main conf scp topk +output such as: + + wav_normal = true + wav_normal_type = linear + wav_norm_mul_factor = 1.0 + model_path = ./inference.onnx + param_path = + dict_path = ./label_list + num_cpu_thread = 1 + samp_freq = 32000 + frame_length_ms = 32 + frame_shift_ms = 10 + num_bins = 64 + low_freq = 50 + high_freq = 14000 + dither = 0.0 + [INFO] fastdeploy/runtime/runtime.cc(293)::CreateOrtBackend Runtime initialized with Backend::ORT in Device::CPU. + --- Init FastDeploy Runitme Done! + --- Model: ./inference.onnx + test.wav{"Clock alarm":"16.5309"} +## android demo +### install +#### copy lib & interface +cd ../../ +sh build_android.sh +cp build/Android/arm64-v8a-api-21/cls-android-out/*.so examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/ +cp build/Android/arm64-v8a-api-21/cls-android-out/panns_interface.h examples/audio_classification/android_demo/app/src/main/cpp/ +includes/ + +#### set path +push resource into android phone + +1. change resource path in conf to gloabal path, such as: + + [CONF] + wav_normal=true + wav_normal_type=linear + wav_norm_mul_factor=1.0 + model_path=/data/local/tmp/inference.onnx + param_path= + dict_path=/data/local/tmp/label_list + num_cpu_thread=1 + samp_freq=32000 + frame_length_ms=32 + frame_shift_ms=10 + num_bins=64 + low_freq=50 + high_freq=14000 + dither=0.0 +2. adb push conf label_list scp test.wav /data/local/tmp/ +3. set reource path in android demo(android_demo/app/src/main/cpp/native-lib.cpp) to actual path, such as: + +std::string conf_path = "/data/local/tmp/conf"; +std::string wav_path = "/data/local/tmp/test.wav"; + +4. excecute android_demo in android studio diff --git a/runtime/examples/audio_classification/android_demo/.gitignore b/runtime/examples/audio_classification/android_demo/.gitignore new file mode 100644 index 000000000..aa724b770 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/runtime/examples/audio_classification/android_demo/.idea/.gitignore b/runtime/examples/audio_classification/android_demo/.idea/.gitignore new file mode 100644 index 000000000..26d33521a --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/runtime/examples/audio_classification/android_demo/.idea/codeStyles/Project.xml b/runtime/examples/audio_classification/android_demo/.idea/codeStyles/Project.xml new file mode 100644 index 000000000..7643783a8 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/.idea/codeStyles/Project.xml @@ -0,0 +1,123 @@ + + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/.idea/codeStyles/codeStyleConfig.xml b/runtime/examples/audio_classification/android_demo/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 000000000..79ee123c2 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/.idea/compiler.xml b/runtime/examples/audio_classification/android_demo/.idea/compiler.xml new file mode 100644 index 000000000..fb7f4a8a4 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/.idea/gradle.xml b/runtime/examples/audio_classification/android_demo/.idea/gradle.xml new file mode 100644 index 000000000..a2d7c2133 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/.idea/gradle.xml @@ -0,0 +1,19 @@ + + + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/.idea/misc.xml b/runtime/examples/audio_classification/android_demo/.idea/misc.xml new file mode 100644 index 000000000..bdd92780c --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/.idea/misc.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/.gitignore b/runtime/examples/audio_classification/android_demo/app/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/build.gradle b/runtime/examples/audio_classification/android_demo/app/build.gradle new file mode 100644 index 000000000..1c9222df6 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/build.gradle @@ -0,0 +1,65 @@ +plugins { + id 'com.android.application' + id 'org.jetbrains.kotlin.android' +} + +android { + namespace 'com.baidu.paddlespeech.cls' + compileSdk 32 + + defaultConfig { + applicationId "com.baidu.paddlespeech.cls" + minSdk 28 + targetSdk 32 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + externalNativeBuild { + cmake { + cppFlags '' + } + } + // 设置ndk编译的cpu架构 + ndk { + abiFilters 'arm64-v8a' + } + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = '1.8' + } + externalNativeBuild { + cmake { + path file('src/main/cpp/CMakeLists.txt') + version '3.22.1' + } + } + //我们将外部so库放在jniLibs文件夹下，因此要将它设置为jniLibs使工程在打包的时候能将它包含进去，否则app运行时会报无法找到so库的错误。 + sourceSets { + main { + jniLibs.srcDirs = ['src/main/cpp/jniLibs'] + resources { srcDirs = ['src/main/cpp/resources'] } + } + } +} + +dependencies { + + //noinspection GradleCompatible + implementation 'com.android.support:appcompat-v7:28.0.0' + implementation 'com.android.support.constraint:constraint-layout:2.0.4' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'com.android.support.test:runner:1.0.2' + androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' +} \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/proguard-rules.pro b/runtime/examples/audio_classification/android_demo/app/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/androidTest/java/com/example/cls/ExampleInstrumentedTest.kt b/runtime/examples/audio_classification/android_demo/app/src/androidTest/java/com/example/cls/ExampleInstrumentedTest.kt new file mode 100644 index 000000000..828089d51 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/androidTest/java/com/example/cls/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.example.cls + +import android.support.test.InstrumentationRegistry +import android.support.test.runner.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.example.cls", appContext.packageName) + } +} \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/AndroidManifest.xml b/runtime/examples/audio_classification/android_demo/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..5f875832f --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/AndroidManifest.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/cpp/CMakeLists.txt b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..27b5e5e4f --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,59 @@ + +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html + +# Sets the minimum version of CMake required to build the native library. + +cmake_minimum_required(VERSION 3.18.1) + +# Declares and names the project. + +project("cls") + +include_directories(src/main/cpp/) +file(GLOB CPP_FILES "src/main/cpp/*.cpp") + +# 添加so库存放位置 +set(distribution_DIR ${CMAKE_SOURCE_DIR}/jniLibs) + +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. + +add_library( # Sets the name of the library. + native-lib + + # Sets the library as a shared library. + SHARED + + # Provides a relative path to your source file(s). + ${CMAKE_CURRENT_SOURCE_DIR}/native-lib.cpp ) + +# Searches for a specified prebuilt library and stores the path as a +# variable. Because CMake includes system libraries in the search path by +# default, you only need to specify the name of the public NDK library +# you want to add. CMake verifies that the library exists before +# completing its build. + +find_library( # Sets the name of the path variable. + log-lib + + # Specifies the name of the NDK library that + # you want CMake to locate. + log ) + +# Specifies libraries CMake should link to your target library. You +# can link multiple libraries, such as libraries you define in this +# build script, prebuilt third-party libraries, or system libraries. + +target_link_libraries( # Specifies the target library. + native-lib + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libc++_shared.so + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libcls.so + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libfastdeploy.so + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libonnxruntime.so + # Links the target library to the log library + # included in the NDK. + ${log-lib} + ) diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/cpp/includes/panns_interface.h b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/includes/panns_interface.h new file mode 100644 index 000000000..0d1ce95f5 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/includes/panns_interface.h @@ -0,0 +1,27 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +namespace ppspeech { + +void* ClsCreateInstance(const char* conf_path); +int ClsDestroyInstance(void* instance); +int ClsFeedForward(void* instance, + const char* wav_path, + int topk, + char* result, + int result_max_len); +int ClsReset(void* instance); +} // namespace ppspeech \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libc++_shared.so b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libc++_shared.so new file mode 100644 index 000000000..61818d3ee Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libc++_shared.so differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libcls.so b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libcls.so new file mode 100755 index 000000000..312e4b450 Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libcls.so differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libfastdeploy.so b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libfastdeploy.so new file mode 100644 index 000000000..2f9b3e624 Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libfastdeploy.so differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libonnxruntime.so b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libonnxruntime.so new file mode 100644 index 000000000..19776e898 Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/jniLibs/arm64-v8a/libonnxruntime.so differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/cpp/native-lib.cpp b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/native-lib.cpp new file mode 100644 index 000000000..b04abe6e1 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/cpp/native-lib.cpp @@ -0,0 +1,67 @@ +// Write C++ code here. +// +// Do not forget to dynamically load the C++ library into your application. +// +// For instance, +// +// In MainActivity.java: +// static { +// System.loadLibrary("mysotest"); +// } +// +// Or, in MainActivity.kt: +// companion object { +// init { +// System.loadLibrary("mysotest") +// } +// } + +#include +#include +#include +#include "includes/panns_interface.h" + +//如果你不想用引入头文件的方法，可以把导入头文件的include语句注释掉，然后将下面这句取消注释。 +//string getStringFromSoLibrary(); + +void* cls_instance = nullptr; + +extern "C" +JNIEXPORT jboolean JNICALL Java_com_baidu_paddlespeech_cls_MainActivity_nClsCreateInstance(JNIEnv *env, jobject instance) +{ + if (cls_instance != nullptr) { + ppspeech::ClsDestroyInstance(cls_instance); + cls_instance = nullptr; + } + std::string conf_path = "/data/local/tmp/masimeng/cls/conf"; + cls_instance = ppspeech::ClsCreateInstance(conf_path.c_str()); + return true; +} + +extern "C" +JNIEXPORT jboolean JNICALL Java_com_baidu_paddlespeech_cls_MainActivity_nClsDestroyInstance(JNIEnv *env, jobject instance){ + if (cls_instance != nullptr) { + ppspeech::ClsDestroyInstance(cls_instance); + cls_instance = nullptr; + } + return true; +} + +extern "C" +JNIEXPORT jstring JNICALL Java_com_baidu_paddlespeech_cls_MainActivity_nClsFeedForward(JNIEnv *env, jobject instance){ + if (cls_instance != nullptr) { + char result[1024] = {0}; + std::string wav_path = "/data/local/tmp/masimeng/cls/test.wav"; + int ret = ppspeech::ClsFeedForward(cls_instance, wav_path.c_str(), 1, result, 1024); + return env->NewStringUTF(result); + } + return env->NewStringUTF(NULL); +} + +extern "C" +JNIEXPORT jboolean JNICALL Java_com_baidu_paddlespeech_cls_MainActivity_nClsReset(JNIEnv *env, jobject instance){ + if (cls_instance != nullptr) { + ppspeech::ClsReset(cls_instance); + } + return true; +} \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/java/com/example/cls/MainActivity.kt b/runtime/examples/audio_classification/android_demo/app/src/main/java/com/example/cls/MainActivity.kt new file mode 100644 index 000000000..ba6cacb2f --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/java/com/example/cls/MainActivity.kt @@ -0,0 +1,31 @@ +package com.baidu.paddlespeech.cls + +import android.support.v7.app.AppCompatActivity +import android.os.Bundle +import android.widget.TextView + +class MainActivity : AppCompatActivity() { + private lateinit var tvContent: TextView + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + tvContent = findViewById(R.id.tv_content) + + nClsCreateInstance() + tvContent.text = nClsFeedForward() + nClsReset() + nClsDestroyInstance() + } + + external fun nClsCreateInstance(): Boolean + external fun nClsDestroyInstance(): Boolean + external fun nClsFeedForward(): String + external fun nClsReset(): Boolean + + companion object{ + init { + System.loadLibrary("native-lib") + } + } +} \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..2b068d114 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/drawable/ic_launcher_background.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..07d5da9cb --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/layout/activity_main.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..2e9672d86 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 000000000..eca70cfe5 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 000000000..eca70cfe5 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-hdpi/ic_launcher.webp new file mode 100644 index 000000000..c209e78ec Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-hdpi/ic_launcher.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp new file mode 100644 index 000000000..b2dfe3d1b Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-mdpi/ic_launcher.webp new file mode 100644 index 000000000..4f0f1d64e Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-mdpi/ic_launcher.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp new file mode 100644 index 000000000..62b611da0 Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xhdpi/ic_launcher.webp new file mode 100644 index 000000000..948a3070f Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xhdpi/ic_launcher.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..1b9a6956b Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp new file mode 100644 index 000000000..28d4b77f9 Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..9287f5083 Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp new file mode 100644 index 000000000..aa7d6427e Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..9126ae37c Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp differ diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/values-night/themes.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/values-night/themes.xml new file mode 100644 index 000000000..afc715153 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/values-night/themes.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/values/colors.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/values/colors.xml new file mode 100644 index 000000000..f8c6127d3 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/values/colors.xml @@ -0,0 +1,10 @@ + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/values/strings.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/values/strings.xml new file mode 100644 index 000000000..2c07784ba --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + cls + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/values/themes.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/values/themes.xml new file mode 100644 index 000000000..ec5a0f15b --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/values/themes.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/xml/backup_rules.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/xml/backup_rules.xml new file mode 100644 index 000000000..fa0f996d2 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/xml/backup_rules.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/app/src/main/res/xml/data_extraction_rules.xml b/runtime/examples/audio_classification/android_demo/app/src/main/res/xml/data_extraction_rules.xml new file mode 100644 index 000000000..9ee9997b0 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/app/src/main/res/xml/data_extraction_rules.xml @@ -0,0 +1,19 @@ + + + + + + + \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/build.gradle b/runtime/examples/audio_classification/android_demo/build.gradle new file mode 100644 index 000000000..253697423 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/build.gradle @@ -0,0 +1,6 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id 'com.android.application' version '7.3.1' apply false + id 'com.android.library' version '7.3.1' apply false + id 'org.jetbrains.kotlin.android' version '1.7.20' apply false +} \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/gradle.properties b/runtime/examples/audio_classification/android_demo/gradle.properties new file mode 100644 index 000000000..1b6cc8957 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/gradle.properties @@ -0,0 +1,19 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/runtime/examples/audio_classification/android_demo/gradle/wrapper/gradle-wrapper.jar b/runtime/examples/audio_classification/android_demo/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..e708b1c02 Binary files /dev/null and b/runtime/examples/audio_classification/android_demo/gradle/wrapper/gradle-wrapper.jar differ diff --git a/runtime/examples/audio_classification/android_demo/gradle/wrapper/gradle-wrapper.properties b/runtime/examples/audio_classification/android_demo/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..3ed474aab --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Jan 16 16:37:04 CST 2023 +distributionBase=GRADLE_USER_HOME +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip +distributionPath=wrapper/dists +zipStorePath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME diff --git a/runtime/examples/audio_classification/android_demo/gradlew b/runtime/examples/audio_classification/android_demo/gradlew new file mode 100755 index 000000000..4f906e0c8 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/runtime/examples/audio_classification/android_demo/gradlew.bat b/runtime/examples/audio_classification/android_demo/gradlew.bat new file mode 100644 index 000000000..ac1b06f93 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/runtime/examples/audio_classification/android_demo/settings.gradle b/runtime/examples/audio_classification/android_demo/settings.gradle new file mode 100644 index 000000000..05c3cf715 --- /dev/null +++ b/runtime/examples/audio_classification/android_demo/settings.gradle @@ -0,0 +1,16 @@ +pluginManagement { + repositories { + gradlePluginPortal() + google() + mavenCentral() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} +rootProject.name = "cls" +include ':app' diff --git a/runtime/examples/audio_classification/conf b/runtime/examples/audio_classification/conf new file mode 100644 index 000000000..eb6c9864d --- /dev/null +++ b/runtime/examples/audio_classification/conf @@ -0,0 +1,15 @@ +[CONF] +wav_normal=true +wav_normal_type=linear +wav_norm_mul_factor=1.0 +model_path=./inference.onnx +param_path= +dict_path=./label_list +num_cpu_thread=1 +samp_freq=32000 +frame_length_ms=32 +frame_shift_ms=10 +num_bins=64 +low_freq=50 +high_freq=14000 +dither=0.0 diff --git a/runtime/examples/audio_classification/label_list b/runtime/examples/audio_classification/label_list new file mode 100644 index 000000000..e27aa8b0f --- /dev/null +++ b/runtime/examples/audio_classification/label_list @@ -0,0 +1,50 @@ +Dog +Rooster +Pig +Cow +Frog +Cat +Hen +Insects (flying) +Sheep +Crow +Rain +Sea waves +Crackling fire +Crickets +Chirping birds +Water drops +Wind +Pouring water +Toilet flush +Thunderstorm +Crying baby +Sneezing +Clapping +Breathing +Coughing +Footsteps +Laughing +Brushing teeth +Snoring +Drinking, sipping +Door knock +Mouse click +Keyboard typing +Door, wood creaks +Can opening +Washing machine +Vacuum cleaner +Clock alarm +Clock tick +Glass breaking +Helicopter +Chainsaw +Siren +Car horn +Engine +Train +Church bells +Airplane +Fireworks +Hand saw diff --git a/runtime/examples/audio_classification/scp b/runtime/examples/audio_classification/scp new file mode 100644 index 000000000..607adc132 --- /dev/null +++ b/runtime/examples/audio_classification/scp @@ -0,0 +1 @@ +test.wav diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/executionHistory/executionHistory.bin b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/executionHistory/executionHistory.bin new file mode 100644 index 000000000..d19142a2d Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/executionHistory/executionHistory.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/executionHistory/executionHistory.lock b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/executionHistory/executionHistory.lock new file mode 100644 index 000000000..4ed55bad5 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/executionHistory/executionHistory.lock differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileChanges/last-build.bin b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileChanges/last-build.bin new file mode 100644 index 000000000..f76dd238a Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileChanges/last-build.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileContent/fileContent.lock b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileContent/fileContent.lock new file mode 100644 index 000000000..e2d7623fa Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileContent/fileContent.lock differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/fileHashes.bin b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/fileHashes.bin new file mode 100644 index 000000000..8bbefe601 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/fileHashes.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/fileHashes.lock b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/fileHashes.lock new file mode 100644 index 000000000..7e2279f67 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/fileHashes.lock differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/resourceHashesCache.bin b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/resourceHashesCache.bin new file mode 100644 index 000000000..b41955f72 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/fileHashes/resourceHashesCache.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/gc.properties b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/gc.properties new file mode 100644 index 000000000..e69de29bb diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/classAnalysis.bin b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/classAnalysis.bin new file mode 100644 index 000000000..d0b09f519 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/classAnalysis.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/jarAnalysis.bin b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/jarAnalysis.bin new file mode 100644 index 000000000..daaa6ebe5 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/jarAnalysis.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/javaCompile.lock b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/javaCompile.lock new file mode 100644 index 000000000..d687a9dc4 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/javaCompile.lock differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/taskHistory.bin b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/taskHistory.bin new file mode 100644 index 000000000..7e687d02f Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/6.1.1/javaCompile/taskHistory.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/buildOutputCleanup.lock b/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/buildOutputCleanup.lock new file mode 100644 index 000000000..eb8dcff5e Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/buildOutputCleanup.lock differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/cache.properties b/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/cache.properties new file mode 100644 index 000000000..92f780437 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/cache.properties @@ -0,0 +1,2 @@ +#Wed Mar 29 16:03:45 CST 2023 +gradle.version=6.1.1 diff --git a/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/outputFiles.bin b/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/outputFiles.bin new file mode 100644 index 000000000..f459a7de9 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/buildOutputCleanup/outputFiles.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/checksums/checksums.lock b/runtime/examples/vad/vad-android-demo/.gradle/checksums/checksums.lock new file mode 100644 index 000000000..f9d50e925 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/checksums/checksums.lock differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/checksums/md5-checksums.bin b/runtime/examples/vad/vad-android-demo/.gradle/checksums/md5-checksums.bin new file mode 100644 index 000000000..65f5ed7cf Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/checksums/md5-checksums.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/checksums/sha1-checksums.bin b/runtime/examples/vad/vad-android-demo/.gradle/checksums/sha1-checksums.bin new file mode 100644 index 000000000..eb663c0ca Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/.gradle/checksums/sha1-checksums.bin differ diff --git a/runtime/examples/vad/vad-android-demo/.gradle/vcs-1/gc.properties b/runtime/examples/vad/vad-android-demo/.gradle/vcs-1/gc.properties new file mode 100644 index 000000000..e69de29bb diff --git a/runtime/examples/vad/vad-android-demo/.idea/.gitignore b/runtime/examples/vad/vad-android-demo/.idea/.gitignore new file mode 100644 index 000000000..26d33521a --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/runtime/examples/vad/vad-android-demo/.idea/.name b/runtime/examples/vad/vad-android-demo/.idea/.name new file mode 100644 index 000000000..f8e3feb46 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.idea/.name @@ -0,0 +1 @@ +VAD \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/.idea/compiler.xml b/runtime/examples/vad/vad-android-demo/.idea/compiler.xml new file mode 100644 index 000000000..fb7f4a8a4 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/.idea/gradle.xml b/runtime/examples/vad/vad-android-demo/.idea/gradle.xml new file mode 100644 index 000000000..deb53b326 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.idea/gradle.xml @@ -0,0 +1,20 @@ + + + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/.idea/jarRepositories.xml b/runtime/examples/vad/vad-android-demo/.idea/jarRepositories.xml new file mode 100644 index 000000000..eb2873e7e --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.idea/jarRepositories.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/.idea/misc.xml b/runtime/examples/vad/vad-android-demo/.idea/misc.xml new file mode 100644 index 000000000..bdd92780c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.idea/misc.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/.idea/vcs.xml b/runtime/examples/vad/vad-android-demo/.idea/vcs.xml new file mode 100644 index 000000000..35eb1ddfb --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/LICENSE.md b/runtime/examples/vad/vad-android-demo/LICENSE.md new file mode 100644 index 000000000..b322dd3a6 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/LICENSE.md @@ -0,0 +1,16 @@ +Copyright 2019 Georgiy Konovalov + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/README b/runtime/examples/vad/vad-android-demo/README new file mode 100644 index 000000000..d8e120b68 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/README @@ -0,0 +1,13 @@ +''' +INSTALL +''' +执行目录为speechx根目录 +sh build_android.sh +cp build/Android/arm64-v8a-api-21/vad-android-out/*.so examples/vad/vad-android-demo/vad/src/main/cpp/jniLibs/arm64-v8a +cp build/Android/arm64-v8a-api-21/vad-android-out/vad_interface.h examples/vad/vad-android-demo/vad/src/main/cpp/includes/ + +''' +USEAGE +''' +1. 根据vad资源实际放在手机中的位置，更改vad-android-demo/app/src/main/cpp/vad/src/main/cpp/native-lib.cpp中的conf_path、fp_pcm_name、fp_log_name +2. 在AndroidStudio中编译执行 diff --git a/runtime/examples/vad/vad-android-demo/README.md b/runtime/examples/vad/vad-android-demo/README.md new file mode 100644 index 000000000..252a3ee7c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/README.md @@ -0,0 +1,147 @@ +This VAD library can process audio in real-time utilizing +[Gaussian Mixture Model](http://en.wikipedia.org/wiki/Mixture_model#Gaussian_mixture_model) (GMM) +which helps identify presence of human speech in an audio sample that contains a mixture of speech +and noise. VAD work offline and all processing done on device. + +Library based on +[WebRTC VAD](https://chromium.googlesource.com/external/webrtc/+/branch-heads/43/webrtc/common_audio/vad/) +from Google which is reportedly one of the best available: it's fast, modern and free. +This algorithm has found wide adoption and has recently become one of +the gold-standards for delay-sensitive scenarios like web-based interaction. + +If you are looking for a higher accuracy and faster processing time I recommend to use Deep Neural +Networks(DNN). Please see for reference the following paper with +[DNN vs GMM](https://www.microsoft.com/en-us/research/uploads/prod/2018/02/KoPhiliposeTashevZarar_ICASSP_2018.pdf) +comparison. + +

+drawing +

+ +## Parameters +VAD library only accepts 16-bit mono PCM audio stream and can work with next Sample Rates, Frame Sizes and Classifiers. + + + + + +
+  + +| Valid Sample Rate | Valid Frame Size | +|:-------------------|:------------------| +| 8000Hz | 80, 160, 240 | +| 16000Hz | 160, 320, 480 | +| 32000Hz | 320, 640, 960 | +| 48000Hz | 480, 960, 1440 | + +  + +| Valid Classifiers | +|:------------------| +| NORMAL | +| LOW_BITRATE | +| AGGRESSIVE | +| VERY_AGGRESSIVE | +
+ + +**Silence duration (ms)** - this parameter used in Continuous Speech detector, +the value of this parameter will define the necessary and sufficient +duration of negative results to recognize it as silence. + +**Voice duration (ms)** - this parameter used in Continuous Speech detector, +the value of this parameter will define the necessary and sufficient +duration of positive results to recognize result as speech. + +Recommended parameters: +* Sample Rate - **16KHz**, +* Frame Size - **160**, +* Mode - **VERY_AGGRESSIVE**, +* Silence Duration - **500ms**, +* Voice Duration - **500ms**; + +## Usage +VAD supports 2 different ways of detecting speech: +1. Continuous Speech listener was designed to detect long utterances +without returning false positive results when user makes pauses between +sentences. +```java + Vad vad = new Vad(VadConfig.newBuilder() + .setSampleRate(VadConfig.SampleRate.SAMPLE_RATE_16K) + .setFrameSize(VadConfig.FrameSize.FRAME_SIZE_160) + .setMode(VadConfig.Mode.VERY_AGGRESSIVE) + .setSilenceDurationMillis(500) + .setVoiceDurationMillis(500) + .build()); + + vad.start(); + + vad.addContinuousSpeechListener(short[] audioFrame, new VadListener() { + @Override + public void onSpeechDetected() { + //speech detected! + } + + @Override + public void onNoiseDetected() { + //noise detected! + } + }); + + vad.stop(); +``` + +2. Speech detector was designed to detect speech/noise in small audio +frames and return result for every frame. This method will not work for +long utterances. +```java + Vad vad = new Vad(VadConfig.newBuilder() + .setSampleRate(VadConfig.SampleRate.SAMPLE_RATE_16K) + .setFrameSize(VadConfig.FrameSize.FRAME_SIZE_160) + .setMode(VadConfig.Mode.VERY_AGGRESSIVE) + .build()); + + vad.start(); + + boolean isSpeech = vad.isSpeech(short[] audioFrame); + + vad.stop(); +``` +## Requirements +Android VAD supports Android 4.1 (Jelly Bean) and later. + +## Development + +To open the project in Android Studio: + +1. Go to *File* menu or the *Welcome Screen* +2. Click on *Open...* +3. Navigate to VAD's root directory. +4. Select `setting.gradle` + +## Download +[![](https://jitpack.io/v/gkonovalov/android-vad.svg)](https://jitpack.io/#gkonovalov/android-vad) + + +Gradle is the only supported build configuration, so just add the dependency to your project `build.gradle` file: +1. Add it in your root build.gradle at the end of repositories: +```groovy +allprojects { + repositories { + maven { url 'https://jitpack.io' } + } +} +``` + +2. Add the dependency +```groovy +dependencies { + implementation 'com.github.gkonovalov:android-vad:1.0.1' +} +``` + +You also can download precompiled AAR library and APK files from GitHub's [releases page](https://github.com/gkonovalov/android-vad/releases). + +------------ +Georgiy Konovalov 2021 (c) [MIT License](https://opensource.org/licenses/MIT) \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/build.gradle b/runtime/examples/vad/vad-android-demo/build.gradle new file mode 100644 index 000000000..ee7e60966 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/build.gradle @@ -0,0 +1,28 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. + +buildscript { + repositories { + google() + jcenter() + maven { url 'https://jitpack.io' } + } + + dependencies { + classpath 'com.android.tools.build:gradle:4.0.2' + classpath 'com.github.dcendents:android-maven-gradle-plugin:2.1' + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + google() + jcenter() + maven { url 'https://jitpack.io' } + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} diff --git a/runtime/examples/vad/vad-android-demo/demo.gif b/runtime/examples/vad/vad-android-demo/demo.gif new file mode 100644 index 000000000..5ceed51b2 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/demo.gif differ diff --git a/runtime/examples/vad/vad-android-demo/example/.gitignore b/runtime/examples/vad/vad-android-demo/example/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/.gitignore @@ -0,0 +1 @@ +/build diff --git a/runtime/examples/vad/vad-android-demo/example/build.gradle b/runtime/examples/vad/vad-android-demo/example/build.gradle new file mode 100644 index 000000000..4b937ff4e --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/build.gradle @@ -0,0 +1,39 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 29 + buildToolsVersion "29.0.2" + defaultConfig { + applicationId "com.konovalov.vad.example" + minSdkVersion 16 + targetSdkVersion 29 + versionCode 2 + versionName "1.0.1" + + setProperty("archivesBaseName", "Android-VAD-v" + versionName) + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation 'androidx.appcompat:appcompat:1.3.0' + implementation 'androidx.constraintlayout:constraintlayout:2.0.4' + + implementation 'com.airbnb.android:lottie:3.4.0' + implementation 'org.permissionsdispatcher:permissionsdispatcher:4.8.0' + annotationProcessor 'org.permissionsdispatcher:permissionsdispatcher-processor:4.8.0' + implementation project(path: ':vad') + + testImplementation 'junit:junit:4.12' + androidTestImplementation 'androidx.test:runner:1.3.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' + implementation 'com.google.android.material:material:1.3.0' +} diff --git a/runtime/examples/vad/vad-android-demo/example/local.properties b/runtime/examples/vad/vad-android-demo/example/local.properties new file mode 100644 index 000000000..a89bcf2cc --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/local.properties @@ -0,0 +1,9 @@ +## This file must *NOT* be checked into Version Control Systems, +# as it contains information specific to your local configuration. +# +# Location of the SDK. This is only used by Gradle. +# For customization when using a Version Control System, please read the +# header note. +#Thu Mar 30 16:37:21 CST 2023 +sdk.dir=/Users/masimeng/Library/Android/sdk +ndk.dir=/Users/masimeng/Library/Android/sdk/ndk/23.1.7779620/ diff --git a/runtime/examples/vad/vad-android-demo/example/proguard-rules.pro b/runtime/examples/vad/vad-android-demo/example/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/runtime/examples/vad/vad-android-demo/example/src/androidTest/java/com/konovalov/vad/example/ExampleInstrumentedTest.java b/runtime/examples/vad/vad-android-demo/example/src/androidTest/java/com/konovalov/vad/example/ExampleInstrumentedTest.java new file mode 100644 index 000000000..cc0191793 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/androidTest/java/com/konovalov/vad/example/ExampleInstrumentedTest.java @@ -0,0 +1,27 @@ +package com.konovalov.vad.example; + +import android.content.Context; + +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + + assertEquals("com.konovalov.vad.example", appContext.getPackageName()); + } +} diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/AndroidManifest.xml b/runtime/examples/vad/vad-android-demo/example/src/main/AndroidManifest.xml new file mode 100644 index 000000000..022f73b2c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/AndroidManifest.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/MainActivity.java b/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/MainActivity.java new file mode 100644 index 000000000..29cc17e01 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/MainActivity.java @@ -0,0 +1,204 @@ +package com.konovalov.vad.example; + +import androidx.appcompat.app.AppCompatActivity; + +import android.Manifest; +import android.os.Bundle; +import android.view.View; +import android.widget.AdapterView; +import android.widget.ArrayAdapter; +import android.widget.Spinner; +import android.widget.TextView; + +import com.google.android.material.floatingactionbutton.FloatingActionButton; +import com.konovalov.vad.Vad; +import com.konovalov.vad.example.recorder.VoiceRecorder; +import com.konovalov.vad.example.recorder.VoiceRecorderConfig; + +import java.util.LinkedList; + +import permissions.dispatcher.NeedsPermission; +import permissions.dispatcher.RuntimePermissions; + +@RuntimePermissions +public class MainActivity extends AppCompatActivity implements VoiceRecorder.Listener, View.OnClickListener, AdapterView.OnItemSelectedListener { + + private VoiceRecorderConfig.SampleRate DEFAULT_SAMPLE_RATE = VoiceRecorderConfig.SampleRate.SAMPLE_RATE_16K; + private VoiceRecorderConfig.FrameSize DEFAULT_FRAME_SIZE = VoiceRecorderConfig.FrameSize.FRAME_SIZE_1536; +// private VoiceRecorderConfig.Mode DEFAULT_MODE = VoiceRecorderConfig.Mode.VERY_AGGRESSIVE; + +// private int DEFAULT_SILENCE_DURATION = 500; +// private int DEFAULT_VOICE_DURATION = 500; + + private final String SPINNER_SAMPLE_RATE_TAG = "sample_rate"; + private final String SPINNER_FRAME_SIZE_TAG = "frame_size"; +// private final String SPINNER_MODE_TAG = "mode"; + + private FloatingActionButton recordingActionButton; + private TextView speechTextView; + private Spinner sampleRateSpinner; + private Spinner frameSpinner; +// private Spinner modeSpinner; + + private ArrayAdapter sampleRateAdapter; + private ArrayAdapter frameAdapter; +// private ArrayAdapter modeAdapter; + + private VoiceRecorder recorder; + private VoiceRecorderConfig config; + private boolean isRecording = false; + + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + + config = VoiceRecorderConfig.newBuilder() + .setSampleRate(DEFAULT_SAMPLE_RATE) + .setFrameSize(DEFAULT_FRAME_SIZE) +// .setMode(DEFAULT_MODE) +// .setSilenceDurationMillis(DEFAULT_SILENCE_DURATION) +// .setVoiceDurationMillis(DEFAULT_VOICE_DURATION) + .build(); + + recorder = new VoiceRecorder(this, config); + + speechTextView = findViewById(R.id.speechTextView); + sampleRateSpinner = findViewById(R.id.sampleRateSpinner); + sampleRateAdapter = new ArrayAdapter<>(this, android.R.layout.simple_spinner_dropdown_item, getSampleRates()); + sampleRateSpinner.setAdapter(sampleRateAdapter); + sampleRateSpinner.setTag(SPINNER_SAMPLE_RATE_TAG); + sampleRateSpinner.setSelection(getSampleRates().indexOf(DEFAULT_SAMPLE_RATE.name()), false); + sampleRateSpinner.setOnItemSelectedListener(this); + + frameSpinner = findViewById(R.id.frameSampleRateSpinner); + frameAdapter = new ArrayAdapter<>(this, android.R.layout.simple_spinner_dropdown_item, getFrameSizes()); + frameSpinner.setAdapter(frameAdapter); + frameSpinner.setTag(SPINNER_FRAME_SIZE_TAG); + frameSpinner.setSelection(getFrameSizes().indexOf(DEFAULT_FRAME_SIZE.name()), false); + frameSpinner.setOnItemSelectedListener(this); + +// modeSpinner = findViewById(R.id.modeSpinner); +// modeAdapter = new ArrayAdapter<>(this, android.R.layout.simple_spinner_dropdown_item, getModes()); +// modeSpinner.setAdapter(modeAdapter); +// modeSpinner.setTag(SPINNER_MODE_TAG); +// modeSpinner.setSelection(getModes().indexOf(DEFAULT_MODE.name()), false); +// modeSpinner.setOnItemSelectedListener(this); + + recordingActionButton = findViewById(R.id.recordingActionButton); + recordingActionButton.setOnClickListener(this); + recordingActionButton.setEnabled(false); + + MainActivityPermissionsDispatcher.activateAudioPermissionWithPermissionCheck(this); + } + + private LinkedList getSampleRates() { + LinkedList result = new LinkedList<>(); + for (VoiceRecorderConfig.SampleRate sampleRate : VoiceRecorderConfig.SampleRate.values()) { + result.add(sampleRate.name()); + } + return result; + } + + private LinkedList getFrameSizes() { + LinkedList result = new LinkedList<>(); + result.add(VoiceRecorderConfig.FrameSize.FRAME_SIZE_1536.name()); + + return result; + } + +// private LinkedList getModes() { +// LinkedList result = new LinkedList<>(); +// for (VadConfig.Mode mode : VadConfig.Mode.values()) { +// result.add(mode.name()); +// } +// return result; +// } + + private void startRecording() { + isRecording = true; + recorder.start(); + recordingActionButton.setImageResource(R.drawable.stop); + } + + private void stopRecording() { + isRecording = false; + recorder.stop(); + recordingActionButton.setImageResource(R.drawable.red_dot); + } + + @Override + public void onItemSelected(AdapterView adapterView, View view, int position, long l) { + stopRecording(); + + switch (String.valueOf(adapterView.getTag())) { + case SPINNER_SAMPLE_RATE_TAG: + config.setSampleRate(VoiceRecorderConfig.SampleRate.valueOf(String.valueOf(sampleRateAdapter.getItem(position)))); + + frameAdapter.clear(); + frameAdapter.addAll(getFrameSizes()); + frameAdapter.notifyDataSetChanged(); + frameSpinner.setSelection(0); + + config.setFrameSize(VoiceRecorderConfig.FrameSize.valueOf(String.valueOf(frameAdapter.getItem(0)))); + break; + case SPINNER_FRAME_SIZE_TAG: + config.setFrameSize(VoiceRecorderConfig.FrameSize.valueOf(String.valueOf(frameAdapter.getItem(position)))); + break; +// case SPINNER_MODE_TAG: +// config.setMode(VadConfig.Mode.valueOf(String.valueOf(modeAdapter.getItem(position)))); +// break; + } + + recorder.updateConfig(config); + } + + @Override + public void onNothingSelected(AdapterView parent) { + } + + + @NeedsPermission(Manifest.permission.RECORD_AUDIO) + public void activateAudioPermission() { + recordingActionButton.setEnabled(true); + } + + @Override + public void onClick(View v) { + if (!isRecording) { + startRecording(); + } else { + stopRecording(); + } + } + + @Override + public void onSpeechDetected() { + runOnUiThread(new Runnable() { + @Override + public void run() { + speechTextView.setText(R.string.speech_detected); + } + }); + } + + @Override + public void onNoiseDetected() { + runOnUiThread(new Runnable() { + @Override + public void run() { + speechTextView.setText(R.string.noise_detected); + } + }); + } + + @Override + public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + // NOTE: delegate the permission handling to generated method + MainActivityPermissionsDispatcher.onRequestPermissionsResult(this, requestCode, grantResults); + } + + +} diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/recorder/VoiceRecorder.java b/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/recorder/VoiceRecorder.java new file mode 100644 index 000000000..60af8eb35 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/recorder/VoiceRecorder.java @@ -0,0 +1,151 @@ +package com.konovalov.vad.example.recorder; + +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaRecorder; +import android.util.Log; + +import com.konovalov.vad.example.recorder.VoiceRecorderConfig; + +import com.konovalov.vad.Vad; +//import com.konovalov.vad.VadConfig; +import com.konovalov.vad.VadListener; + +import static android.media.AudioFormat.CHANNEL_IN_MONO; +import static android.media.AudioFormat.CHANNEL_IN_STEREO; + +import java.io.File; + +/** + * Created by George Konovalov on 11/16/2019. + */ + +public class VoiceRecorder { + private static final int PCM_CHANNEL = CHANNEL_IN_MONO; + private static final int PCM_ENCODING_BIT = AudioFormat.ENCODING_PCM_16BIT; + + private VoiceRecorderConfig config; + private Vad vad; + private AudioRecord audioRecord; + private Listener callback; + private Thread thread; + + private boolean isListening = false; + + private static final String TAG = VoiceRecorder.class.getSimpleName(); + + public VoiceRecorder(Listener callback, VoiceRecorderConfig config) { + this.callback = callback; + this.config = config; + this.vad = new Vad(); + } + + public void updateConfig(VoiceRecorderConfig config) { + this.config = config; + } + + public void start() { + stop(); + audioRecord = createAudioRecord(); + if (audioRecord != null) { + isListening = true; + audioRecord.startRecording(); + + thread = new Thread(new ProcessVoice()); + thread.start(); + vad.start(); + } else { + Log.w(TAG, "Failed start Voice Recorder!"); + } + } + + + public void stop() { + isListening = false; + if (thread != null) { + thread.interrupt(); + thread = null; + } + if (audioRecord != null) { + try { + audioRecord.release(); + } catch (Exception e) { + Log.e(TAG, "Error stop AudioRecord ", e); + } + audioRecord = null; + } + if (vad != null) { + vad.stop(); + } + } + + + private AudioRecord createAudioRecord() { + try { + final int minBufSize = AudioRecord.getMinBufferSize(config.getSampleRate().getValue(), PCM_CHANNEL, PCM_ENCODING_BIT); + int frame_size = config.getFrameSize().getValue(); + if (minBufSize > frame_size) { + Log.e(TAG, "minBufSize > frame_size"); + return null; + } + Log.i(TAG, "minBufSize : " + minBufSize); + final AudioRecord audioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC, config.getSampleRate().getValue(), PCM_CHANNEL, PCM_ENCODING_BIT, frame_size); + Log.i(TAG, "config.getSampleRate().getValue() : " + config.getSampleRate().getValue()); + + if (audioRecord.getState() == AudioRecord.STATE_INITIALIZED) { + return audioRecord; + } else { + audioRecord.release(); + } + } catch (IllegalArgumentException e) { + Log.e(TAG, "Error can't create AudioRecord ", e); + } + + return null; + } + + private int getNumberOfChannels() { + switch (PCM_CHANNEL) { + case CHANNEL_IN_MONO: + return 1; + case CHANNEL_IN_STEREO: + return 2; + } + return 1; + } + + private class ProcessVoice implements Runnable { + + @Override + public void run() { + android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO); + final int minBufSize = AudioRecord.getMinBufferSize(config.getSampleRate().getValue(), PCM_CHANNEL, PCM_ENCODING_BIT); + while (!Thread.interrupted() && isListening && audioRecord != null) { + short[] buffer = new short[config.getFrameSize().getValue()]; + audioRecord.read(buffer, 0, buffer.length); + detectSpeech(buffer); + } + } + + private void detectSpeech(short[] buffer) { + vad.addContinuousSpeechListener(buffer, new VadListener() { + @Override + public void onSpeechDetected() { + callback.onSpeechDetected(); + } + + @Override + public void onNoiseDetected() { + callback.onNoiseDetected(); + } + }); + } + } + + public interface Listener { + void onSpeechDetected(); + + void onNoiseDetected(); + } + +} diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/recorder/VoiceRecorderConfig.java b/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/recorder/VoiceRecorderConfig.java new file mode 100644 index 000000000..ea7daac68 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/java/com/konovalov/vad/example/recorder/VoiceRecorderConfig.java @@ -0,0 +1,157 @@ +package com.konovalov.vad.example.recorder; + + +/** + * Created by George Konovalov on 11/16/2019. + */ + +public class VoiceRecorderConfig { + private SampleRate sampleRate; + private Mode mode; + private FrameSize frameSize; + private int voiceDurationMillis; + private int silenceDurationMillis; + + public VoiceRecorderConfig() { + } + + public VoiceRecorderConfig(Builder builder) { + this.voiceDurationMillis = builder.voiceDurationMillis; + this.silenceDurationMillis = builder.silenceDurationMillis; + this.sampleRate = builder.sampleRate; + this.frameSize = builder.frameSize; + this.mode = builder.mode; + } + + public SampleRate getSampleRate() { + return sampleRate; + } + + public Mode getMode() { + return mode; + } + + public FrameSize getFrameSize() { + return frameSize; + } + + public int getVoiceDurationMillis() { + return voiceDurationMillis; + } + + public int getSilenceDurationMillis() { + return silenceDurationMillis; + } + + public void setSampleRate(SampleRate sampleRate) { + this.sampleRate = sampleRate; + } + + public void setMode(Mode mode) { + this.mode = mode; + } + + public void setFrameSize(FrameSize frameSize) { + this.frameSize = frameSize; + } + + public void setVoiceDurationMillis(int voiceDurationMillis) { + this.voiceDurationMillis = voiceDurationMillis; + } + + public void setSilenceDurationMillis(int silenceDurationMillis) { + this.silenceDurationMillis = silenceDurationMillis; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + private SampleRate sampleRate = SampleRate.SAMPLE_RATE_16K; + private Mode mode = Mode.VERY_AGGRESSIVE; + private FrameSize frameSize; + private int voiceDurationMillis = 500; + private int silenceDurationMillis = 500; + + private Builder() { + } + + public Builder setSampleRate(SampleRate sampleRate) { + this.sampleRate = sampleRate; + return this; + } + + public Builder setMode(Mode mode) { + this.mode = mode; + return this; + } + + public Builder setFrameSize(FrameSize frameSize) { + this.frameSize = frameSize; + return this; + } + + public Builder setVoiceDurationMillis(int voiceDurationMillis) { + this.voiceDurationMillis = voiceDurationMillis; + return this; + } + + public Builder setSilenceDurationMillis(int silenceDurationMillis) { + this.silenceDurationMillis = silenceDurationMillis; + return this; + } + + public VoiceRecorderConfig build() { + return new VoiceRecorderConfig(this); + } + } + + public enum SampleRate { + SAMPLE_RATE_16K(16000); + + private int sampleRate; + + public int getValue() { + return sampleRate; + } + + SampleRate(int sampleRate) { + this.sampleRate = sampleRate; + } + } + + public enum Mode { + NORMAL(0), + LOW_BITRATE(1), + AGGRESSIVE(2), + VERY_AGGRESSIVE(3); + + private int mode; + + public int getValue() { + return mode; + } + + Mode(int mode) { + this.mode = mode; + } + } + + public enum FrameSize { + FRAME_SIZE_1536(1536); + + private int frameSize; + + public int getValue() { + return frameSize; + } + + FrameSize(int frameSize) { + this.frameSize = frameSize; + } + } + + + +} diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-hdpi/red_dot.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-hdpi/red_dot.png new file mode 100644 index 000000000..af201525f Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-hdpi/red_dot.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-hdpi/stop.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-hdpi/stop.png new file mode 100644 index 000000000..0840f636e Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-hdpi/stop.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-mdpi/red_dot.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-mdpi/red_dot.png new file mode 100644 index 000000000..56b9a31ce Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-mdpi/red_dot.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-mdpi/stop.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-mdpi/stop.png new file mode 100644 index 000000000..dbb7933d8 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-mdpi/stop.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-v24/ic_launcher_foreground.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..1f6bb2906 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xhdpi/red_dot.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xhdpi/red_dot.png new file mode 100644 index 000000000..3eb74341a Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xhdpi/red_dot.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xhdpi/stop.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xhdpi/stop.png new file mode 100644 index 000000000..26316d385 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xhdpi/stop.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxhdpi/red_dot.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxhdpi/red_dot.png new file mode 100644 index 000000000..c9c2b833d Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxhdpi/red_dot.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxhdpi/stop.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxhdpi/stop.png new file mode 100644 index 000000000..8c053b2f3 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxhdpi/stop.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxxhdpi/red_dot.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxxhdpi/red_dot.png new file mode 100644 index 000000000..2b316ddd9 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxxhdpi/red_dot.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxxhdpi/stop.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxxhdpi/stop.png new file mode 100644 index 000000000..544112d2a Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable-xxxhdpi/stop.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable/ic_launcher_background.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..0d025f9bf --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/layout/activity_main.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..c3841c399 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/layout/activity_main.xml @@ -0,0 +1,141 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 000000000..eca70cfe5 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 000000000..eca70cfe5 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-hdpi/ic_launcher.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 000000000..898f3ed59 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-hdpi/ic_launcher.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-hdpi/ic_launcher_round.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-hdpi/ic_launcher_round.png new file mode 100644 index 000000000..dffca3601 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-hdpi/ic_launcher_round.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-mdpi/ic_launcher.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 000000000..64ba76f75 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-mdpi/ic_launcher.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-mdpi/ic_launcher_round.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-mdpi/ic_launcher_round.png new file mode 100644 index 000000000..dae5e0823 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-mdpi/ic_launcher_round.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xhdpi/ic_launcher.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 000000000..e5ed46597 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xhdpi/ic_launcher_round.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xhdpi/ic_launcher_round.png new file mode 100644 index 000000000..14ed0af35 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxhdpi/ic_launcher.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 000000000..b0907cac3 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxhdpi/ic_launcher_round.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxhdpi/ic_launcher_round.png new file mode 100644 index 000000000..d8ae03154 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxxhdpi/ic_launcher.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 000000000..2c18de9e6 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png new file mode 100644 index 000000000..beed3cdd2 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/example/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/values/colors.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/values/colors.xml new file mode 100644 index 000000000..4a195006d --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/values/colors.xml @@ -0,0 +1,7 @@ + + + #000000 + #00574B + #FFFFFF + #FFFFFF + diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/values/strings.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/values/strings.xml new file mode 100644 index 000000000..d818c6e54 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/values/strings.xml @@ -0,0 +1,11 @@ + + Android VAD + Speech detected! + Noise detected! + Press button to start VAD! + Start + Stop + Sample Rate: + Frame Size: + Mode: + diff --git a/runtime/examples/vad/vad-android-demo/example/src/main/res/values/styles.xml b/runtime/examples/vad/vad-android-demo/example/src/main/res/values/styles.xml new file mode 100644 index 000000000..8c16ac70c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/example/src/main/res/values/styles.xml @@ -0,0 +1,18 @@ + + + + + + + + diff --git a/runtime/examples/vad/vad-android-demo/gradle.properties b/runtime/examples/vad/vad-android-demo/gradle.properties new file mode 100644 index 000000000..199d16ede --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/gradle.properties @@ -0,0 +1,20 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx1536m +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Automatically convert third-party libraries to use AndroidX +android.enableJetifier=true + diff --git a/runtime/examples/vad/vad-android-demo/gradle/wrapper/gradle-wrapper.jar b/runtime/examples/vad/vad-android-demo/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..f6b961fd5 Binary files /dev/null and b/runtime/examples/vad/vad-android-demo/gradle/wrapper/gradle-wrapper.jar differ diff --git a/runtime/examples/vad/vad-android-demo/gradle/wrapper/gradle-wrapper.properties b/runtime/examples/vad/vad-android-demo/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..0d0195493 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Jun 09 15:12:01 EDT 2021 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-all.zip diff --git a/runtime/examples/vad/vad-android-demo/gradlew b/runtime/examples/vad/vad-android-demo/gradlew new file mode 100755 index 000000000..cccdd3d51 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/runtime/examples/vad/vad-android-demo/gradlew.bat b/runtime/examples/vad/vad-android-demo/gradlew.bat new file mode 100644 index 000000000..f9553162f --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/runtime/examples/vad/vad-android-demo/local.properties b/runtime/examples/vad/vad-android-demo/local.properties new file mode 100644 index 000000000..b1369dd5c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/local.properties @@ -0,0 +1,9 @@ +## This file must *NOT* be checked into Version Control Systems, +# as it contains information specific to your local configuration. +# +# Location of the SDK. This is only used by Gradle. +# For customization when using a Version Control System, please read the +# header note. +#Wed Mar 29 16:03:45 CST 2023 +sdk.dir=/Users/masimeng/Library/Android/sdk +ndk.dir=/Users/masimeng/Library/Android/sdk/ndk/20.0.5594570/ diff --git a/runtime/examples/vad/vad-android-demo/settings.gradle b/runtime/examples/vad/vad-android-demo/settings.gradle new file mode 100644 index 000000000..745e4c5e4 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/settings.gradle @@ -0,0 +1,2 @@ +include ':example', ':vad' +rootProject.name='VAD' diff --git a/runtime/examples/vad/vad-android-demo/vad/.gitignore b/runtime/examples/vad/vad-android-demo/vad/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/.gitignore @@ -0,0 +1 @@ +/build diff --git a/runtime/examples/vad/vad-android-demo/vad/build.gradle b/runtime/examples/vad/vad-android-demo/vad/build.gradle new file mode 100644 index 000000000..c7d5a256f --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/build.gradle @@ -0,0 +1,67 @@ +apply plugin: 'com.android.library' +apply plugin: 'com.github.dcendents.android-maven' + +group='com.github.gkonovalov' + +android { + compileSdkVersion 31 + buildToolsVersion "29.0.3" + + + defaultConfig { + minSdkVersion 16 + targetSdkVersion 31 + versionCode 2 + versionName "1.0.1" + + setProperty("archivesBaseName", "android-vad-v" + versionName) + + externalNativeBuild { + cmake { + cppFlags '' + } + } + + ndk { + abiFilters "arm64-v8a" + } + + ndk { + abiFilters "arm64-v8a" + } + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + consumerProguardFiles 'consumer-rules.pro' + } + + externalNativeBuild { + cmake { + path file('src/main/cpp/CMakeLists.txt') + version '3.10.2' + } + } + + //我们将外部so库放在jniLibs文件夹下，因此要将它设置为jniLibs使工程在打包的时候能将它包含进去，否则app运行时会报无法找到so库的错误。 + sourceSets { + main { + jniLibs.srcDirs = ['src/main/cpp/jniLibs'] + resources { srcDirs = ['src/main/cpp/resources'] } + } + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + + testImplementation 'junit:junit:4.12' + androidTestImplementation 'androidx.test:runner:1.3.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' +} diff --git a/runtime/examples/vad/vad-android-demo/vad/consumer-rules.pro b/runtime/examples/vad/vad-android-demo/vad/consumer-rules.pro new file mode 100644 index 000000000..e69de29bb diff --git a/runtime/examples/vad/vad-android-demo/vad/proguard-rules.pro b/runtime/examples/vad/vad-android-demo/vad/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/runtime/examples/vad/vad-android-demo/vad/src/androidTest/java/com/konovalov/vad/ExampleInstrumentedTest.java b/runtime/examples/vad/vad-android-demo/vad/src/androidTest/java/com/konovalov/vad/ExampleInstrumentedTest.java new file mode 100644 index 000000000..3e5cbef2b --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/androidTest/java/com/konovalov/vad/ExampleInstrumentedTest.java @@ -0,0 +1,27 @@ +package com.konovalov.vad; + +import android.content.Context; + +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.ext.junit.runners.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + + assertEquals("com.konovalov.vad.test", appContext.getPackageName()); + } +} diff --git a/runtime/examples/vad/vad-android-demo/vad/src/main/AndroidManifest.xml b/runtime/examples/vad/vad-android-demo/vad/src/main/AndroidManifest.xml new file mode 100644 index 000000000..455bb333b --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/main/AndroidManifest.xml @@ -0,0 +1,2 @@ + diff --git a/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/CMakeLists.txt b/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..898c54e01 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/CMakeLists.txt @@ -0,0 +1,59 @@ + +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html + +# Sets the minimum version of CMake required to build the native library. + +cmake_minimum_required(VERSION 3.10) + +# Declares and names the project. + +project("vad") + +include_directories(src/main/cpp/) +file(GLOB CPP_FILES "src/main/cpp/*.cpp") + +# 添加so库存放位置 +set(distribution_DIR ${CMAKE_SOURCE_DIR}/jniLibs) + +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. + +add_library( # Sets the name of the library. + native-lib + + # Sets the library as a shared library. + SHARED + + # Provides a relative path to your source file(s). + ${CMAKE_CURRENT_SOURCE_DIR}/native-lib.cpp ) + +# Searches for a specified prebuilt library and stores the path as a +# variable. Because CMake includes system libraries in the search path by +# default, you only need to specify the name of the public NDK library +# you want to add. CMake verifies that the library exists before +# completing its build. + +find_library( # Sets the name of the path variable. + log-lib + + # Specifies the name of the NDK library that + # you want CMake to locate. + log ) + +# Specifies libraries CMake should link to your target library. You +# can link multiple libraries, such as libraries you define in this +# build script, prebuilt third-party libraries, or system libraries. + +target_link_libraries( # Specifies the target library. + native-lib + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libc++_shared.so + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libfastdeploy.so + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libonnxruntime.so + ${CMAKE_SOURCE_DIR}/jniLibs/${ANDROID_ABI}/libpps_vad_interface.so + # Links the target library to the log library + # included in the NDK. + ${log-lib} + ) diff --git a/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/includes/vad_interface.h b/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/includes/vad_interface.h new file mode 100644 index 000000000..5d7ca7091 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/includes/vad_interface.h @@ -0,0 +1,46 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* PPSHandle_t; + +typedef enum { + PPS_VAD_ILLEGAL = 0, // error + PPS_VAD_SIL, // silence + PPS_VAD_START, // start speech + PPS_VAD_SPEECH, // in speech + PPS_VAD_END, // end speech + PPS_VAD_NUMSTATES, // number of states +} PPSVadState_t; + +PPSHandle_t PPSVadCreateInstance(const char* conf_path); + +int PPSVadDestroyInstance(PPSHandle_t instance); + +int PPSVadReset(PPSHandle_t instance); + +int PPSVadChunkSizeSamples(PPSHandle_t instance); + +PPSVadState_t PPSVadFeedForward(PPSHandle_t instance, + float* chunk, + int num_element); + +#ifdef __cplusplus +} +#endif // __cplusplus \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/native-lib.cpp b/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/native-lib.cpp new file mode 100644 index 000000000..a0af4491d --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/main/cpp/native-lib.cpp @@ -0,0 +1,121 @@ +// Write C++ code here. +// +// Do not forget to dynamically load the C++ library into your application. +// +// For instance, +// +// In MainActivity.java: +// static { +// System.loadLibrary("mysotest"); +// } +// +// Or, in MainActivity.kt: +// companion object { +// init { +// System.loadLibrary("mysotest") +// } +// } + +#include +#include +#include +#include +#include "includes/vad_interface.h" +#include +#include + +//如果你不想用引入头文件的方法，可以把导入头文件的include语句注释掉，然后将下面这句取消注释。 +//string getStringFromSoLibrary(); + +void* vad_instance = nullptr; +int num_chunk = 0; +bool is_start = 0; +std::ofstream fp_pcm; +std::ofstream fp_log; + +extern "C" +JNIEXPORT jboolean JNICALL Java_com_konovalov_vad_Vad_PPSVadCreateInstance(JNIEnv *env, jobject instance) +{ + if (vad_instance != nullptr) { + int ret = PPSVadDestroyInstance(vad_instance); + vad_instance = nullptr; + } + std::string conf_path = "/storage/self/primary/masimeng/vad/autotest/vad.conf"; + vad_instance = PPSVadCreateInstance(conf_path.c_str()); + __android_log_print(ANDROID_LOG_INFO,"[vad]", "vad instance create success"); + std::string fp_pcm_name = "/storage/self/primary/masimeng/vad/autotest/ori_pcm.pcm"; + std::string fp_log_name = "/storage/self/primary/masimeng/vad/autotest/ori_pcm.pcm.log"; + fp_pcm = std::ofstream(fp_pcm_name, std::ofstream::binary); + fp_log = std::ofstream(fp_log_name); + return true; +} + +extern "C" +JNIEXPORT jboolean JNICALL Java_com_konovalov_vad_Vad_PPSVadDestroyInstance(JNIEnv *env, jobject instance){ + if (vad_instance != nullptr) { + int ret =PPSVadDestroyInstance(vad_instance); + vad_instance = nullptr; + num_chunk = 0; + fp_pcm.close(); + fp_log.close(); + __android_log_print(ANDROID_LOG_INFO,"[vad]", "vad instance destroy success"); + } + return true; +} + +extern "C" +JNIEXPORT jboolean JNICALL Java_com_konovalov_vad_Vad_PPSVadReset(JNIEnv *env, jobject instance){ + if (vad_instance != nullptr) { + int ret = PPSVadReset(vad_instance); + __android_log_print(ANDROID_LOG_INFO,"[vad]", "vad instance reset success"); + } + return true; +} + +extern "C" +JNIEXPORT jint JNICALL Java_com_konovalov_vad_Vad_PPSVadChunkSizeSamples(JNIEnv *env, jobject instance) { + int ret = -1; + if (vad_instance != nullptr) { + ret = PPSVadChunkSizeSamples(vad_instance); + } + return ret; +} + +extern "C" +JNIEXPORT jboolean JNICALL Java_com_konovalov_vad_Vad_PPSVadFeedForward(JNIEnv *env, jobject instance, jshortArray audio){ + jshort *arrayElements = (*env).GetShortArrayElements(audio, 0); + jint array_size = (*env).GetArrayLength(audio); + fp_pcm.write(reinterpret_cast(arrayElements), array_size * sizeof(jshort)); + + if (vad_instance != nullptr) { + int chunk_size = PPSVadChunkSizeSamples(vad_instance); +// __android_log_print(ANDROID_LOG_INFO,"[vad]", "chunk size is : %d", chunk_size); + if (array_size != chunk_size){ + __android_log_print(ANDROID_LOG_INFO,"[vad]", "error array_size : %d != chunk_size : %d", array_size, chunk_size); + return false; + } + std::vector chunk(array_size, 0); +// __android_log_print(ANDROID_LOG_INFO,"[vad]", "array size : %d array[0]: %d", array_size, arrayElements[0]); + + for (int i = 0; i < array_size; ++i){ + chunk[i] = (float)arrayElements[i]; + } + PPSVadState_t ret = PPSVadFeedForward(vad_instance, chunk.data(), array_size); + if (ret == PPS_VAD_START){ + __android_log_print(ANDROID_LOG_INFO,"[vad]", "st:%s s", std::to_string(num_chunk * chunk_size * 1.0 / 16 / 1000).c_str()); + is_start = true; + fp_log << "st: " << std::to_string(num_chunk * chunk_size * 1.0 / 16 / 1000).c_str() << "s "; + } else if(is_start == true && ret == PPS_VAD_END){ + __android_log_print(ANDROID_LOG_INFO,"[vad]", "et:%s s", std::to_string(num_chunk * chunk_size * 1.0 / 16 / 1000).c_str()); + is_start = false; + fp_log << "et: " << std::to_string(num_chunk * chunk_size * 1.0 / 16 / 1000).c_str() << "s\n"; + } + num_chunk ++; +// __android_log_print(ANDROID_LOG_INFO,"[vad]", "num_chunk:%d", num_chunk); + if (ret == PPS_VAD_START || ret == PPS_VAD_SPEECH){ + return true; + } + return false; + } + return false; +} \ No newline at end of file diff --git a/runtime/examples/vad/vad-android-demo/vad/src/main/java/com/konovalov/vad/Vad.java b/runtime/examples/vad/vad-android-demo/vad/src/main/java/com/konovalov/vad/Vad.java new file mode 100644 index 000000000..311187898 --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/main/java/com/konovalov/vad/Vad.java @@ -0,0 +1,89 @@ +package com.konovalov.vad; + +import android.text.Html; + +import java.util.LinkedHashMap; +import java.util.LinkedList; + +/** + * Created by George Konovalov on 11/16/2019. + */ + +public class Vad { + public native boolean PPSVadCreateInstance(); + public native boolean PPSVadDestroyInstance(); + public native boolean PPSVadReset(); + public native int PPSVadChunkSizeSamples(); + public native boolean PPSVadFeedForward(short[] audio); + + private boolean needResetDetectedSamples = true; + private long detectedVoiceSamplesMillis = 0; + private long detectedSilenceSamplesMillis = 0; + private long previousTimeMillis = System.currentTimeMillis(); + + public Vad() { + } + + public void start() { + try { + boolean result = PPSVadCreateInstance(); + + if (result != true) { + throw new RuntimeException("PPSVadCreateInstance error!"); + } + } catch (Exception e) { + throw new RuntimeException("Error can't start VAD!", e); + } + } + + public void stop() { + try { + boolean result = PPSVadDestroyInstance(); + } catch (Exception e) { + throw new RuntimeException("PPSVadDestroyInstance error!", e); + } + } + + public boolean isSpeech(short[] audio) { + if (audio == null) { + throw new NullPointerException("Audio data is NULL!"); + } + + try { + return PPSVadFeedForward(audio); + } catch (Exception e) { + throw new RuntimeException("PPSVadFeedForward error!", e); + } + } + + @Deprecated + public void isContinuousSpeech(short[] audio, VadListener listener) { + addContinuousSpeechListener(audio, listener); + } + + public void addContinuousSpeechListener(short[] audio, VadListener listener) { + if (audio == null) { + throw new NullPointerException("Audio data is NULL!"); + } + + if (listener == null) { + throw new NullPointerException("VadListener is NULL!"); + } + + long currentTimeMillis = System.currentTimeMillis(); + + if (isSpeech(audio)) { + needResetDetectedSamples = true; + listener.onSpeechDetected(); + } else { + if (needResetDetectedSamples) { + needResetDetectedSamples = false; + } + listener.onNoiseDetected(); + } + } + + static { + System.loadLibrary("native-lib"); + } +} diff --git a/runtime/examples/vad/vad-android-demo/vad/src/main/java/com/konovalov/vad/VadListener.java b/runtime/examples/vad/vad-android-demo/vad/src/main/java/com/konovalov/vad/VadListener.java new file mode 100644 index 000000000..14f6cd79c --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/main/java/com/konovalov/vad/VadListener.java @@ -0,0 +1,12 @@ +package com.konovalov.vad; + +/** + * Created by George Konovalov on 11/16/2019. + */ + + +public interface VadListener { + void onSpeechDetected(); + + void onNoiseDetected(); +} diff --git a/runtime/examples/vad/vad-android-demo/vad/src/main/res/values/strings.xml b/runtime/examples/vad/vad-android-demo/vad/src/main/res/values/strings.xml new file mode 100644 index 000000000..e6e96e9df --- /dev/null +++ b/runtime/examples/vad/vad-android-demo/vad/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + VAD +