Skip to content

Commit 452dd6b

Browse files
kanatkanat
authored
Patch/android/external audio processing (#37)
* [PBE-5300] add android-external-audio-processing * [PBE-5300] add external_processor.cc * [PBE-5300] clean up jni * [PBE-5300] clean java layer * [PBE-5300] rename files & classes * [PBE-5300] add external processor * [PBE-5300] fix java compilation * [PBE-5300] remove NonNull annotation * [PBE-5300] add missing include * [PBE-5300] pass external_processor * [PBE-5300] fix unguarded headers * [PBE-5300] fix JNI_ExternalAudioProcessingFactory params * [PBE-5300] change include order * [PBE-5300] jni experiment 1 * [PBE-5300] jni experiment 2 * [PBE-5300] jni experiment 3 * [PBE-5300] jni experiment 4 * [PBE-5300] jni experiment 5 * [PBE-5300] jni experiment 6 * [PBE-5300] jni experiment 7 * [PBE-5300] jni experiment 8 * [PBE-5300] jni experiment 9 * [PBE-5300] jni experiment 10 * [PBE-5300] jni experiment 11 * [PBE-5300] jni experiment 12 * [PBE-5300] jni experiment 13 * [PBE-5300] jni experiment 14 * [PBE-5300] jni experiment 15 * [PBE-5300] add dynamic processing * fix nativeGetInstance * rename to nativeGetApm * fix compilation * hardcode name * fix jni compilation * fix compilation issue * fix dynamic_processing.cc * rename to dynamic_apm_ptr * convert dynamic to external * fix compilation issues * fix nativeDestroyAudioProcessingModule return type * include external_processor_loader * update BUILD.gn * fix loadExternalProcessor return type * define Create & Destroy * delete loader include * fix Load to Create * migrate to external functions * clean up logs
1 parent 05c2ec7 commit 452dd6b

File tree

7 files changed

+413
-0
lines changed

7 files changed

+413
-0
lines changed

‎extensions/android-external-audio-processing/BUILD.gn

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import("../../webrtc.gni")
2+
import("//build/config/android/config.gni")
3+
import("//build/config/android/rules.gni")
4+
5+
rtc_library("external_processing") {
6+
visibility = [ "*" ]
7+
configs += [ "../../modules/audio_processing:apm_debug_dump", ]
8+
configs += [ "//build/config/android:hide_all_but_jni" ]
9+
defines = ["WEBRTC_ANDROID", "WEBRTC_POSIX",]
10+
sources = [
11+
"include/external_processor.hpp",
12+
"external_processing.hpp",
13+
"external_processing.cc",
14+
"external_processing_factory_jni.cc",
15+
]
16+
lib_dirs = []
17+
deps = [
18+
":generated_external_jni",
19+
":external_java",
20+
"../../rtc_base:logging",
21+
"../../sdk/android:native_api_jni",
22+
"../../modules/audio_processing:audio_processing",
23+
]
24+
}
25+
26+
rtc_android_library("external_java") {
27+
visibility = [ "*" ]
28+
sources = [
29+
"java/src/org/webrtc/ExternalAudioProcessingFactory.java",
30+
]
31+
deps = ["//sdk/android:peerconnection_java",]
32+
}
33+
34+
generate_jni("generated_external_jni") {
35+
sources = [
36+
"java/src/org/webrtc/ExternalAudioProcessingFactory.java",
37+
]
38+
namespace = "external"
39+
jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
40+
}

‎extensions/android-external-audio-processing/external_processing.cc

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
#include "external_processing.hpp"
2+
3+
#include <dlfcn.h>
4+
#include <syslog.h>
5+
6+
#include "rtc_base/time_utils.h"
7+
8+
namespace external {
9+
10+
ExternalProcessing* ExternalProcessing::m_instance = nullptr;
11+
12+
ExternalProcessing::ExternalProcessing() {
13+
::syslog(LOG_INFO, "ExternalProcessing: #Constructor;");
14+
}
15+
16+
ExternalProcessing::~ExternalProcessing() {
17+
::syslog(LOG_INFO, "ExternalProcessing: #Destructor;");
18+
19+
Destroy();
20+
}
21+
22+
bool ExternalProcessing::Create(const char* libname) {
23+
::syslog(LOG_INFO, "ExternalProcessing: #Create; libname: %s", libname);
24+
25+
// Load the shared library
26+
dlerror(); // Clear any existing errors
27+
void* handle = dlopen(libname, RTLD_LAZY);
28+
if (!handle) {
29+
::syslog(LOG_ERR, "ExternalProcessing: #Create; Failed to load library: %s",
30+
dlerror());
31+
return false;
32+
}
33+
34+
// Load external processor functions
35+
for (size_t functionId = 0; functionId < kFunctionCount; ++functionId) {
36+
const char* functionName = kFunctionNames[functionId];
37+
syslog(LOG_INFO, "ExternalProcessing: #Create; Loading function: %s",
38+
functionName);
39+
void* functionPtr = dlsym(handle, functionName);
40+
const char* dlsym_error = dlerror();
41+
if (dlsym_error) {
42+
syslog(LOG_ERR,
43+
"ExternalProcessing: #Create; Failed to load the function: %s",
44+
dlsym_error);
45+
return false;
46+
}
47+
m_functionPointers[functionId] = functionPtr;
48+
}
49+
50+
void* createPtr = m_functionPointers[FunctionId::ExternalProcessorCreate];
51+
if (!createPtr) {
52+
::syslog(LOG_ERR,
53+
"ExternalProcessing: #Create; Failed to access "
54+
"ExternalProcessorCreate function");
55+
dlclose(handle);
56+
return false;
57+
}
58+
59+
auto createFunc =
60+
reinterpret_cast<ExternalProcessorCreateFuncType>(createPtr);
61+
if (!createFunc()) {
62+
::syslog(LOG_ERR,
63+
"ExternalProcessing: #Create; Failed to invoke "
64+
"ExternalProcessorCreate function");
65+
dlclose(handle);
66+
return false;
67+
}
68+
69+
m_handle = handle;
70+
71+
::syslog(LOG_INFO, "ExternalProcessing: #Create; completed successfully");
72+
73+
return true;
74+
}
75+
76+
bool ExternalProcessing::Destroy() {
77+
::syslog(LOG_INFO, "ExternalProcessing: #Destroy;");
78+
79+
void* destroyPtr = m_functionPointers[FunctionId::ExternalProcessorDestroy];
80+
if (destroyPtr) {
81+
::syslog(LOG_INFO,
82+
"ExternalProcessing: #Destroy; Invoke ExternalProcessorDestroy "
83+
"function");
84+
85+
auto destroyFunc =
86+
reinterpret_cast<ExternalProcessorDestroyFuncType>(destroyPtr);
87+
if (destroyFunc()) {
88+
::syslog(LOG_INFO,
89+
"ExternalProcessing: #Destroy; Invoked ExternalProcessorDestroy "
90+
"successfully");
91+
}
92+
}
93+
for (auto& functionPtr : m_functionPointers) {
94+
functionPtr = nullptr;
95+
}
96+
if (m_handle) {
97+
dlclose(m_handle);
98+
m_handle = nullptr;
99+
}
100+
101+
return true;
102+
}
103+
104+
void ExternalProcessing::Initialize(int sample_rate_hz, int num_channels) {
105+
if (m_functionPointers.size() <=
106+
static_cast<size_t>(FunctionId::ExternalProcessorInitialize)) {
107+
::syslog(LOG_ERR,
108+
"ExternalProcessing: #Initialize; m_functionPointers is not large "
109+
"enough");
110+
return;
111+
}
112+
void* initializePtr =
113+
m_functionPointers[FunctionId::ExternalProcessorInitialize];
114+
if (!initializePtr) {
115+
::syslog(LOG_ERR,
116+
"ExternalProcessing: #Initialize; Failed to access "
117+
"ExternalProcessorInitialize function");
118+
return;
119+
}
120+
121+
auto initializeFunc =
122+
reinterpret_cast<ExternalProcessorInitializeFuncType>(initializePtr);
123+
if (!initializeFunc(sample_rate_hz, num_channels)) {
124+
::syslog(LOG_ERR,
125+
"ExternalProcessing: #Initialize; Failed to invoke "
126+
"ExternalProcessorInitialize function");
127+
return;
128+
}
129+
::syslog(LOG_INFO,
130+
"ExternalProcessing: #Initialize; Invoked "
131+
"ExternalProcessorInitialize; sample_rate_hz: %i, "
132+
"num_channels: %i",
133+
sample_rate_hz, num_channels);
134+
}
135+
136+
void ExternalProcessing::Process(webrtc::AudioBuffer* audio) {
137+
float* const* channels = audio->channels();
138+
size_t num_frames = audio->num_frames();
139+
size_t num_bands = audio->num_bands();
140+
size_t num_channels = audio->num_channels();
141+
142+
if (m_functionPointers.size() <=
143+
static_cast<size_t>(FunctionId::ExternalProcessorProcessFrame)) {
144+
::syslog(
145+
LOG_ERR,
146+
"ExternalProcessing: #Process; m_functionPointers is not large enough");
147+
return;
148+
}
149+
void* processPtr =
150+
m_functionPointers[FunctionId::ExternalProcessorProcessFrame];
151+
if (!processPtr) {
152+
::syslog(LOG_ERR,
153+
"ExternalProcessing: #Process; Failed to access "
154+
"ExternalProcessorProcessFrame function");
155+
return;
156+
}
157+
158+
auto processFunc =
159+
reinterpret_cast<ExternalProcessorProcessFrameFuncType>(processPtr);
160+
if (!processFunc(channels, num_frames, num_bands, num_channels)) {
161+
::syslog(LOG_ERR,
162+
"ExternalProcessing: #Process; Failed to invoke "
163+
"ExternalProcessorProcessFrame function");
164+
return;
165+
}
166+
}
167+
168+
std::string ExternalProcessing::ToString() const {
169+
return "ExternalProcessing";
170+
}
171+
172+
void ExternalProcessing::SetRuntimeSetting(
173+
webrtc::AudioProcessing::RuntimeSetting setting) {
174+
::syslog(LOG_INFO, "ExternalProcessing: #SetRuntimeSetting;");
175+
}
176+
177+
} // end of namespace external

‎extensions/android-external-audio-processing/external_processing.hpp

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#ifndef EXTERNAL_PROCESSING_HPP
2+
#define EXTERNAL_PROCESSING_HPP
3+
4+
#include <syslog.h>
5+
6+
#include "include/external_processor.hpp"
7+
8+
#include "modules/audio_processing/audio_buffer.h"
9+
#include "modules/audio_processing/audio_processing_impl.h"
10+
#include "modules/audio_processing/include/audio_processing.h"
11+
12+
namespace external {
13+
14+
class ExternalProcessing : public webrtc::CustomProcessing {
15+
public:
16+
ExternalProcessing(const ExternalProcessing&) = delete;
17+
ExternalProcessing(ExternalProcessing&&) = delete;
18+
ExternalProcessing& operator=(const ExternalProcessing&) = delete;
19+
ExternalProcessing& operator=(ExternalProcessing&&) = delete;
20+
~ExternalProcessing();
21+
22+
static ExternalProcessing* getInstance() {
23+
if (m_instance == nullptr) {
24+
m_instance = new ExternalProcessing();
25+
}
26+
return m_instance;
27+
}
28+
29+
bool Create(const char* libname);
30+
31+
bool Destroy();
32+
33+
void Initialize(int sample_rate_hz, int num_channels) override;
34+
void Process(webrtc::AudioBuffer* audio) override;
35+
std::string ToString() const override;
36+
void SetRuntimeSetting(
37+
webrtc::AudioProcessing::RuntimeSetting setting) override;
38+
39+
private:
40+
ExternalProcessing();
41+
42+
void* m_handle = nullptr;
43+
std::array<void *, kFunctionCount> m_functionPointers = {};
44+
45+
static ExternalProcessing* m_instance;
46+
};
47+
} // namespace external
48+
49+
#endif // EXTERNAL_PROCESSING_HPP

‎extensions/android-external-audio-processing/external_processing_factory_jni.cc

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#include <syslog.h>
2+
3+
#include <cstring>
4+
5+
#include "extensions/android-external-audio-processing/generated_external_jni/ExternalAudioProcessingFactory_jni.h"
6+
#include "external_processing.hpp"
7+
#include "rtc_base/checks.h"
8+
#include "rtc_base/ref_counted_object.h"
9+
#include "rtc_base/thread.h"
10+
#include "rtc_base/time_utils.h"
11+
#include "sdk/android/src/jni/jni_helpers.h"
12+
13+
namespace external {
14+
15+
webrtc::AudioProcessing* apm_ptr = nullptr;
16+
17+
static jlong JNI_ExternalAudioProcessingFactory_CreateAudioProcessingModule(
18+
JNIEnv* env,
19+
const webrtc::JavaParamRef<jstring>& libnameRef
20+
) {
21+
22+
if (libnameRef.is_null()) {
23+
::syslog(LOG_ERR, "EXTERNAL-JNI: #GetApm; libname is null");
24+
return 0;
25+
}
26+
27+
const char* libname = env->GetStringUTFChars(libnameRef.obj(), nullptr);
28+
29+
::syslog(LOG_INFO, "EXTERNAL-JNI: #GetApm; libname: %s", libname);
30+
31+
auto instance = ExternalProcessing::getInstance();
32+
if (!instance->Create(libname)) {
33+
::syslog(LOG_ERR, "EXTERNAL-JNI: #GetApm; Failed to load external processor");
34+
env->ReleaseStringUTFChars(libnameRef.obj(), libname);
35+
return 0;
36+
}
37+
38+
env->ReleaseStringUTFChars(libnameRef.obj(), libname);
39+
40+
std::unique_ptr<webrtc::CustomProcessing> external_processing(instance);
41+
auto apm = webrtc::AudioProcessingBuilder()
42+
.SetCapturePostProcessing(std::move(external_processing))
43+
.Create();
44+
webrtc::AudioProcessing::Config config;
45+
config.echo_canceller.enabled = false;
46+
config.echo_canceller.mobile_mode = true;
47+
apm->ApplyConfig(config);
48+
apm_ptr = apm.release();
49+
return webrtc::jni::jlongFromPointer(apm_ptr);
50+
}
51+
52+
static void JNI_ExternalAudioProcessingFactory_DestroyAudioProcessingModule(
53+
JNIEnv* env
54+
) {
55+
ExternalProcessing::getInstance()->Destroy();
56+
delete apm_ptr;
57+
apm_ptr = nullptr;
58+
}
59+
60+
} // end of namespace external

‎extensions/android-external-audio-processing/include/external_processor.hpp

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#ifndef EXTERNAL_PROCESSOR_HPP
2+
#define EXTERNAL_PROCESSOR_HPP
3+
4+
#include <cstdint>
5+
#include <array>
6+
7+
namespace external {
8+
9+
static constexpr unsigned int kFunctionCount = 4;
10+
11+
// Function ID struct
12+
struct FunctionId {
13+
static constexpr unsigned int ExternalProcessorCreate = 0;
14+
static constexpr unsigned int ExternalProcessorInitialize = 1;
15+
static constexpr unsigned int ExternalProcessorProcessFrame = 2;
16+
static constexpr unsigned int ExternalProcessorDestroy = 3;
17+
};
18+
19+
static constexpr std::array<const char *, kFunctionCount> kFunctionNames =
20+
{
21+
"ExternalProcessorCreate",
22+
"ExternalProcessorInitialize",
23+
"ExternalProcessorProcessFrame",
24+
"ExternalProcessorDestroy"
25+
};
26+
27+
// Function type definitions
28+
using ExternalProcessorCreateFuncType = bool(*)();
29+
using ExternalProcessorInitializeFuncType = bool(*)(int sample_rate_hz, int num_channels);
30+
using ExternalProcessorProcessFrameFuncType = bool(*)(float* const* channels,
31+
size_t num_frames,
32+
size_t num_bands,
33+
size_t num_channels);
34+
using ExternalProcessorDestroyFuncType = bool(*)();
35+
36+
extern "C" bool ExternalProcessorCreate();
37+
38+
extern "C" bool ExternalProcessorInitialize(int sample_rate_hz,
39+
int num_channels);
40+
41+
extern "C" bool ExternalProcessorProcessFrame(float* const* channels,
42+
size_t num_frames,
43+
size_t num_bands,
44+
size_t num_channels);
45+
46+
extern "C" bool ExternalProcessorDestroy();
47+
48+
} // namespace external
49+
50+
#endif // EXTERNAL_PROCESSOR_HPP

0 commit comments

Comments
 (0)