[AND-179] Refactor external audio processing

extensions/android-external-audio-processing/BUILD.gn

@@ -27,14 +27,14 @@ rtc_library("external_processing") {
 rtc_android_library("external_java") {
     visibility = [ "*" ]
     sources = [
-        "java/src/org/webrtc/ExternalAudioProcessingFactory.java",
+        "java/src/org/webrtc/NativeExternalAudioProcessingFactory.java",
     ]
     deps = ["//sdk/android:peerconnection_java",]
 }
 
 generate_jni("generated_external_jni") {
     sources = [
-        "java/src/org/webrtc/ExternalAudioProcessingFactory.java",
+        "java/src/org/webrtc/NativeExternalAudioProcessingFactory.java",
     ]
     namespace = "external"
     jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"

extensions/android-external-audio-processing/external_processing_factory_jni.cc

@@ -2,7 +2,7 @@
 
 #include <cstring>
 
-#include "extensions/android-external-audio-processing/generated_external_jni/ExternalAudioProcessingFactory_jni.h"
+#include "extensions/android-external-audio-processing/generated_external_jni/NativeExternalAudioProcessingFactory_jni.h"
 #include "external_processing.hpp"
 #include "rtc_base/checks.h"
 #include "rtc_base/ref_counted_object.h"
@@ -14,7 +14,7 @@ namespace external {
 
 webrtc::AudioProcessing* apm_ptr = nullptr;
 
-static jlong JNI_ExternalAudioProcessingFactory_CreateAudioProcessingModule(
+static jlong JNI_NativeExternalAudioProcessingFactory_CreateAudioProcessingModule(
     JNIEnv* env,
     const webrtc::JavaParamRef<jstring>& libnameRef
 ) {
@@ -49,7 +49,7 @@ static jlong JNI_ExternalAudioProcessingFactory_CreateAudioProcessingModule(
   return webrtc::jni::jlongFromPointer(apm_ptr);
 }
 
-static void JNI_ExternalAudioProcessingFactory_DestroyAudioProcessingModule(
+static void JNI_NativeExternalAudioProcessingFactory_DestroyAudioProcessingModule(
     JNIEnv* env
 ) {
   ExternalProcessing::getInstance()->Destroy();

extensions/android-external-audio-processing/java/src/org/webrtc/ExternalAudioProcessingFactory.java → extensions/android-external-audio-processing/java/src/org/webrtc/NativeExternalAudioProcessingFactory.java

@@ -2,11 +2,11 @@
 
 import org.webrtc.AudioProcessingFactory;
 
-public class ExternalAudioProcessingFactory implements AudioProcessingFactory {
+public class NativeExternalAudioProcessingFactory implements AudioProcessingFactory {
 
   private final String libname;
 
-  public ExternalAudioProcessingFactory(String libname) {
+  public NativeExternalAudioProcessingFactory(String libname) {
     if (libname == null) {
       throw new NullPointerException("libname must not be null.");
     }

sdk/android/BUILD.gn

@@ -270,6 +270,7 @@ if (is_android) {
       "api/org/webrtc/CryptoOptions.java",
       "api/org/webrtc/DataChannel.java",
       "api/org/webrtc/DtmfSender.java",
+      "api/org/webrtc/ExternalAudioProcessingFactory.java",
       "api/org/webrtc/FecControllerFactoryFactoryInterface.java",
       "api/org/webrtc/FrameCryptor.java",
       "api/org/webrtc/FrameCryptorAlgorithm.java",
@@ -744,6 +745,11 @@ if (current_os == "linux" || is_android) {
       "src/jni/pc/data_channel.cc",
       "src/jni/pc/data_channel.h",
       "src/jni/pc/dtmf_sender.cc",
+      "src/jni/pc/external_audio_processing_factory.cc",
+      "src/jni/pc/external_audio_processing_factory.h",
+      "src/jni/pc/external_audio_processing_interface.h",
+      "src/jni/pc/external_audio_processor.cc",
+      "src/jni/pc/external_audio_processor.h",
       "src/jni/pc/frame_cryptor.cc",
       "src/jni/pc/frame_cryptor.h",
       "src/jni/pc/frame_cryptor_key_provider.cc",
@@ -1445,6 +1451,7 @@ if (current_os == "linux" || is_android) {
       "api/org/webrtc/CryptoOptions.java",
       "api/org/webrtc/DataChannel.java",
       "api/org/webrtc/DtmfSender.java",
+      "api/org/webrtc/ExternalAudioProcessingFactory.java",
       "api/org/webrtc/FrameCryptor.java",
       "api/org/webrtc/FrameCryptorFactory.java",
       "api/org/webrtc/FrameCryptorKeyProvider.java",

sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright 2022 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.webrtc;
+
+import java.nio.ByteBuffer;
+
+import androidx.annotation.Nullable;
+import org.webrtc.AudioProcessingFactory;
+
+
+public class ExternalAudioProcessingFactory implements AudioProcessingFactory {
+
+  /**
+   * Interface for external audio processing.
+   */
+  public static interface AudioProcessing {
+    /**
+     * Called when the processor should be initialized with a new sample rate and
+     * number of channels.
+     */
+    @CalledByNative("AudioProcessing")
+    void initialize(int sampleRateHz, int numChannels);
+    /** Called when the processor should be reset with a new sample rate. */  
+    @CalledByNative("AudioProcessing")
+    void reset(int newRate);
+    /**  
+     * Processes the given capture or render signal. NOTE: `buffer.data` will be  
+     * freed once this function returns so callers who want to use the data  
+     * asynchronously must make sure to copy it first.  
+     */
+    @CalledByNative("AudioProcessing")
+    void process(int numBands, int numFrames, ByteBuffer buffer);
+  }
+
+  private long apmPtr;
+  private long capturePostProcessingPtr;
+  private long renderPreProcessingPtr;
+
+  public ExternalAudioProcessingFactory() {
+    apmPtr = nativeGetDefaultApm();
+    capturePostProcessingPtr = 0;
+    renderPreProcessingPtr = 0;
+  }
+
+  @Override
+  public long createNative() {
+    if(apmPtr == 0) {
+      apmPtr = nativeGetDefaultApm();
+    }
+    return apmPtr;
+  }
+
+  /**
+   * Sets the capture post processing module. 
+   * This module is applied to the audio signal after capture and before sending 
+   * to the audio encoder.
+   */
+  public void setCapturePostProcessing(@Nullable AudioProcessing processing) {
+    checkExternalAudioProcessorExists();
+    long newPtr = nativeSetCapturePostProcessing(processing);
+    if (capturePostProcessingPtr != 0) {
+      JniCommon.nativeReleaseRef(capturePostProcessingPtr);
+      capturePostProcessingPtr = 0;
+    }
+    capturePostProcessingPtr = newPtr;
+  }
+
+  /**
+   * Sets the render pre processing module.
+   * This module is applied to the audio signal after receiving from the audio
+   * decoder and before rendering.
+   */
+  public void setRenderPreProcessing(@Nullable AudioProcessing processing) {
+    checkExternalAudioProcessorExists();
+    long newPtr = nativeSetRenderPreProcessing(processing);
+    if (renderPreProcessingPtr != 0) {
+      JniCommon.nativeReleaseRef(renderPreProcessingPtr);
+      renderPreProcessingPtr = 0;
+    }
+    renderPreProcessingPtr = newPtr;
+  }
+
+  /**
+   * Sets the bypass flag for the capture post processing module.
+   * If true, the registered audio processing will be bypassed.
+   */
+  public void setBypassFlagForCapturePost( boolean bypass) {
+    checkExternalAudioProcessorExists();
+    nativeSetBypassFlagForCapturePost(bypass);
+  }
+
+  /**
+   * Sets the bypass flag for the render pre processing module.
+   * If true, the registered audio processing will be bypassed.
+   */
+  public void setBypassFlagForRenderPre( boolean bypass) {
+    checkExternalAudioProcessorExists();
+    nativeSetBypassFlagForRenderPre(bypass);
+  }
+
+  /**
+   * Destroys the ExternalAudioProcessor.
+   */
+  public void destroy() {
+    checkExternalAudioProcessorExists();
+    if (renderPreProcessingPtr != 0) {
+      JniCommon.nativeReleaseRef(renderPreProcessingPtr);
+      renderPreProcessingPtr = 0;
+    }
+    if (capturePostProcessingPtr != 0) {
+      JniCommon.nativeReleaseRef(capturePostProcessingPtr);
+      capturePostProcessingPtr = 0;
+    }
+    nativeDestroy();
+    apmPtr = 0;
+  }
+
+  private void checkExternalAudioProcessorExists() {
+    if (apmPtr == 0) {
+      throw new IllegalStateException("ExternalAudioProcessor has been disposed.");
+    }
+  }
+
+  private static native long nativeGetDefaultApm();
+  private static native long nativeSetCapturePostProcessing(AudioProcessing processing);
+  private static native long nativeSetRenderPreProcessing(AudioProcessing processing);
+  private static native void nativeSetBypassFlagForCapturePost(boolean bypass);
+  private static native void nativeSetBypassFlagForRenderPre(boolean bypass);
+  private static native void nativeDestroy();
+}

sdk/android/src/jni/pc/external_audio_processing_factory.cc

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2022 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "sdk/android/src/jni/pc/external_audio_processing_factory.h"
+
+#include <jni.h>
+#include <syslog.h>
+
+#include "api/make_ref_counted.h"
+#include "rtc_base/ref_counted_object.h"
+#include "sdk/android/generated_peerconnection_jni/ExternalAudioProcessingFactory_jni.h"
+#include "sdk/android/native_api/jni/java_types.h"
+#include "sdk/android/native_api/jni/scoped_java_ref.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/pc/external_audio_processor.h"
+
+namespace webrtc {
+namespace jni {
+
+ExternalAudioProcessingJni::ExternalAudioProcessingJni(
+    JNIEnv* jni,
+    const JavaRef<jobject>& j_processing)
+    : j_processing_global_(jni, j_processing) {}
+ExternalAudioProcessingJni::~ExternalAudioProcessingJni() {}
+void ExternalAudioProcessingJni::Initialize(int sample_rate_hz,
+                                            int num_channels) {
+  JNIEnv* env = AttachCurrentThreadIfNeeded();
+  Java_AudioProcessing_initialize(env, j_processing_global_, sample_rate_hz,
+                                  num_channels);
+}
+
+void ExternalAudioProcessingJni::Reset(int new_rate) {
+  JNIEnv* env = AttachCurrentThreadIfNeeded();
+  Java_AudioProcessing_reset(env, j_processing_global_, new_rate);
+}
+
+void ExternalAudioProcessingJni::Process(int num_bands, int num_frames, int buffer_size, float* buffer) {
+  JNIEnv* env = AttachCurrentThreadIfNeeded();
+  ScopedJavaLocalRef<jobject> audio_buffer =
+      NewDirectByteBuffer(env, (void*)buffer, buffer_size * sizeof(float));
+  Java_AudioProcessing_process(env, j_processing_global_, num_bands, num_frames, audio_buffer);
+}
+
+ExternalAudioProcessingFactory::ExternalAudioProcessingFactory() {
+  capture_post_processor_ = new ExternalAudioProcessor();
+  std::unique_ptr<webrtc::CustomProcessing> capture_post_processor(
+      capture_post_processor_);
+
+  render_pre_processor_ = new ExternalAudioProcessor();
+  std::unique_ptr<webrtc::CustomProcessing> render_pre_processor(
+      render_pre_processor_);
+
+  apm_ = webrtc::AudioProcessingBuilder()
+             .SetCapturePostProcessing(std::move(capture_post_processor))
+             .SetRenderPreProcessing(std::move(render_pre_processor))
+             .Create();
+
+  webrtc::AudioProcessing::Config config;
+  apm_->ApplyConfig(config);
+}
+
+static ExternalAudioProcessingFactory* default_processor_ptr;
+
+static jlong JNI_ExternalAudioProcessingFactory_GetDefaultApm(JNIEnv* env) {
+  if (!default_processor_ptr) {
+    auto default_processor = rtc::make_ref_counted<ExternalAudioProcessingFactory>();
+    default_processor_ptr = default_processor.release();
+  }
+  return webrtc::jni::jlongFromPointer(default_processor_ptr->apm().get());
+}
+
+static jlong JNI_ExternalAudioProcessingFactory_SetCapturePostProcessing(
+    JNIEnv* env,
+    const JavaParamRef<jobject>& j_processing) {
+  if (!default_processor_ptr) {
+    return 0;
+  }
+  auto processing =
+      rtc::make_ref_counted<ExternalAudioProcessingJni>(env, j_processing);
+  processing->AddRef();
+  default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing(
+      processing.get());
+  return jlongFromPointer(processing.get());
+}
+
+static jlong JNI_ExternalAudioProcessingFactory_SetRenderPreProcessing(
+    JNIEnv* env,
+    const JavaParamRef<jobject>& j_processing) {
+  if (!default_processor_ptr) {
+    return 0;
+  }
+  auto processing =
+      rtc::make_ref_counted<ExternalAudioProcessingJni>(env, j_processing);
+  processing->AddRef();
+  default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing(
+      processing.get());
+  return jlongFromPointer(processing.get());
+}
+
+static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForCapturePost(
+    JNIEnv* env,
+    jboolean bypass) {
+  if (!default_processor_ptr) {
+    return;
+  }
+  default_processor_ptr->capture_post_processor()->SetBypassFlag(bypass);
+}
+
+static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForRenderPre(
+    JNIEnv* env,
+    jboolean bypass) {
+  if (!default_processor_ptr) {
+    return;
+  }
+  default_processor_ptr->render_pre_processor()->SetBypassFlag(bypass);
+}
+
+static void JNI_ExternalAudioProcessingFactory_Destroy(JNIEnv* env) {
+  if (!default_processor_ptr) {
+    return;
+  }
+  default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing(
+      nullptr);
+  default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing(
+      nullptr);
+  delete default_processor_ptr;
+}
+
+}  // namespace jni
+}  // namespace webrtc