Allow custom audio processing by exposing AudioProcessingModule

sdk/BUILD.gn

@@ -119,7 +119,6 @@ if (is_ios || is_mac) {
       "objc/base/RTCVideoFrame.mm",
       "objc/base/RTCVideoFrameBuffer.h",
       "objc/base/RTCVideoRenderer.h",
-      "objc/base/RTCAudioRenderer.h",
       "objc/base/RTCYUVPlanarBuffer.h",
     ]
 
@@ -1050,6 +1049,20 @@ if (is_ios || is_mac) {
         "objc/api/peerconnection/RTCVideoTrack+Private.h",
         "objc/api/peerconnection/RTCVideoTrack.h",
         "objc/api/peerconnection/RTCVideoTrack.mm",
+        "objc/components/audio/RTCAudioBuffer.h",
+        "objc/components/audio/RTCAudioBuffer.mm",
+        "objc/components/audio/RTCAudioBuffer+Private.h",
+        "objc/components/audio/RTCAudioCustomProcessingAdapter.h",
+        "objc/components/audio/RTCAudioCustomProcessingAdapter.mm",
+        "objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h",
+        "objc/components/audio/RTCAudioCustomProcessingDelegate.h",
+        "objc/components/audio/RTCAudioProcessingConfig.h",
+        "objc/components/audio/RTCAudioProcessingConfig.mm",
+        "objc/components/audio/RTCAudioProcessingConfig+Private.h",
+        "objc/components/audio/RTCAudioProcessingModule.h",
+        "objc/components/audio/RTCDefaultAudioProcessingModule.h",
+        "objc/components/audio/RTCDefaultAudioProcessingModule.mm",
+        "objc/components/audio/RTCDefaultAudioProcessingModule+Private.h",
       ]
 
       configs += [
@@ -1363,6 +1376,11 @@ if (is_ios || is_mac) {
           # Added for Simulcast support
           "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
           "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
+          "objc/components/audio/RTCAudioBuffer.h",
+          "objc/components/audio/RTCAudioCustomProcessingDelegate.h",
+          "objc/components/audio/RTCAudioProcessingConfig.h",
+          "objc/components/audio/RTCAudioProcessingModule.h",
+          "objc/components/audio/RTCDefaultAudioProcessingModule.h",
         ]
 
         if (!build_with_chromium) {
@@ -1513,6 +1531,11 @@ if (is_ios || is_mac) {
           # Added for Simulcast support
           "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
           "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
+          "objc/components/audio/RTCAudioBuffer.h",
+          "objc/components/audio/RTCAudioCustomProcessingDelegate.h",
+          "objc/components/audio/RTCAudioProcessingConfig.h",
+          "objc/components/audio/RTCAudioProcessingModule.h",
+          "objc/components/audio/RTCDefaultAudioProcessingModule.h",
         ]
         if (!build_with_chromium) {
           sources += [

sdk/objc/api/peerconnection/RTCAudioTrack.mm

@@ -209,6 +209,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto
   NSParameterAssert(type == RTCMediaStreamTrackTypeAudio);
   if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) {
     RTC_LOG(LS_INFO) << "RTCAudioTrack init";
+    _lock = OS_UNFAIR_LOCK_INIT;
     _renderers = [NSHashTable weakObjectsHashTable];
     _audioConverter = new rtc::RefCountedObject<webrtc::AudioSinkConverter>(self, &_lock);
   }

sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h

@@ -73,12 +73,6 @@ NS_ASSUME_NONNULL_BEGIN
     initWithEncoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
             decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
 
-- (instancetype)
-    initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
-                   encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
-
 /** Initialize an RTCPeerConnection with a configuration, constraints, and
  *  dependencies.
  */

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h

@@ -34,6 +34,8 @@ NS_ASSUME_NONNULL_BEGIN
 (RTCSSLCertificateVerifier);
 @protocol RTC_OBJC_TYPE
 (RTCAudioDevice);
+@protocol RTC_OBJC_TYPE
+(RTCAudioProcessingModule);
 
 RTC_OBJC_EXPORT
 @interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) : NSObject
@@ -56,8 +58,9 @@ RTC_OBJC_EXPORT
 - (instancetype)
     initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
                    encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory;
+                   decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioProcessingModule:
+                (nullable id<RTC_OBJC_TYPE(RTCAudioProcessingModule)>)audioProcessingModule;
 
 @property(nonatomic, readonly) RTCAudioDeviceModule *audioDeviceModule;
 

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm

@@ -50,6 +50,9 @@
 #include "sdk/objc/native/src/objc_video_decoder_factory.h"
 #include "sdk/objc/native/src/objc_video_encoder_factory.h"
 
+#import "components/audio/RTCAudioProcessingModule.h"
+#import "components/audio/RTCDefaultAudioProcessingModule+Private.h"
+
 #if defined(WEBRTC_IOS)
 #import "sdk/objc/native/api/audio_device_module.h"
 #endif
@@ -59,6 +62,7 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) {
   std::unique_ptr<rtc::Thread> _workerThread;
   std::unique_ptr<rtc::Thread> _signalingThread;
   rtc::scoped_refptr<webrtc::AudioDeviceModule> _nativeAudioDeviceModule;
+  RTCDefaultAudioProcessingModule *_defaultAudioProcessingModule;
 
   BOOL _hasStartedAecDump;
 }
@@ -127,8 +131,9 @@ - (instancetype)init {
 - (instancetype)
     initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
                    encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory
-                   decoderFactory:
-                       (nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory {
+                   decoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoDecoderFactory)>)decoderFactory
+            audioProcessingModule:
+                (nullable id<RTC_OBJC_TYPE(RTCAudioProcessingModule)>)audioProcessingModule{
 #ifdef HAVE_NO_MEDIA
   return [self initWithNoMedia];
 #else

sdk/objc/components/audio/RTCAudioBuffer+Private.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioBuffer.h"
+
+#include "modules/audio_processing/audio_buffer.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTC_OBJC_TYPE (RTCAudioBuffer)()
+
+- (instancetype)initWithNativeType: (webrtc::AudioBuffer *) audioBuffer;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioBuffer.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCAudioBuffer) : NSObject
+
+@property(nonatomic, readonly) size_t channels;
+@property(nonatomic, readonly) size_t frames;
+@property(nonatomic, readonly) size_t framesPerBand;
+@property(nonatomic, readonly) size_t bands;
+
+// Returns pointer arrays. Index range from 0 to `frames`.
+- (float* _Nonnull)rawBufferForChannel:(size_t)channel;
+
+// TODO: More convenience methods...
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioBuffer.mm

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioBuffer.h"
+
+#include "modules/audio_processing/audio_buffer.h"
+
+@implementation RTC_OBJC_TYPE (RTCAudioBuffer) {
+  // Raw
+  webrtc::AudioBuffer *_audioBuffer;
+}
+
+- (size_t)channels {
+  return _audioBuffer->num_channels();
+}
+
+- (size_t)frames {
+  return _audioBuffer->num_frames();
+}
+
+- (size_t)framesPerBand {
+  return _audioBuffer->num_frames_per_band();
+}
+
+- (size_t)bands {
+  return _audioBuffer->num_bands();
+}
+
+- (float *)rawBufferForChannel:(size_t)channel {
+  return _audioBuffer->channels()[channel];
+}
+
+#pragma mark - Private
+
+- (instancetype)initWithNativeType:(webrtc::AudioBuffer *)audioBuffer {
+  if (self = [super init]) {
+    _audioBuffer = audioBuffer;
+  }
+  return self;
+}
+
+@end

sdk/objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "RTCAudioCustomProcessingAdapter.h"
+#import "RTCAudioCustomProcessingDelegate.h"
+#import "RTCMacros.h"
+
+#include "modules/audio_processing/include/audio_processing.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTC_OBJC_TYPE(RTCAudioCustomProcessingAdapter) ()
+
+// Thread safe set/get with os_unfair_lock.
+@property(nonatomic, weak, nullable) id<RTC_OBJC_TYPE(RTCAudioCustomProcessingDelegate)>
+    audioCustomProcessingDelegate;
+
+// Direct read access without lock.
+@property(nonatomic, readonly, weak, nullable) id<RTC_OBJC_TYPE(RTCAudioCustomProcessingDelegate)>
+    rawAudioCustomProcessingDelegate;
+
+@property(nonatomic, readonly) std::unique_ptr<webrtc::CustomProcessing>
+    nativeAudioCustomProcessingModule;
+
+- (instancetype)initWithDelegate:
+    (nullable id<RTC_OBJC_TYPE(RTCAudioCustomProcessingDelegate)>)audioCustomProcessingDelegate;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTC_OBJC_TYPE(RTCAudioCustomProcessingAdapter) : NSObject
+
+- (instancetype)init NS_UNAVAILABLE;
+
+@end
+
+NS_ASSUME_NONNULL_END