Merge pull request #4 from webrtc-sdk/ios-simulcast

Simulcast support for iOS SDK

Author: cloudwebrtc

sdk/BUILD.gn

@@ -741,6 +741,7 @@ if (is_ios || is_mac) {
       ]
 
       deps = [
+        ":simulcast",
         ":base_objc",
         ":native_video",
         ":videocodec_objc",
@@ -842,6 +843,22 @@ if (is_ios || is_mac) {
       ]
     }
 
+    rtc_library("simulcast") {
+      sources = [
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
+      ]
+
+      deps = [
+        ":base_objc",
+        ":wrapped_native_codec_objc",
+        "../media:rtc_media_base",
+        "../media:rtc_simulcast_encoder_adapter",
+      ]
+    }
+  
     rtc_library("mediaconstraints_objc") {
       configs += [ "..:no_global_constructors" ]
       sources = [
@@ -1304,6 +1321,7 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoDecoderH264.h",
           "objc/components/video_codec/RTCVideoEncoderFactoryH264.h",
           "objc/components/video_codec/RTCVideoEncoderH264.h",
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCCameraPreviewView.h",
           "objc/helpers/RTCDispatcher.h",
@@ -1351,6 +1369,7 @@ if (is_ios || is_mac) {
           "objc/api/video_codec/RTCVideoEncoderVP8.h",
           "objc/api/video_codec/RTCVideoEncoderVP9.h",
           "objc/api/video_codec/RTCVideoEncoderAV1.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
           "objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
           "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
         ]

sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h

@@ -0,0 +1,13 @@
+#import "RTCMacros.h"
+#import "RTCVideoEncoder.h"
+#import "RTCVideoEncoderFactory.h"
+#import "RTCVideoCodecInfo.h"
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) : NSObject
+
++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                                                         fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
+                                                   videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo;
+
+@end

sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm

@@ -0,0 +1,26 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoEncoderSimulcast.h"
+#import "RTCWrappedNativeVideoEncoder.h"
+#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
+
+#include "native/api/video_encoder_factory.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+
+@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast)
+
++ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                                                         fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
+                                                   videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo {
+    auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(primary);
+    auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(fallback);
+    auto nativeFormat = [videoCodecInfo nativeSdpVideoFormat];
+    return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc]
+        initWithNativeEncoder: std::make_unique<webrtc::SimulcastEncoderAdapter>(
+            nativePrimary.release(),
+            nativeFallback.release(),
+            std::move(nativeFormat))];
+}
+
+@end

sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h

@@ -0,0 +1,16 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoEncoderFactory.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)>
+
+- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                       fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm

@@ -0,0 +1,39 @@
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoCodecInfo.h"
+#import "RTCVideoEncoderFactorySimulcast.h"
+#import "api/video_codec/RTCVideoEncoderSimulcast.h"
+
+@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) ()
+
+@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> primary;
+@property id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> fallback;
+
+@end
+
+
+@implementation RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast)
+
+@synthesize primary = _primary;
+@synthesize fallback = _fallback;
+
+- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
+                       fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback {
+    if (self = [super init]) {
+        _primary = primary;
+        _fallback = fallback;
+    }
+    return self;
+}
+
+- (nullable id<RTC_OBJC_TYPE(RTCVideoEncoder)>)createEncoder: (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info {
+    return [RTCVideoEncoderSimulcast simulcastEncoderWithPrimary: _primary fallback: _fallback videoCodecInfo: info];
+}
+
+- (NSArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *)supportedCodecs {
+    return [[_primary supportedCodecs] arrayByAddingObjectsFromArray: [_fallback supportedCodecs]];
+}
+
+
+@end

sdk/objc/native/src/audio/audio_device_ios.mm

@@ -61,16 +61,6 @@
 const UInt16 kFixedPlayoutDelayEstimate = 30;
 const UInt16 kFixedRecordDelayEstimate = 30;
 
-enum AudioDeviceMessageType : uint32_t {
-  kMessageTypeInterruptionBegin,
-  kMessageTypeInterruptionEnd,
-  kMessageTypeValidRouteChange,
-  kMessageTypeCanPlayOrRecordChange,
-  kMessageTypePlayoutGlitchDetected,
-  kMessageOutputVolumeChange,
-  kMessageTypeAudioWillRecord,
-};
-
 using ios::CheckAndLogError;
 
 #if !defined(NDEBUG)
@@ -372,7 +362,7 @@ static void LogDeviceInfo() {
 
 void AudioDeviceIOS::OnAudioWillRecord() {
   RTC_DCHECK(thread_);
-  thread_->Post(RTC_FROM_HERE, this, kMessageTypeAudioWillRecord);
+  thread_->PostTask(SafeTask(safety_, [this] { HandleAudioWillRecord(); }));
 }
 
 OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
@@ -482,34 +472,6 @@ static void LogDeviceInfo() {
   return noErr;
 }
 
-void AudioDeviceIOS::OnMessage(rtc::Message* msg) {
-  switch (msg->message_id) {
-    case kMessageTypeInterruptionBegin:
-      HandleInterruptionBegin();
-      break;
-    case kMessageTypeInterruptionEnd:
-      HandleInterruptionEnd();
-      break;
-    case kMessageTypeValidRouteChange:
-      HandleValidRouteChange();
-      break;
-    case kMessageTypeCanPlayOrRecordChange: {
-      rtc::TypedMessageData<bool>* data = static_cast<rtc::TypedMessageData<bool>*>(msg->pdata);
-      HandleCanPlayOrRecordChange(data->data());
-      delete data;
-      break;
-    }
-    case kMessageTypePlayoutGlitchDetected:
-      HandlePlayoutGlitchDetected();
-      break;
-    case kMessageOutputVolumeChange:
-      HandleOutputVolumeChange();
-      break;
-    case kMessageTypeAudioWillRecord:
-      HandleAudioWillRecord();
-  }
-}
-
 void AudioDeviceIOS::HandleInterruptionBegin() {
   RTC_DCHECK_RUN_ON(thread_);
   RTCLog(@"Interruption begin. IsInterrupted changed from %d to 1.", is_interrupted_);
@@ -677,7 +639,7 @@ static void LogDeviceInfo() {
 }
 
 void AudioDeviceIOS::HandleAudioWillRecord() {
-  RTC_DCHECK_RUN_ON(&thread_checker_);
+  RTC_DCHECK_RUN_ON(&io_thread_checker_);
 
   LOGI() << "HandleAudioWillRecord";