Skip to content

Commit 3dbf83e

Browse files
kanathiroshihorie
authored andcommitted
Expose remote audio sample buffers on RTCAudioTrack (#84) (#30)
* audio renderer protocol * basic set up * progress * update header year * impl * stereo * fail gracefully * minor fix * doc * optimize AudioStreamBasicDescription * logging * minor refactoring * weak reference to delegates * fix timestamp computation * change swift delegate signature * use os_unfair_lock instead * avoid deadlock Co-authored-by: Hiroshi Horie <[email protected]>
1 parent 2a00cfc commit 3dbf83e

File tree

5 files changed

+264
-6
lines changed

5 files changed

+264
-6
lines changed

sdk/BUILD.gn

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ if (is_ios || is_mac) {
119119
"objc/base/RTCVideoFrame.mm",
120120
"objc/base/RTCVideoFrameBuffer.h",
121121
"objc/base/RTCVideoRenderer.h",
122+
"objc/base/RTCAudioRenderer.h",
122123
"objc/base/RTCYUVPlanarBuffer.h",
123124
]
124125

@@ -1297,6 +1298,7 @@ if (is_ios || is_mac) {
12971298
"objc/base/RTCVideoFrame.h",
12981299
"objc/base/RTCVideoFrameBuffer.h",
12991300
"objc/base/RTCVideoRenderer.h",
1301+
"objc/base/RTCAudioRenderer.h",
13001302
"objc/base/RTCYUVPlanarBuffer.h",
13011303
"objc/components/audio/RTCAudioDevice.h",
13021304
"objc/components/audio/RTCAudioSession.h",
@@ -1506,6 +1508,7 @@ if (is_ios || is_mac) {
15061508
"objc/base/RTCVideoFrame.h",
15071509
"objc/base/RTCVideoFrameBuffer.h",
15081510
"objc/base/RTCVideoRenderer.h",
1511+
"objc/base/RTCAudioRenderer.h",
15091512
"objc/base/RTCYUVPlanarBuffer.h",
15101513
"objc/components/capturer/RTCCameraVideoCapturer.h",
15111514
"objc/components/capturer/RTCFileVideoCapturer.h",

sdk/objc/api/peerconnection/RTCAudioTrack+Private.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,26 @@
88
* be found in the AUTHORS file in the root of the source tree.
99
*/
1010

11+
#import <AVFoundation/AVFoundation.h>
1112
#import "RTCAudioTrack.h"
1213

1314
#include "api/media_stream_interface.h"
1415

1516
NS_ASSUME_NONNULL_BEGIN
1617

1718
@class RTC_OBJC_TYPE(RTCPeerConnectionFactory);
18-
@interface RTC_OBJC_TYPE (RTCAudioTrack)
19-
()
19+
@interface RTC_OBJC_TYPE (RTCAudioTrack) ()
2020

21-
/** AudioTrackInterface created or passed in at construction. */
22-
@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::AudioTrackInterface> nativeAudioTrack;
21+
/** AudioTrackInterface created or passed in at construction. */
22+
@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::AudioTrackInterface> nativeAudioTrack;
2323

2424
/** Initialize an RTCAudioTrack with an id. */
2525
- (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory
2626
source:(RTC_OBJC_TYPE(RTCAudioSource) *)source
2727
trackId:(NSString *)trackId;
2828

29+
- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer;
30+
2931
@end
3032

3133
NS_ASSUME_NONNULL_END

sdk/objc/api/peerconnection/RTCAudioTrack.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
NS_ASSUME_NONNULL_BEGIN
1515

16+
@protocol RTC_OBJC_TYPE (RTCAudioRenderer);
1617
@class RTC_OBJC_TYPE(RTCAudioSource);
1718

1819
RTC_OBJC_EXPORT
@@ -23,6 +24,13 @@ RTC_OBJC_EXPORT
2324
/** The audio source for this audio track. */
2425
@property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source;
2526

27+
/** Register a renderer that will receive all audio CMSampleBuffers on this track.
28+
* Does not retain. */
29+
- (void)addRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer;
30+
31+
/** Deregister a renderer */
32+
- (void)removeRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer;
33+
2634
@end
2735

2836
NS_ASSUME_NONNULL_END

sdk/objc/api/peerconnection/RTCAudioTrack.mm

Lines changed: 213 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,180 @@
88
* be found in the AUTHORS file in the root of the source tree.
99
*/
1010

11+
#import <AVFoundation/AVFoundation.h>
12+
#import <os/lock.h>
13+
1114
#import "RTCAudioTrack+Private.h"
1215

16+
#import "RTCAudioRenderer.h"
1317
#import "RTCAudioSource+Private.h"
1418
#import "RTCMediaStreamTrack+Private.h"
1519
#import "RTCPeerConnectionFactory+Private.h"
1620
#import "helpers/NSString+StdString.h"
1721

1822
#include "rtc_base/checks.h"
1923

20-
@implementation RTC_OBJC_TYPE (RTCAudioTrack)
24+
namespace webrtc {
25+
/**
26+
* Captures audio data and converts to CMSampleBuffers
27+
*/
28+
class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface {
29+
private:
30+
os_unfair_lock *lock_;
31+
__weak RTCAudioTrack *audio_track_;
32+
int64_t total_frames_ = 0;
33+
bool attached_ = false;
34+
35+
public:
36+
AudioSinkConverter(RTCAudioTrack *audioTrack, os_unfair_lock *lock) {
37+
RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter init";
38+
audio_track_ = audioTrack;
39+
lock_ = lock;
40+
}
41+
42+
~AudioSinkConverter() {
43+
//
44+
RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter dealloc";
45+
}
46+
47+
// Must be called while locked
48+
void TryAttach() {
49+
if (attached_) {
50+
// Already attached
51+
return;
52+
}
53+
RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink...";
54+
// Reset for creating CMSampleTimingInfo correctly
55+
audio_track_.nativeAudioTrack->AddSink(this);
56+
total_frames_ = 0;
57+
attached_ = true;
58+
}
59+
60+
// Must be called while locked
61+
void TryDetach() {
62+
if (!attached_) {
63+
// Already detached
64+
return;
65+
}
66+
RTC_LOG(LS_INFO) << "RTCAudioTrack detaching sink...";
67+
audio_track_.nativeAudioTrack->RemoveSink(this);
68+
attached_ = false;
69+
}
70+
71+
void OnData(const void *audio_data,
72+
int bits_per_sample,
73+
int sample_rate,
74+
size_t number_of_channels,
75+
size_t number_of_frames,
76+
absl::optional<int64_t> absolute_capture_timestamp_ms) override {
77+
RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData bits_per_sample: "
78+
<< bits_per_sample << " sample_rate: " << sample_rate
79+
<< " number_of_channels: " << number_of_channels
80+
<< " number_of_frames: " << number_of_frames
81+
<< " absolute_capture_timestamp_ms: "
82+
<< (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0);
83+
84+
bool is_locked = os_unfair_lock_trylock(lock_);
85+
if (!is_locked) {
86+
RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already locked, skipping...";
87+
return;
88+
}
89+
bool is_attached = attached_;
90+
os_unfair_lock_unlock(lock_);
91+
92+
if (!is_attached) {
93+
RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already detached, skipping...";
94+
return;
95+
}
96+
97+
/*
98+
* Convert to CMSampleBuffer
99+
*/
100+
101+
if (!(number_of_channels == 1 || number_of_channels == 2)) {
102+
NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu",
103+
number_of_channels);
104+
return;
105+
}
106+
107+
OSStatus status;
108+
109+
AudioChannelLayout acl;
110+
bzero(&acl, sizeof(acl));
111+
acl.mChannelLayoutTag =
112+
number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono;
113+
114+
AudioStreamBasicDescription sd;
115+
sd.mSampleRate = sample_rate;
116+
sd.mFormatID = kAudioFormatLinearPCM;
117+
sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked;
118+
sd.mFramesPerPacket = 1;
119+
sd.mChannelsPerFrame = number_of_channels;
120+
sd.mBitsPerChannel = bits_per_sample; /* 16 */
121+
sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8);
122+
sd.mBytesPerPacket = sd.mBytesPerFrame;
123+
124+
CMSampleTimingInfo timing = {
125+
CMTimeMake(1, sample_rate),
126+
CMTimeMake(total_frames_, sample_rate),
127+
kCMTimeInvalid,
128+
};
129+
130+
total_frames_ += number_of_frames; // update the total
131+
132+
CMFormatDescriptionRef format = NULL;
133+
status = CMAudioFormatDescriptionCreate(
134+
kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, NULL, &format);
135+
136+
if (status != 0) {
137+
NSLog(@"RTCAudioTrack: Failed to create audio format description");
138+
return;
139+
}
140+
141+
CMSampleBufferRef buffer;
142+
status = CMSampleBufferCreate(kCFAllocatorDefault,
143+
NULL,
144+
false,
145+
NULL,
146+
NULL,
147+
format,
148+
(CMItemCount)number_of_frames,
149+
1,
150+
&timing,
151+
0,
152+
NULL,
153+
&buffer);
154+
if (status != 0) {
155+
NSLog(@"RTCAudioTrack: Failed to allocate sample buffer");
156+
return;
157+
}
158+
159+
AudioBufferList bufferList;
160+
bufferList.mNumberBuffers = 1;
161+
bufferList.mBuffers[0].mNumberChannels = sd.mChannelsPerFrame;
162+
bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * sd.mBytesPerFrame);
163+
bufferList.mBuffers[0].mData = (void *)audio_data;
164+
status = CMSampleBufferSetDataBufferFromAudioBufferList(
165+
buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList);
166+
if (status != 0) {
167+
NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer");
168+
return;
169+
}
170+
171+
// Report back to RTCAudioTrack
172+
[audio_track_ didCaptureSampleBuffer:buffer];
173+
174+
CFRelease(buffer);
175+
}
176+
};
177+
} // namespace webrtc
178+
179+
@implementation RTC_OBJC_TYPE (RTCAudioTrack) {
180+
rtc::scoped_refptr<webrtc::AudioSinkConverter> _audioConverter;
181+
// Stores weak references to renderers
182+
NSHashTable *_renderers;
183+
os_unfair_lock _lock;
184+
}
21185

22186
@synthesize source = _source;
23187

@@ -43,7 +207,21 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto
43207
NSParameterAssert(factory);
44208
NSParameterAssert(nativeTrack);
45209
NSParameterAssert(type == RTCMediaStreamTrackTypeAudio);
46-
return [super initWithFactory:factory nativeTrack:nativeTrack type:type];
210+
if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) {
211+
RTC_LOG(LS_INFO) << "RTCAudioTrack init";
212+
_renderers = [NSHashTable weakObjectsHashTable];
213+
_audioConverter = new rtc::RefCountedObject<webrtc::AudioSinkConverter>(self, &_lock);
214+
}
215+
216+
return self;
217+
}
218+
219+
- (void)dealloc {
220+
os_unfair_lock_lock(&_lock);
221+
_audioConverter->TryDetach();
222+
os_unfair_lock_unlock(&_lock);
223+
224+
RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc";
47225
}
48226

49227
- (RTC_OBJC_TYPE(RTCAudioSource) *)source {
@@ -57,11 +235,44 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto
57235
return _source;
58236
}
59237

238+
- (void)addRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer {
239+
os_unfair_lock_lock(&_lock);
240+
[_renderers addObject:renderer];
241+
_audioConverter->TryAttach();
242+
os_unfair_lock_unlock(&_lock);
243+
}
244+
245+
- (void)removeRenderer:(id<RTC_OBJC_TYPE(RTCAudioRenderer)>)renderer {
246+
os_unfair_lock_lock(&_lock);
247+
[_renderers removeObject:renderer];
248+
NSUInteger renderersCount = _renderers.allObjects.count;
249+
250+
if (renderersCount == 0) {
251+
// Detach if no more renderers...
252+
_audioConverter->TryDetach();
253+
}
254+
os_unfair_lock_unlock(&_lock);
255+
}
256+
60257
#pragma mark - Private
61258

62259
- (rtc::scoped_refptr<webrtc::AudioTrackInterface>)nativeAudioTrack {
63260
return rtc::scoped_refptr<webrtc::AudioTrackInterface>(
64261
static_cast<webrtc::AudioTrackInterface *>(self.nativeTrack.get()));
65262
}
66263

264+
- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer {
265+
bool is_locked = os_unfair_lock_trylock(&_lock);
266+
if (!is_locked) {
267+
RTC_LOG(LS_INFO) << "RTCAudioTrack didCaptureSampleBuffer already locked, skipping...";
268+
return;
269+
}
270+
NSArray *renderers = [_renderers allObjects];
271+
os_unfair_lock_unlock(&_lock);
272+
273+
for (id<RTCAudioRenderer> renderer in renderers) {
274+
[renderer renderSampleBuffer:sampleBuffer];
275+
}
276+
}
277+
67278
@end

sdk/objc/base/RTCAudioRenderer.h

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright 2023 LiveKit
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#import <Foundation/Foundation.h>
18+
#if TARGET_OS_IPHONE
19+
#import <UIKit/UIKit.h>
20+
#endif
21+
22+
#import "RTCMacros.h"
23+
24+
NS_ASSUME_NONNULL_BEGIN
25+
26+
RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE
27+
(RTCAudioRenderer)<NSObject>
28+
29+
- (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer
30+
NS_SWIFT_NAME(render(sampleBuffer:));
31+
32+
@end
33+
34+
NS_ASSUME_NONNULL_END

0 commit comments

Comments
 (0)