You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
823 lines
32 KiB
823 lines
32 KiB
//
|
|
// SCManagedVideoStreamer.m
|
|
// Snapchat
|
|
//
|
|
// Created by Liu Liu on 4/30/15.
|
|
// Copyright (c) 2015 Liu Liu. All rights reserved.
|
|
//
|
|
|
|
#import "SCManagedVideoStreamer.h"
|
|
|
|
#import "ARConfiguration+SCConfiguration.h"
|
|
#import "SCCameraTweaks.h"
|
|
#import "SCCapturerDefines.h"
|
|
#import "SCLogger+Camera.h"
|
|
#import "SCManagedCapturePreviewLayerController.h"
|
|
#import "SCMetalUtils.h"
|
|
#import "SCProcessingPipeline.h"
|
|
#import "SCProcessingPipelineBuilder.h"
|
|
|
|
#import <SCCameraFoundation/SCManagedVideoDataSourceListenerAnnouncer.h>
|
|
#import <SCFoundation/NSString+SCFormat.h>
|
|
#import <SCFoundation/SCLog.h>
|
|
#import <SCFoundation/SCQueuePerformer.h>
|
|
#import <SCFoundation/SCTrace.h>
|
|
#import <SCLogger/SCCameraMetrics.h>
|
|
|
|
#import <Looksery/Looksery.h>
|
|
|
|
#import <libkern/OSAtomic.h>
|
|
#import <stdatomic.h>
|
|
|
|
@import ARKit;
|
|
@import AVFoundation;
|
|
|
|
#define SCLogVideoStreamerInfo(fmt, ...) SCLogCoreCameraInfo(@"[SCManagedVideoStreamer] " fmt, ##__VA_ARGS__)
|
|
#define SCLogVideoStreamerWarning(fmt, ...) SCLogCoreCameraWarning(@"[SCManagedVideoStreamer] " fmt, ##__VA_ARGS__)
|
|
#define SCLogVideoStreamerError(fmt, ...) SCLogCoreCameraError(@"[SCManagedVideoStreamer] " fmt, ##__VA_ARGS__)
|
|
|
|
static NSInteger const kSCCaptureFrameRate = 30;
|
|
static CGFloat const kSCLogInterval = 3.0;
|
|
static char *const kSCManagedVideoStreamerQueueLabel = "com.snapchat.managed-video-streamer";
|
|
static char *const kSCManagedVideoStreamerCallbackQueueLabel = "com.snapchat.managed-video-streamer.dequeue";
|
|
static NSTimeInterval const kSCManagedVideoStreamerMaxAllowedLatency = 1; // Drop the frame if it is 1 second late.
|
|
|
|
static NSTimeInterval const kSCManagedVideoStreamerStalledDisplay =
|
|
5; // If the frame is not updated for 5 seconds, it is considered to be stalled.
|
|
|
|
static NSTimeInterval const kSCManagedVideoStreamerARSessionFramerateCap =
|
|
1.0 / (kSCCaptureFrameRate + 1); // Restrict ARSession to 30fps
|
|
static int32_t const kSCManagedVideoStreamerMaxProcessingBuffers = 15;
|
|
|
|
@interface SCManagedVideoStreamer () <AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureDepthDataOutputDelegate,
|
|
AVCaptureDataOutputSynchronizerDelegate, ARSessionDelegate>
|
|
|
|
@property (nonatomic, strong) AVCaptureSession *captureSession;
|
|
|
|
@end
|
|
|
|
@implementation SCManagedVideoStreamer {
|
|
AVCaptureVideoDataOutput *_videoDataOutput;
|
|
AVCaptureDepthDataOutput *_depthDataOutput NS_AVAILABLE_IOS(11_0);
|
|
AVCaptureDataOutputSynchronizer *_dataOutputSynchronizer NS_AVAILABLE_IOS(11_0);
|
|
BOOL _performingConfigurations;
|
|
SCManagedCaptureDevicePosition _devicePosition;
|
|
BOOL _videoStabilizationEnabledIfSupported;
|
|
SCManagedVideoDataSourceListenerAnnouncer *_announcer;
|
|
|
|
BOOL _sampleBufferDisplayEnabled;
|
|
id<SCManagedSampleBufferDisplayController> _sampleBufferDisplayController;
|
|
dispatch_block_t _flushOutdatedPreviewBlock;
|
|
NSMutableArray<NSArray *> *_waitUntilSampleBufferDisplayedBlocks;
|
|
SCProcessingPipeline *_processingPipeline;
|
|
|
|
NSTimeInterval _lastDisplayedFrameTimestamp;
|
|
#ifdef SC_USE_ARKIT_FACE
|
|
NSTimeInterval _lastDisplayedDepthFrameTimestamp;
|
|
#endif
|
|
|
|
BOOL _depthCaptureEnabled;
|
|
CGPoint _portraitModePointOfInterest;
|
|
|
|
// For sticky video tweaks
|
|
BOOL _keepLateFrames;
|
|
SCQueuePerformer *_callbackPerformer;
|
|
atomic_int _processingBuffersCount;
|
|
}
|
|
|
|
@synthesize isStreaming = _isStreaming;
|
|
@synthesize performer = _performer;
|
|
@synthesize currentFrame = _currentFrame;
|
|
@synthesize fieldOfView = _fieldOfView;
|
|
#ifdef SC_USE_ARKIT_FACE
|
|
@synthesize lastDepthData = _lastDepthData;
|
|
#endif
|
|
@synthesize videoOrientation = _videoOrientation;
|
|
|
|
- (instancetype)initWithSession:(AVCaptureSession *)session
|
|
devicePosition:(SCManagedCaptureDevicePosition)devicePosition
|
|
{
|
|
SCTraceStart();
|
|
self = [super init];
|
|
if (self) {
|
|
_sampleBufferDisplayEnabled = YES;
|
|
_announcer = [[SCManagedVideoDataSourceListenerAnnouncer alloc] init];
|
|
// We discard frames to support lenses in real time
|
|
_keepLateFrames = NO;
|
|
_performer = [[SCQueuePerformer alloc] initWithLabel:kSCManagedVideoStreamerQueueLabel
|
|
qualityOfService:QOS_CLASS_USER_INTERACTIVE
|
|
queueType:DISPATCH_QUEUE_SERIAL
|
|
context:SCQueuePerformerContextCamera];
|
|
|
|
_videoOrientation = AVCaptureVideoOrientationLandscapeRight;
|
|
|
|
[self setupWithSession:session devicePosition:devicePosition];
|
|
SCLogVideoStreamerInfo(@"init with position:%lu", (unsigned long)devicePosition);
|
|
}
|
|
return self;
|
|
}
|
|
|
|
- (instancetype)initWithSession:(AVCaptureSession *)session
|
|
arSession:(ARSession *)arSession
|
|
devicePosition:(SCManagedCaptureDevicePosition)devicePosition NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
self = [self initWithSession:session devicePosition:devicePosition];
|
|
if (self) {
|
|
[self setupWithARSession:arSession];
|
|
self.currentFrame = nil;
|
|
#ifdef SC_USE_ARKIT_FACE
|
|
self.lastDepthData = nil;
|
|
#endif
|
|
}
|
|
return self;
|
|
}
|
|
|
|
- (AVCaptureVideoDataOutput *)_newVideoDataOutput
|
|
{
|
|
AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init];
|
|
// All inbound frames are going to be the native format of the camera avoid
|
|
// any need for transcoding.
|
|
output.videoSettings =
|
|
@{(NSString *) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) };
|
|
return output;
|
|
}
|
|
|
|
- (void)setupWithSession:(AVCaptureSession *)session devicePosition:(SCManagedCaptureDevicePosition)devicePosition
|
|
{
|
|
[self stopStreaming];
|
|
self.captureSession = session;
|
|
_devicePosition = devicePosition;
|
|
|
|
_videoDataOutput = [self _newVideoDataOutput];
|
|
if (SCDeviceSupportsMetal()) {
|
|
// We default to start the streaming if the Metal is supported at startup time.
|
|
_isStreaming = YES;
|
|
// Set the sample buffer delegate before starting it.
|
|
[_videoDataOutput setSampleBufferDelegate:self queue:[self callbackPerformer].queue];
|
|
}
|
|
|
|
if ([session canAddOutput:_videoDataOutput]) {
|
|
[session addOutput:_videoDataOutput];
|
|
[self _enableVideoMirrorForDevicePosition:devicePosition];
|
|
}
|
|
|
|
if (SCCameraTweaksEnablePortraitModeButton()) {
|
|
if (@available(iOS 11.0, *)) {
|
|
_depthDataOutput = [[AVCaptureDepthDataOutput alloc] init];
|
|
[[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:NO];
|
|
if ([session canAddOutput:_depthDataOutput]) {
|
|
[session addOutput:_depthDataOutput];
|
|
[_depthDataOutput setDelegate:self callbackQueue:_performer.queue];
|
|
}
|
|
_depthCaptureEnabled = NO;
|
|
}
|
|
_portraitModePointOfInterest = CGPointMake(0.5, 0.5);
|
|
}
|
|
|
|
[self setVideoStabilizationEnabledIfSupported:YES];
|
|
}
|
|
|
|
- (void)setupWithARSession:(ARSession *)arSession NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
arSession.delegateQueue = _performer.queue;
|
|
arSession.delegate = self;
|
|
}
|
|
|
|
- (void)addSampleBufferDisplayController:(id<SCManagedSampleBufferDisplayController>)sampleBufferDisplayController
|
|
{
|
|
[_performer perform:^{
|
|
_sampleBufferDisplayController = sampleBufferDisplayController;
|
|
SCLogVideoStreamerInfo(@"add sampleBufferDisplayController:%@", _sampleBufferDisplayController);
|
|
}];
|
|
}
|
|
|
|
- (void)setSampleBufferDisplayEnabled:(BOOL)sampleBufferDisplayEnabled
|
|
{
|
|
[_performer perform:^{
|
|
_sampleBufferDisplayEnabled = sampleBufferDisplayEnabled;
|
|
SCLogVideoStreamerInfo(@"sampleBufferDisplayEnabled set to:%d", _sampleBufferDisplayEnabled);
|
|
}];
|
|
}
|
|
|
|
- (void)waitUntilSampleBufferDisplayed:(dispatch_queue_t)queue completionHandler:(dispatch_block_t)completionHandler
|
|
{
|
|
SCAssert(queue, @"callback queue must be provided");
|
|
SCAssert(completionHandler, @"completion handler must be provided");
|
|
SCLogVideoStreamerInfo(@"waitUntilSampleBufferDisplayed queue:%@ completionHandler:%p isStreaming:%d", queue,
|
|
completionHandler, _isStreaming);
|
|
if (_isStreaming) {
|
|
[_performer perform:^{
|
|
if (!_waitUntilSampleBufferDisplayedBlocks) {
|
|
_waitUntilSampleBufferDisplayedBlocks = [NSMutableArray array];
|
|
}
|
|
[_waitUntilSampleBufferDisplayedBlocks addObject:@[ queue, completionHandler ]];
|
|
SCLogVideoStreamerInfo(@"waitUntilSampleBufferDisplayed add block:%p", completionHandler);
|
|
}];
|
|
} else {
|
|
dispatch_async(queue, completionHandler);
|
|
}
|
|
}
|
|
|
|
- (void)startStreaming
|
|
{
|
|
SCTraceStart();
|
|
SCLogVideoStreamerInfo(@"start streaming. _isStreaming:%d", _isStreaming);
|
|
if (!_isStreaming) {
|
|
_isStreaming = YES;
|
|
[self _cancelFlushOutdatedPreview];
|
|
if (@available(ios 11.0, *)) {
|
|
if (_depthCaptureEnabled) {
|
|
[[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:YES];
|
|
}
|
|
}
|
|
[_videoDataOutput setSampleBufferDelegate:self queue:[self callbackPerformer].queue];
|
|
}
|
|
}
|
|
|
|
- (void)setAsOutput:(AVCaptureSession *)session devicePosition:(SCManagedCaptureDevicePosition)devicePosition
|
|
{
|
|
SCTraceStart();
|
|
if ([session canAddOutput:_videoDataOutput]) {
|
|
SCLogVideoStreamerError(@"add videoDataOutput:%@", _videoDataOutput);
|
|
[session addOutput:_videoDataOutput];
|
|
[self _enableVideoMirrorForDevicePosition:devicePosition];
|
|
} else {
|
|
SCLogVideoStreamerError(@"cannot add videoDataOutput:%@ to session:%@", _videoDataOutput, session);
|
|
}
|
|
[self _enableVideoStabilizationIfSupported];
|
|
}
|
|
|
|
- (void)removeAsOutput:(AVCaptureSession *)session
|
|
{
|
|
SCTraceStart();
|
|
SCLogVideoStreamerInfo(@"remove videoDataOutput:%@ from session:%@", _videoDataOutput, session);
|
|
[session removeOutput:_videoDataOutput];
|
|
}
|
|
|
|
- (void)_cancelFlushOutdatedPreview
|
|
{
|
|
SCLogVideoStreamerInfo(@"cancel flush outdated preview:%p", _flushOutdatedPreviewBlock);
|
|
if (_flushOutdatedPreviewBlock) {
|
|
dispatch_block_cancel(_flushOutdatedPreviewBlock);
|
|
_flushOutdatedPreviewBlock = nil;
|
|
}
|
|
}
|
|
|
|
- (SCQueuePerformer *)callbackPerformer
|
|
{
|
|
// If sticky video tweak is on, use a separated performer queue
|
|
if (_keepLateFrames) {
|
|
if (!_callbackPerformer) {
|
|
_callbackPerformer = [[SCQueuePerformer alloc] initWithLabel:kSCManagedVideoStreamerCallbackQueueLabel
|
|
qualityOfService:QOS_CLASS_USER_INTERACTIVE
|
|
queueType:DISPATCH_QUEUE_SERIAL
|
|
context:SCQueuePerformerContextCamera];
|
|
}
|
|
return _callbackPerformer;
|
|
}
|
|
return _performer;
|
|
}
|
|
|
|
- (void)pauseStreaming
|
|
{
|
|
SCTraceStart();
|
|
SCLogVideoStreamerInfo(@"pauseStreaming isStreaming:%d", _isStreaming);
|
|
if (_isStreaming) {
|
|
_isStreaming = NO;
|
|
[_videoDataOutput setSampleBufferDelegate:nil queue:NULL];
|
|
if (@available(ios 11.0, *)) {
|
|
if (_depthCaptureEnabled) {
|
|
[[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:NO];
|
|
}
|
|
}
|
|
@weakify(self);
|
|
_flushOutdatedPreviewBlock = dispatch_block_create(0, ^{
|
|
SCLogVideoStreamerInfo(@"execute flushOutdatedPreviewBlock");
|
|
@strongify(self);
|
|
SC_GUARD_ELSE_RETURN(self);
|
|
[self->_sampleBufferDisplayController flushOutdatedPreview];
|
|
});
|
|
[_performer perform:_flushOutdatedPreviewBlock
|
|
after:SCCameraTweaksEnableKeepLastFrameOnCamera() ? kSCManagedVideoStreamerStalledDisplay : 0];
|
|
[_performer perform:^{
|
|
[self _performCompletionHandlersForWaitUntilSampleBufferDisplayed];
|
|
}];
|
|
}
|
|
}
|
|
|
|
- (void)stopStreaming
|
|
{
|
|
SCTraceStart();
|
|
SCLogVideoStreamerInfo(@"stopStreaming isStreaming:%d", _isStreaming);
|
|
if (_isStreaming) {
|
|
_isStreaming = NO;
|
|
[_videoDataOutput setSampleBufferDelegate:nil queue:NULL];
|
|
if (@available(ios 11.0, *)) {
|
|
if (_depthCaptureEnabled) {
|
|
[[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:NO];
|
|
}
|
|
}
|
|
}
|
|
[self _cancelFlushOutdatedPreview];
|
|
[_performer perform:^{
|
|
SCLogVideoStreamerInfo(@"stopStreaming in perfome queue");
|
|
[_sampleBufferDisplayController flushOutdatedPreview];
|
|
[self _performCompletionHandlersForWaitUntilSampleBufferDisplayed];
|
|
}];
|
|
}
|
|
|
|
- (void)beginConfiguration
|
|
{
|
|
SCLogVideoStreamerInfo(@"enter beginConfiguration");
|
|
[_performer perform:^{
|
|
SCLogVideoStreamerInfo(@"performingConfigurations set to YES");
|
|
_performingConfigurations = YES;
|
|
}];
|
|
}
|
|
|
|
- (void)setDevicePosition:(SCManagedCaptureDevicePosition)devicePosition
|
|
{
|
|
SCLogVideoStreamerInfo(@"setDevicePosition with newPosition:%lu", (unsigned long)devicePosition);
|
|
[self _enableVideoMirrorForDevicePosition:devicePosition];
|
|
[self _enableVideoStabilizationIfSupported];
|
|
[_performer perform:^{
|
|
SCLogVideoStreamerInfo(@"setDevicePosition in perform queue oldPosition:%lu newPosition:%lu",
|
|
(unsigned long)_devicePosition, (unsigned long)devicePosition);
|
|
if (_devicePosition != devicePosition) {
|
|
_devicePosition = devicePosition;
|
|
}
|
|
}];
|
|
}
|
|
|
|
- (void)setVideoOrientation:(AVCaptureVideoOrientation)videoOrientation
|
|
{
|
|
SCTraceStart();
|
|
// It is not neccessary call these changes on private queue, because is is just only data output configuration.
|
|
// It should be called from manged capturer queue to prevent lock capture session in two different(private and
|
|
// managed capturer) queues that will cause the deadlock.
|
|
SCLogVideoStreamerInfo(@"setVideoOrientation oldOrientation:%lu newOrientation:%lu",
|
|
(unsigned long)_videoOrientation, (unsigned long)videoOrientation);
|
|
_videoOrientation = videoOrientation;
|
|
AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
|
|
connection.videoOrientation = _videoOrientation;
|
|
}
|
|
|
|
- (void)setKeepLateFrames:(BOOL)keepLateFrames
|
|
{
|
|
SCTraceStart();
|
|
[_performer perform:^{
|
|
SCTraceStart();
|
|
if (keepLateFrames != _keepLateFrames) {
|
|
_keepLateFrames = keepLateFrames;
|
|
// Get and set corresponding queue base on keepLateFrames.
|
|
// We don't use AVCaptureVideoDataOutput.alwaysDiscardsLateVideo anymore, because it will potentially
|
|
// result in lenses regression, and we could use all 15 sample buffers by adding a separated calllback
|
|
// queue.
|
|
[_videoDataOutput setSampleBufferDelegate:self queue:[self callbackPerformer].queue];
|
|
SCLogVideoStreamerInfo(@"keepLateFrames was set to:%d", keepLateFrames);
|
|
}
|
|
}];
|
|
}
|
|
|
|
- (void)setDepthCaptureEnabled:(BOOL)enabled NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
_depthCaptureEnabled = enabled;
|
|
[[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:enabled];
|
|
if (enabled) {
|
|
_dataOutputSynchronizer =
|
|
[[AVCaptureDataOutputSynchronizer alloc] initWithDataOutputs:@[ _videoDataOutput, _depthDataOutput ]];
|
|
[_dataOutputSynchronizer setDelegate:self queue:_performer.queue];
|
|
} else {
|
|
_dataOutputSynchronizer = nil;
|
|
}
|
|
}
|
|
|
|
- (void)setPortraitModePointOfInterest:(CGPoint)pointOfInterest
|
|
{
|
|
_portraitModePointOfInterest = pointOfInterest;
|
|
}
|
|
|
|
- (BOOL)getKeepLateFrames
|
|
{
|
|
return _keepLateFrames;
|
|
}
|
|
|
|
- (void)commitConfiguration
|
|
{
|
|
SCLogVideoStreamerInfo(@"enter commitConfiguration");
|
|
[_performer perform:^{
|
|
SCLogVideoStreamerInfo(@"performingConfigurations set to NO");
|
|
_performingConfigurations = NO;
|
|
}];
|
|
}
|
|
|
|
- (void)addListener:(id<SCManagedVideoDataSourceListener>)listener
|
|
{
|
|
SCTraceStart();
|
|
SCLogVideoStreamerInfo(@"add listener:%@", listener);
|
|
[_announcer addListener:listener];
|
|
}
|
|
|
|
- (void)removeListener:(id<SCManagedVideoDataSourceListener>)listener
|
|
{
|
|
SCTraceStart();
|
|
SCLogVideoStreamerInfo(@"remove listener:%@", listener);
|
|
[_announcer removeListener:listener];
|
|
}
|
|
|
|
- (void)addProcessingPipeline:(SCProcessingPipeline *)processingPipeline
|
|
{
|
|
SCLogVideoStreamerInfo(@"enter addProcessingPipeline:%@", processingPipeline);
|
|
[_performer perform:^{
|
|
SCLogVideoStreamerInfo(@"processingPipeline set to %@", processingPipeline);
|
|
_processingPipeline = processingPipeline;
|
|
}];
|
|
}
|
|
|
|
- (void)removeProcessingPipeline
|
|
{
|
|
SCLogVideoStreamerInfo(@"enter removeProcessingPipeline");
|
|
[_performer perform:^{
|
|
SCLogVideoStreamerInfo(@"processingPipeline set to nil");
|
|
_processingPipeline = nil;
|
|
}];
|
|
}
|
|
|
|
- (BOOL)isVideoMirrored
|
|
{
|
|
SCTraceStart();
|
|
AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
|
|
return connection.isVideoMirrored;
|
|
}
|
|
|
|
#pragma mark - Common Sample Buffer Handling
|
|
|
|
- (void)didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
|
{
|
|
return [self didOutputSampleBuffer:sampleBuffer depthData:nil];
|
|
}
|
|
|
|
- (void)didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer depthData:(CVPixelBufferRef)depthDataMap
|
|
{
|
|
// Don't send the sample buffer if we are perform configurations
|
|
if (_performingConfigurations) {
|
|
SCLogVideoStreamerError(@"didOutputSampleBuffer return because performingConfigurations is YES");
|
|
return;
|
|
}
|
|
SC_GUARD_ELSE_RETURN([_performer isCurrentPerformer]);
|
|
|
|
// We can't set alwaysDiscardsLateVideoFrames to YES when lens is activated because it will cause camera freezing.
|
|
// When alwaysDiscardsLateVideoFrames is set to NO, the late frames will not be dropped until it reach 15 frames,
|
|
// so we should simulate the dropping behaviour as AVFoundation do.
|
|
NSTimeInterval presentationTime = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer));
|
|
_lastDisplayedFrameTimestamp = presentationTime;
|
|
NSTimeInterval frameLatency = CACurrentMediaTime() - presentationTime;
|
|
// Log interval definied in macro LOG_INTERVAL, now is 3.0s
|
|
BOOL shouldLog =
|
|
(long)(CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * kSCCaptureFrameRate) %
|
|
((long)(kSCCaptureFrameRate * kSCLogInterval)) ==
|
|
0;
|
|
if (shouldLog) {
|
|
SCLogVideoStreamerInfo(@"didOutputSampleBuffer:%p", sampleBuffer);
|
|
}
|
|
if (_processingPipeline) {
|
|
RenderData renderData = {
|
|
.sampleBuffer = sampleBuffer,
|
|
.depthDataMap = depthDataMap,
|
|
.depthBlurPointOfInterest =
|
|
SCCameraTweaksEnablePortraitModeAutofocus() || SCCameraTweaksEnablePortraitModeTapToFocus()
|
|
? &_portraitModePointOfInterest
|
|
: nil,
|
|
};
|
|
// Ensure we are doing all render operations (i.e. accessing textures) on performer to prevent race condition
|
|
SCAssertPerformer(_performer);
|
|
sampleBuffer = [_processingPipeline render:renderData];
|
|
|
|
if (shouldLog) {
|
|
SCLogVideoStreamerInfo(@"rendered sampleBuffer:%p in processingPipeline:%@", sampleBuffer,
|
|
_processingPipeline);
|
|
}
|
|
}
|
|
|
|
if (sampleBuffer && _sampleBufferDisplayEnabled) {
|
|
// Send the buffer only if it is valid, set it to be displayed immediately (See the enqueueSampleBuffer method
|
|
// header, need to get attachments array and set the dictionary).
|
|
CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
|
|
if (!attachmentsArray) {
|
|
SCLogVideoStreamerError(@"Error getting attachment array for CMSampleBuffer");
|
|
} else if (CFArrayGetCount(attachmentsArray) > 0) {
|
|
CFMutableDictionaryRef attachment = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachmentsArray, 0);
|
|
CFDictionarySetValue(attachment, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
|
|
}
|
|
// Warn if frame that went through is not most recent enough.
|
|
if (frameLatency >= kSCManagedVideoStreamerMaxAllowedLatency) {
|
|
SCLogVideoStreamerWarning(
|
|
@"The sample buffer we received is too late, why? presentationTime:%lf frameLatency:%f",
|
|
presentationTime, frameLatency);
|
|
}
|
|
[_sampleBufferDisplayController enqueueSampleBuffer:sampleBuffer];
|
|
if (shouldLog) {
|
|
SCLogVideoStreamerInfo(@"displayed sampleBuffer:%p in Metal", sampleBuffer);
|
|
}
|
|
|
|
[self _performCompletionHandlersForWaitUntilSampleBufferDisplayed];
|
|
}
|
|
|
|
if (shouldLog) {
|
|
SCLogVideoStreamerInfo(@"begin annoucing sampleBuffer:%p of devicePosition:%lu", sampleBuffer,
|
|
(unsigned long)_devicePosition);
|
|
}
|
|
[_announcer managedVideoDataSource:self didOutputSampleBuffer:sampleBuffer devicePosition:_devicePosition];
|
|
if (shouldLog) {
|
|
SCLogVideoStreamerInfo(@"end annoucing sampleBuffer:%p", sampleBuffer);
|
|
}
|
|
}
|
|
|
|
- (void)didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
|
{
|
|
if (_performingConfigurations) {
|
|
return;
|
|
}
|
|
SC_GUARD_ELSE_RETURN([_performer isCurrentPerformer]);
|
|
NSTimeInterval currentProcessingTime = CACurrentMediaTime();
|
|
NSTimeInterval currentSampleTime = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer));
|
|
// Only logging it when sticky tweak is on, which means sticky time is too long, and AVFoundation have to drop the
|
|
// sampleBuffer
|
|
if (_keepLateFrames) {
|
|
SCLogVideoStreamerInfo(@"didDropSampleBuffer:%p timestamp:%f latency:%f", sampleBuffer, currentProcessingTime,
|
|
currentSampleTime);
|
|
}
|
|
[_announcer managedVideoDataSource:self didDropSampleBuffer:sampleBuffer devicePosition:_devicePosition];
|
|
}
|
|
|
|
#pragma mark - AVCaptureVideoDataOutputSampleBufferDelegate
|
|
|
|
- (void)captureOutput:(AVCaptureOutput *)captureOutput
|
|
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
|
fromConnection:(AVCaptureConnection *)connection NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
// Sticky video tweak is off, i.e. lenses is on,
|
|
// we use same queue for callback and processing, and let AVFoundation decide which frame should be dropped
|
|
if (!_keepLateFrames) {
|
|
[self didOutputSampleBuffer:sampleBuffer];
|
|
}
|
|
// Sticky video tweak is on
|
|
else {
|
|
if ([_performer isCurrentPerformer]) {
|
|
// Note: there might be one frame callbacked in processing queue when switching callback queue,
|
|
// it should be fine. But if following log appears too much, it is not our design.
|
|
SCLogVideoStreamerWarning(@"The callback queue should be a separated queue when sticky tweak is on");
|
|
}
|
|
// TODO: In sticky video v2, we should consider check free memory
|
|
if (_processingBuffersCount >= kSCManagedVideoStreamerMaxProcessingBuffers - 1) {
|
|
SCLogVideoStreamerWarning(@"processingBuffersCount reached to the max. current count:%d",
|
|
_processingBuffersCount);
|
|
[self didDropSampleBuffer:sampleBuffer];
|
|
return;
|
|
}
|
|
atomic_fetch_add(&_processingBuffersCount, 1);
|
|
CFRetain(sampleBuffer);
|
|
// _performer should always be the processing queue
|
|
[_performer perform:^{
|
|
[self didOutputSampleBuffer:sampleBuffer];
|
|
CFRelease(sampleBuffer);
|
|
atomic_fetch_sub(&_processingBuffersCount, 1);
|
|
}];
|
|
}
|
|
}
|
|
|
|
- (void)captureOutput:(AVCaptureOutput *)captureOutput
|
|
didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
|
fromConnection:(AVCaptureConnection *)connection
|
|
{
|
|
[self didDropSampleBuffer:sampleBuffer];
|
|
}
|
|
|
|
#pragma mark - AVCaptureDataOutputSynchronizer (Video + Depth)
|
|
|
|
- (void)dataOutputSynchronizer:(AVCaptureDataOutputSynchronizer *)synchronizer
|
|
didOutputSynchronizedDataCollection:(AVCaptureSynchronizedDataCollection *)synchronizedDataCollection
|
|
NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
AVCaptureSynchronizedDepthData *syncedDepthData = (AVCaptureSynchronizedDepthData *)[synchronizedDataCollection
|
|
synchronizedDataForCaptureOutput:_depthDataOutput];
|
|
AVDepthData *depthData = nil;
|
|
if (syncedDepthData && !syncedDepthData.depthDataWasDropped) {
|
|
depthData = syncedDepthData.depthData;
|
|
}
|
|
|
|
AVCaptureSynchronizedSampleBufferData *syncedVideoData =
|
|
(AVCaptureSynchronizedSampleBufferData *)[synchronizedDataCollection
|
|
synchronizedDataForCaptureOutput:_videoDataOutput];
|
|
if (syncedVideoData && !syncedVideoData.sampleBufferWasDropped) {
|
|
CMSampleBufferRef videoSampleBuffer = syncedVideoData.sampleBuffer;
|
|
[self didOutputSampleBuffer:videoSampleBuffer depthData:depthData ? depthData.depthDataMap : nil];
|
|
}
|
|
}
|
|
|
|
#pragma mark - ARSessionDelegate
|
|
|
|
- (void)session:(ARSession *)session cameraDidChangeTrackingState:(ARCamera *)camera NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
NSString *state = nil;
|
|
NSString *reason = nil;
|
|
switch (camera.trackingState) {
|
|
case ARTrackingStateNormal:
|
|
state = @"Normal";
|
|
break;
|
|
case ARTrackingStateLimited:
|
|
state = @"Limited";
|
|
break;
|
|
case ARTrackingStateNotAvailable:
|
|
state = @"Not Available";
|
|
break;
|
|
}
|
|
switch (camera.trackingStateReason) {
|
|
case ARTrackingStateReasonNone:
|
|
reason = @"None";
|
|
break;
|
|
case ARTrackingStateReasonInitializing:
|
|
reason = @"Initializing";
|
|
break;
|
|
case ARTrackingStateReasonExcessiveMotion:
|
|
reason = @"Excessive Motion";
|
|
break;
|
|
case ARTrackingStateReasonInsufficientFeatures:
|
|
reason = @"Insufficient Features";
|
|
break;
|
|
#if SC_AT_LEAST_SDK_11_3
|
|
case ARTrackingStateReasonRelocalizing:
|
|
reason = @"Relocalizing";
|
|
break;
|
|
#endif
|
|
}
|
|
SCLogVideoStreamerInfo(@"ARKit changed tracking state - %@ (reason: %@)", state, reason);
|
|
}
|
|
|
|
- (void)session:(ARSession *)session didUpdateFrame:(ARFrame *)frame NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
#ifdef SC_USE_ARKIT_FACE
|
|
// This is extremely weird, but LOOK-10251 indicates that despite the class having it defined, on some specific
|
|
// devices there are ARFrame instances that don't respond to `capturedDepthData`.
|
|
// (note: this was discovered to be due to some people staying on iOS 11 betas).
|
|
AVDepthData *depth = nil;
|
|
if ([frame respondsToSelector:@selector(capturedDepthData)]) {
|
|
depth = frame.capturedDepthData;
|
|
}
|
|
#endif
|
|
|
|
CGFloat timeSince = frame.timestamp - _lastDisplayedFrameTimestamp;
|
|
// Don't deliver more than 30 frames per sec
|
|
BOOL framerateMinimumElapsed = timeSince >= kSCManagedVideoStreamerARSessionFramerateCap;
|
|
|
|
#ifdef SC_USE_ARKIT_FACE
|
|
if (depth) {
|
|
CGFloat timeSince = frame.timestamp - _lastDisplayedDepthFrameTimestamp;
|
|
framerateMinimumElapsed |= timeSince >= kSCManagedVideoStreamerARSessionFramerateCap;
|
|
}
|
|
|
|
#endif
|
|
|
|
SC_GUARD_ELSE_RETURN(framerateMinimumElapsed);
|
|
|
|
#ifdef SC_USE_ARKIT_FACE
|
|
if (depth) {
|
|
self.lastDepthData = depth;
|
|
_lastDisplayedDepthFrameTimestamp = frame.timestamp;
|
|
}
|
|
#endif
|
|
|
|
// Make sure that current frame is no longer being used, otherwise drop current frame.
|
|
SC_GUARD_ELSE_RETURN(self.currentFrame == nil);
|
|
|
|
CVPixelBufferRef pixelBuffer = frame.capturedImage;
|
|
CVPixelBufferLockBaseAddress(pixelBuffer, 0);
|
|
CMTime time = CMTimeMakeWithSeconds(frame.timestamp, 1000000);
|
|
CMSampleTimingInfo timing = {kCMTimeInvalid, time, kCMTimeInvalid};
|
|
|
|
CMVideoFormatDescriptionRef videoInfo;
|
|
CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &videoInfo);
|
|
|
|
CMSampleBufferRef buffer;
|
|
CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, YES, nil, nil, videoInfo, &timing, &buffer);
|
|
CFRelease(videoInfo);
|
|
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
|
|
|
|
self.currentFrame = frame;
|
|
[self didOutputSampleBuffer:buffer];
|
|
[self _updateFieldOfViewWithARFrame:frame];
|
|
|
|
CFRelease(buffer);
|
|
}
|
|
|
|
- (void)session:(ARSession *)session didAddAnchors:(NSArray<ARAnchor *> *)anchors NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
for (ARAnchor *anchor in anchors) {
|
|
if ([anchor isKindOfClass:[ARPlaneAnchor class]]) {
|
|
SCLogVideoStreamerInfo(@"ARKit added plane anchor");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
- (void)session:(ARSession *)session didFailWithError:(NSError *)error NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
SCLogVideoStreamerError(@"ARKit session failed with error: %@. Resetting", error);
|
|
[session runWithConfiguration:[ARConfiguration sc_configurationForDevicePosition:_devicePosition]];
|
|
}
|
|
|
|
- (void)sessionWasInterrupted:(ARSession *)session NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
SCLogVideoStreamerWarning(@"ARKit session interrupted");
|
|
}
|
|
|
|
- (void)sessionInterruptionEnded:(ARSession *)session NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
SCLogVideoStreamerInfo(@"ARKit interruption ended");
|
|
}
|
|
|
|
#pragma mark - Private methods
|
|
|
|
- (void)_performCompletionHandlersForWaitUntilSampleBufferDisplayed
|
|
{
|
|
for (NSArray *completion in _waitUntilSampleBufferDisplayedBlocks) {
|
|
// Call the completion handlers.
|
|
dispatch_async(completion[0], completion[1]);
|
|
}
|
|
[_waitUntilSampleBufferDisplayedBlocks removeAllObjects];
|
|
}
|
|
|
|
// This is the magic that ensures the VideoDataOutput will have the correct
|
|
// orientation.
|
|
- (void)_enableVideoMirrorForDevicePosition:(SCManagedCaptureDevicePosition)devicePosition
|
|
{
|
|
SCLogVideoStreamerInfo(@"enable video mirror for device position:%lu", (unsigned long)devicePosition);
|
|
AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
|
|
connection.videoOrientation = _videoOrientation;
|
|
if (devicePosition == SCManagedCaptureDevicePositionFront) {
|
|
connection.videoMirrored = YES;
|
|
}
|
|
}
|
|
|
|
- (void)_enableVideoStabilizationIfSupported
|
|
{
|
|
SCTraceStart();
|
|
if (!SCCameraTweaksEnableVideoStabilization()) {
|
|
SCLogVideoStreamerWarning(@"SCCameraTweaksEnableVideoStabilization is NO, won't enable video stabilization");
|
|
return;
|
|
}
|
|
|
|
AVCaptureConnection *videoConnection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
|
|
if (!videoConnection) {
|
|
SCLogVideoStreamerError(@"cannot get videoConnection from videoDataOutput:%@", videoConnection);
|
|
return;
|
|
}
|
|
// Set the video stabilization mode to auto. Default is off.
|
|
if ([videoConnection isVideoStabilizationSupported]) {
|
|
videoConnection.preferredVideoStabilizationMode = _videoStabilizationEnabledIfSupported
|
|
? AVCaptureVideoStabilizationModeStandard
|
|
: AVCaptureVideoStabilizationModeOff;
|
|
NSDictionary *params = @{ @"iOS8_Mode" : @(videoConnection.activeVideoStabilizationMode) };
|
|
[[SCLogger sharedInstance] logEvent:@"VIDEO_STABILIZATION_MODE" parameters:params];
|
|
SCLogVideoStreamerInfo(@"set video stabilization mode:%ld to videoConnection:%@",
|
|
(long)videoConnection.preferredVideoStabilizationMode, videoConnection);
|
|
} else {
|
|
SCLogVideoStreamerInfo(@"video stabilization isn't supported on videoConnection:%@", videoConnection);
|
|
}
|
|
}
|
|
|
|
- (void)setVideoStabilizationEnabledIfSupported:(BOOL)videoStabilizationIfSupported
|
|
{
|
|
SCLogVideoStreamerInfo(@"setVideoStabilizationEnabledIfSupported:%d", videoStabilizationIfSupported);
|
|
_videoStabilizationEnabledIfSupported = videoStabilizationIfSupported;
|
|
[self _enableVideoStabilizationIfSupported];
|
|
}
|
|
|
|
- (void)_updateFieldOfViewWithARFrame:(ARFrame *)frame NS_AVAILABLE_IOS(11_0)
|
|
{
|
|
SC_GUARD_ELSE_RETURN(frame.camera);
|
|
CGSize imageResolution = frame.camera.imageResolution;
|
|
matrix_float3x3 intrinsics = frame.camera.intrinsics;
|
|
float xFovDegrees = 2 * atan(imageResolution.width / (2 * intrinsics.columns[0][0])) * 180 / M_PI;
|
|
if (_fieldOfView != xFovDegrees) {
|
|
self.fieldOfView = xFovDegrees;
|
|
}
|
|
}
|
|
|
|
- (NSString *)description
|
|
{
|
|
return [self debugDescription];
|
|
}
|
|
|
|
- (NSString *)debugDescription
|
|
{
|
|
NSDictionary *debugDict = @{
|
|
@"_sampleBufferDisplayEnabled" : _sampleBufferDisplayEnabled ? @"Yes" : @"No",
|
|
@"_videoStabilizationEnabledIfSupported" : _videoStabilizationEnabledIfSupported ? @"Yes" : @"No",
|
|
@"_performingConfigurations" : _performingConfigurations ? @"Yes" : @"No",
|
|
@"alwaysDiscardLateVideoFrames" : _videoDataOutput.alwaysDiscardsLateVideoFrames ? @"Yes" : @"No"
|
|
};
|
|
return [NSString sc_stringWithFormat:@"%@", debugDict];
|
|
}
|
|
|
|
@end
|