snapchat/ManagedCapturer/SCManagedVideoStreamer.m

//
//  SCManagedVideoStreamer.m
//  Snapchat
//
//  Created by Liu Liu on 4/30/15.
//  Copyright (c) 2015 Liu Liu. All rights reserved.
//

#import "SCManagedVideoStreamer.h"

#import "ARConfiguration+SCConfiguration.h"
#import "SCCameraTweaks.h"
#import "SCCapturerDefines.h"
#import "SCLogger+Camera.h"
#import "SCManagedCapturePreviewLayerController.h"
#import "SCMetalUtils.h"
#import "SCProcessingPipeline.h"
#import "SCProcessingPipelineBuilder.h"

#import <SCCameraFoundation/SCManagedVideoDataSourceListenerAnnouncer.h>
#import <SCFoundation/NSString+SCFormat.h>
#import <SCFoundation/SCLog.h>
#import <SCFoundation/SCQueuePerformer.h>
#import <SCFoundation/SCTrace.h>
#import <SCLogger/SCCameraMetrics.h>

#import <Looksery/Looksery.h>

#import <libkern/OSAtomic.h>
#import <stdatomic.h>

@import ARKit;
@import AVFoundation;

#define SCLogVideoStreamerInfo(fmt, ...) SCLogCoreCameraInfo(@"[SCManagedVideoStreamer] " fmt, ##__VA_ARGS__)
#define SCLogVideoStreamerWarning(fmt, ...) SCLogCoreCameraWarning(@"[SCManagedVideoStreamer] " fmt, ##__VA_ARGS__)
#define SCLogVideoStreamerError(fmt, ...) SCLogCoreCameraError(@"[SCManagedVideoStreamer] " fmt, ##__VA_ARGS__)

static NSInteger const kSCCaptureFrameRate = 30;
static CGFloat const kSCLogInterval = 3.0;
static char *const kSCManagedVideoStreamerQueueLabel = "com.snapchat.managed-video-streamer";
static char *const kSCManagedVideoStreamerCallbackQueueLabel = "com.snapchat.managed-video-streamer.dequeue";
static NSTimeInterval const kSCManagedVideoStreamerMaxAllowedLatency = 1; // Drop the frame if it is 1 second late.

static NSTimeInterval const kSCManagedVideoStreamerStalledDisplay =
    5; // If the frame is not updated for 5 seconds, it is considered to be stalled.

static NSTimeInterval const kSCManagedVideoStreamerARSessionFramerateCap =
    1.0 / (kSCCaptureFrameRate + 1); // Restrict ARSession to 30fps
static int32_t const kSCManagedVideoStreamerMaxProcessingBuffers = 15;

@interface SCManagedVideoStreamer () <AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureDepthDataOutputDelegate,
                                      AVCaptureDataOutputSynchronizerDelegate, ARSessionDelegate>

@property (nonatomic, strong) AVCaptureSession *captureSession;

@end

@implementation SCManagedVideoStreamer {
    AVCaptureVideoDataOutput *_videoDataOutput;
    AVCaptureDepthDataOutput *_depthDataOutput NS_AVAILABLE_IOS(11_0);
    AVCaptureDataOutputSynchronizer *_dataOutputSynchronizer NS_AVAILABLE_IOS(11_0);
    BOOL _performingConfigurations;
    SCManagedCaptureDevicePosition _devicePosition;
    BOOL _videoStabilizationEnabledIfSupported;
    SCManagedVideoDataSourceListenerAnnouncer *_announcer;

    BOOL _sampleBufferDisplayEnabled;
    id<SCManagedSampleBufferDisplayController> _sampleBufferDisplayController;
    dispatch_block_t _flushOutdatedPreviewBlock;
    NSMutableArray<NSArray *> *_waitUntilSampleBufferDisplayedBlocks;
    SCProcessingPipeline *_processingPipeline;

    NSTimeInterval _lastDisplayedFrameTimestamp;
#ifdef SC_USE_ARKIT_FACE
    NSTimeInterval _lastDisplayedDepthFrameTimestamp;
#endif

    BOOL _depthCaptureEnabled;
    CGPoint _portraitModePointOfInterest;

    // For sticky video tweaks
    BOOL _keepLateFrames;
    SCQueuePerformer *_callbackPerformer;
    atomic_int _processingBuffersCount;
}

@synthesize isStreaming = _isStreaming;
@synthesize performer = _performer;
@synthesize currentFrame = _currentFrame;
@synthesize fieldOfView = _fieldOfView;
#ifdef SC_USE_ARKIT_FACE
@synthesize lastDepthData = _lastDepthData;
#endif
@synthesize videoOrientation = _videoOrientation;

- (instancetype)initWithSession:(AVCaptureSession *)session
                 devicePosition:(SCManagedCaptureDevicePosition)devicePosition
{
    SCTraceStart();
    self = [super init];
    if (self) {
        _sampleBufferDisplayEnabled = YES;
        _announcer = [[SCManagedVideoDataSourceListenerAnnouncer alloc] init];
        // We discard frames to support lenses in real time
        _keepLateFrames = NO;
        _performer = [[SCQueuePerformer alloc] initWithLabel:kSCManagedVideoStreamerQueueLabel
                                            qualityOfService:QOS_CLASS_USER_INTERACTIVE
                                                   queueType:DISPATCH_QUEUE_SERIAL
                                                     context:SCQueuePerformerContextCamera];

        _videoOrientation = AVCaptureVideoOrientationLandscapeRight;

        [self setupWithSession:session devicePosition:devicePosition];
        SCLogVideoStreamerInfo(@"init with position:%lu", (unsigned long)devicePosition);
    }
    return self;
}

- (instancetype)initWithSession:(AVCaptureSession *)session
                      arSession:(ARSession *)arSession
                 devicePosition:(SCManagedCaptureDevicePosition)devicePosition NS_AVAILABLE_IOS(11_0)
{
    self = [self initWithSession:session devicePosition:devicePosition];
    if (self) {
        [self setupWithARSession:arSession];
        self.currentFrame = nil;
#ifdef SC_USE_ARKIT_FACE
        self.lastDepthData = nil;
#endif
    }
    return self;
}

- (AVCaptureVideoDataOutput *)_newVideoDataOutput
{
    AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init];
    // All inbound frames are going to be the native format of the camera avoid
    // any need for transcoding.
    output.videoSettings =
        @{(NSString *) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) };
    return output;
}

- (void)setupWithSession:(AVCaptureSession *)session devicePosition:(SCManagedCaptureDevicePosition)devicePosition
{
    [self stopStreaming];
    self.captureSession = session;
    _devicePosition = devicePosition;

    _videoDataOutput = [self _newVideoDataOutput];
    if (SCDeviceSupportsMetal()) {
        // We default to start the streaming if the Metal is supported at startup time.
        _isStreaming = YES;
        // Set the sample buffer delegate before starting it.
        [_videoDataOutput setSampleBufferDelegate:self queue:[self callbackPerformer].queue];
    }

    if ([session canAddOutput:_videoDataOutput]) {
        [session addOutput:_videoDataOutput];
        [self _enableVideoMirrorForDevicePosition:devicePosition];
    }

    if (SCCameraTweaksEnablePortraitModeButton()) {
        if (@available(iOS 11.0, *)) {
            _depthDataOutput = [[AVCaptureDepthDataOutput alloc] init];
            [[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:NO];
            if ([session canAddOutput:_depthDataOutput]) {
                [session addOutput:_depthDataOutput];
                [_depthDataOutput setDelegate:self callbackQueue:_performer.queue];
            }
            _depthCaptureEnabled = NO;
        }
        _portraitModePointOfInterest = CGPointMake(0.5, 0.5);
    }

    [self setVideoStabilizationEnabledIfSupported:YES];
}

- (void)setupWithARSession:(ARSession *)arSession NS_AVAILABLE_IOS(11_0)
{
    arSession.delegateQueue = _performer.queue;
    arSession.delegate = self;
}

- (void)addSampleBufferDisplayController:(id<SCManagedSampleBufferDisplayController>)sampleBufferDisplayController
{
    [_performer perform:^{
        _sampleBufferDisplayController = sampleBufferDisplayController;
        SCLogVideoStreamerInfo(@"add sampleBufferDisplayController:%@", _sampleBufferDisplayController);
    }];
}

- (void)setSampleBufferDisplayEnabled:(BOOL)sampleBufferDisplayEnabled
{
    [_performer perform:^{
        _sampleBufferDisplayEnabled = sampleBufferDisplayEnabled;
        SCLogVideoStreamerInfo(@"sampleBufferDisplayEnabled set to:%d", _sampleBufferDisplayEnabled);
    }];
}

- (void)waitUntilSampleBufferDisplayed:(dispatch_queue_t)queue completionHandler:(dispatch_block_t)completionHandler
{
    SCAssert(queue, @"callback queue must be provided");
    SCAssert(completionHandler, @"completion handler must be provided");
    SCLogVideoStreamerInfo(@"waitUntilSampleBufferDisplayed queue:%@ completionHandler:%p isStreaming:%d", queue,
                           completionHandler, _isStreaming);
    if (_isStreaming) {
        [_performer perform:^{
            if (!_waitUntilSampleBufferDisplayedBlocks) {
                _waitUntilSampleBufferDisplayedBlocks = [NSMutableArray array];
            }
            [_waitUntilSampleBufferDisplayedBlocks addObject:@[ queue, completionHandler ]];
            SCLogVideoStreamerInfo(@"waitUntilSampleBufferDisplayed add block:%p", completionHandler);
        }];
    } else {
        dispatch_async(queue, completionHandler);
    }
}

- (void)startStreaming
{
    SCTraceStart();
    SCLogVideoStreamerInfo(@"start streaming. _isStreaming:%d", _isStreaming);
    if (!_isStreaming) {
        _isStreaming = YES;
        [self _cancelFlushOutdatedPreview];
        if (@available(ios 11.0, *)) {
            if (_depthCaptureEnabled) {
                [[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:YES];
            }
        }
        [_videoDataOutput setSampleBufferDelegate:self queue:[self callbackPerformer].queue];
    }
}

- (void)setAsOutput:(AVCaptureSession *)session devicePosition:(SCManagedCaptureDevicePosition)devicePosition
{
    SCTraceStart();
    if ([session canAddOutput:_videoDataOutput]) {
        SCLogVideoStreamerError(@"add videoDataOutput:%@", _videoDataOutput);
        [session addOutput:_videoDataOutput];
        [self _enableVideoMirrorForDevicePosition:devicePosition];
    } else {
        SCLogVideoStreamerError(@"cannot add videoDataOutput:%@ to session:%@", _videoDataOutput, session);
    }
    [self _enableVideoStabilizationIfSupported];
}

- (void)removeAsOutput:(AVCaptureSession *)session
{
    SCTraceStart();
    SCLogVideoStreamerInfo(@"remove videoDataOutput:%@ from session:%@", _videoDataOutput, session);
    [session removeOutput:_videoDataOutput];
}

- (void)_cancelFlushOutdatedPreview
{
    SCLogVideoStreamerInfo(@"cancel flush outdated preview:%p", _flushOutdatedPreviewBlock);
    if (_flushOutdatedPreviewBlock) {
        dispatch_block_cancel(_flushOutdatedPreviewBlock);
        _flushOutdatedPreviewBlock = nil;
    }
}

- (SCQueuePerformer *)callbackPerformer
{
    // If sticky video tweak is on, use a separated performer queue
    if (_keepLateFrames) {
        if (!_callbackPerformer) {
            _callbackPerformer = [[SCQueuePerformer alloc] initWithLabel:kSCManagedVideoStreamerCallbackQueueLabel
                                                        qualityOfService:QOS_CLASS_USER_INTERACTIVE
                                                               queueType:DISPATCH_QUEUE_SERIAL
                                                                 context:SCQueuePerformerContextCamera];
        }
        return _callbackPerformer;
    }
    return _performer;
}

- (void)pauseStreaming
{
    SCTraceStart();
    SCLogVideoStreamerInfo(@"pauseStreaming isStreaming:%d", _isStreaming);
    if (_isStreaming) {
        _isStreaming = NO;
        [_videoDataOutput setSampleBufferDelegate:nil queue:NULL];
        if (@available(ios 11.0, *)) {
            if (_depthCaptureEnabled) {
                [[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:NO];
            }
        }
        @weakify(self);
        _flushOutdatedPreviewBlock = dispatch_block_create(0, ^{
            SCLogVideoStreamerInfo(@"execute flushOutdatedPreviewBlock");
            @strongify(self);
            SC_GUARD_ELSE_RETURN(self);
            [self->_sampleBufferDisplayController flushOutdatedPreview];
        });
        [_performer perform:_flushOutdatedPreviewBlock
                      after:SCCameraTweaksEnableKeepLastFrameOnCamera() ? kSCManagedVideoStreamerStalledDisplay : 0];
        [_performer perform:^{
            [self _performCompletionHandlersForWaitUntilSampleBufferDisplayed];
        }];
    }
}

- (void)stopStreaming
{
    SCTraceStart();
    SCLogVideoStreamerInfo(@"stopStreaming isStreaming:%d", _isStreaming);
    if (_isStreaming) {
        _isStreaming = NO;
        [_videoDataOutput setSampleBufferDelegate:nil queue:NULL];
        if (@available(ios 11.0, *)) {
            if (_depthCaptureEnabled) {
                [[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:NO];
            }
        }
    }
    [self _cancelFlushOutdatedPreview];
    [_performer perform:^{
        SCLogVideoStreamerInfo(@"stopStreaming in perfome queue");
        [_sampleBufferDisplayController flushOutdatedPreview];
        [self _performCompletionHandlersForWaitUntilSampleBufferDisplayed];
    }];
}

- (void)beginConfiguration
{
    SCLogVideoStreamerInfo(@"enter beginConfiguration");
    [_performer perform:^{
        SCLogVideoStreamerInfo(@"performingConfigurations set to YES");
        _performingConfigurations = YES;
    }];
}

- (void)setDevicePosition:(SCManagedCaptureDevicePosition)devicePosition
{
    SCLogVideoStreamerInfo(@"setDevicePosition with newPosition:%lu", (unsigned long)devicePosition);
    [self _enableVideoMirrorForDevicePosition:devicePosition];
    [self _enableVideoStabilizationIfSupported];
    [_performer perform:^{
        SCLogVideoStreamerInfo(@"setDevicePosition in perform queue oldPosition:%lu newPosition:%lu",
                               (unsigned long)_devicePosition, (unsigned long)devicePosition);
        if (_devicePosition != devicePosition) {
            _devicePosition = devicePosition;
        }
    }];
}

- (void)setVideoOrientation:(AVCaptureVideoOrientation)videoOrientation
{
    SCTraceStart();
    // It is not neccessary call these changes on private queue, because is is just only data output configuration.
    // It should be called from manged capturer queue to prevent lock capture session in two different(private and
    // managed capturer) queues that will cause the deadlock.
    SCLogVideoStreamerInfo(@"setVideoOrientation oldOrientation:%lu newOrientation:%lu",
                           (unsigned long)_videoOrientation, (unsigned long)videoOrientation);
    _videoOrientation = videoOrientation;
    AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
    connection.videoOrientation = _videoOrientation;
}

- (void)setKeepLateFrames:(BOOL)keepLateFrames
{
    SCTraceStart();
    [_performer perform:^{
        SCTraceStart();
        if (keepLateFrames != _keepLateFrames) {
            _keepLateFrames = keepLateFrames;
            // Get and set corresponding queue base on keepLateFrames.
            // We don't use AVCaptureVideoDataOutput.alwaysDiscardsLateVideo anymore, because it will potentially
            // result in lenses regression, and we could use all 15 sample buffers by adding a separated calllback
            // queue.
            [_videoDataOutput setSampleBufferDelegate:self queue:[self callbackPerformer].queue];
            SCLogVideoStreamerInfo(@"keepLateFrames was set to:%d", keepLateFrames);
        }
    }];
}

- (void)setDepthCaptureEnabled:(BOOL)enabled NS_AVAILABLE_IOS(11_0)
{
    _depthCaptureEnabled = enabled;
    [[_depthDataOutput connectionWithMediaType:AVMediaTypeDepthData] setEnabled:enabled];
    if (enabled) {
        _dataOutputSynchronizer =
            [[AVCaptureDataOutputSynchronizer alloc] initWithDataOutputs:@[ _videoDataOutput, _depthDataOutput ]];
        [_dataOutputSynchronizer setDelegate:self queue:_performer.queue];
    } else {
        _dataOutputSynchronizer = nil;
    }
}

- (void)setPortraitModePointOfInterest:(CGPoint)pointOfInterest
{
    _portraitModePointOfInterest = pointOfInterest;
}

- (BOOL)getKeepLateFrames
{
    return _keepLateFrames;
}

- (void)commitConfiguration
{
    SCLogVideoStreamerInfo(@"enter commitConfiguration");
    [_performer perform:^{
        SCLogVideoStreamerInfo(@"performingConfigurations set to NO");
        _performingConfigurations = NO;
    }];
}

- (void)addListener:(id<SCManagedVideoDataSourceListener>)listener
{
    SCTraceStart();
    SCLogVideoStreamerInfo(@"add listener:%@", listener);
    [_announcer addListener:listener];
}

- (void)removeListener:(id<SCManagedVideoDataSourceListener>)listener
{
    SCTraceStart();
    SCLogVideoStreamerInfo(@"remove listener:%@", listener);
    [_announcer removeListener:listener];
}

- (void)addProcessingPipeline:(SCProcessingPipeline *)processingPipeline
{
    SCLogVideoStreamerInfo(@"enter addProcessingPipeline:%@", processingPipeline);
    [_performer perform:^{
        SCLogVideoStreamerInfo(@"processingPipeline set to %@", processingPipeline);
        _processingPipeline = processingPipeline;
    }];
}

- (void)removeProcessingPipeline
{
    SCLogVideoStreamerInfo(@"enter removeProcessingPipeline");
    [_performer perform:^{
        SCLogVideoStreamerInfo(@"processingPipeline set to nil");
        _processingPipeline = nil;
    }];
}

- (BOOL)isVideoMirrored
{
    SCTraceStart();
    AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
    return connection.isVideoMirrored;
}

#pragma mark - Common Sample Buffer Handling

- (void)didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
{
    return [self didOutputSampleBuffer:sampleBuffer depthData:nil];
}

- (void)didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer depthData:(CVPixelBufferRef)depthDataMap
{
    // Don't send the sample buffer if we are perform configurations
    if (_performingConfigurations) {
        SCLogVideoStreamerError(@"didOutputSampleBuffer return because performingConfigurations is YES");
        return;
    }
    SC_GUARD_ELSE_RETURN([_performer isCurrentPerformer]);

    // We can't set alwaysDiscardsLateVideoFrames to YES when lens is activated because it will cause camera freezing.
    // When alwaysDiscardsLateVideoFrames is set to NO, the late frames will not be dropped until it reach 15 frames,
    // so we should simulate the dropping behaviour as AVFoundation do.
    NSTimeInterval presentationTime = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer));
    _lastDisplayedFrameTimestamp = presentationTime;
    NSTimeInterval frameLatency = CACurrentMediaTime() - presentationTime;
    // Log interval definied in macro LOG_INTERVAL, now is 3.0s
    BOOL shouldLog =
        (long)(CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * kSCCaptureFrameRate) %
            ((long)(kSCCaptureFrameRate * kSCLogInterval)) ==
        0;
    if (shouldLog) {
        SCLogVideoStreamerInfo(@"didOutputSampleBuffer:%p", sampleBuffer);
    }
    if (_processingPipeline) {
        RenderData renderData = {
            .sampleBuffer = sampleBuffer,
            .depthDataMap = depthDataMap,
            .depthBlurPointOfInterest =
                SCCameraTweaksEnablePortraitModeAutofocus() || SCCameraTweaksEnablePortraitModeTapToFocus()
                    ? &_portraitModePointOfInterest
                    : nil,
        };
        // Ensure we are doing all render operations (i.e. accessing textures) on performer to prevent race condition
        SCAssertPerformer(_performer);
        sampleBuffer = [_processingPipeline render:renderData];

        if (shouldLog) {
            SCLogVideoStreamerInfo(@"rendered sampleBuffer:%p in processingPipeline:%@", sampleBuffer,
                                   _processingPipeline);
        }
    }

    if (sampleBuffer && _sampleBufferDisplayEnabled) {
        // Send the buffer only if it is valid, set it to be displayed immediately (See the enqueueSampleBuffer method
        // header, need to get attachments array and set the dictionary).
        CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
        if (!attachmentsArray) {
            SCLogVideoStreamerError(@"Error getting attachment array for CMSampleBuffer");
        } else if (CFArrayGetCount(attachmentsArray) > 0) {
            CFMutableDictionaryRef attachment = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachmentsArray, 0);
            CFDictionarySetValue(attachment, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
        }
        // Warn if frame that went through is not most recent enough.
        if (frameLatency >= kSCManagedVideoStreamerMaxAllowedLatency) {
            SCLogVideoStreamerWarning(
                @"The sample buffer we received is too late, why? presentationTime:%lf frameLatency:%f",
                presentationTime, frameLatency);
        }
        [_sampleBufferDisplayController enqueueSampleBuffer:sampleBuffer];
        if (shouldLog) {
            SCLogVideoStreamerInfo(@"displayed sampleBuffer:%p in Metal", sampleBuffer);
        }

        [self _performCompletionHandlersForWaitUntilSampleBufferDisplayed];
    }

    if (shouldLog) {
        SCLogVideoStreamerInfo(@"begin annoucing sampleBuffer:%p of devicePosition:%lu", sampleBuffer,
                               (unsigned long)_devicePosition);
    }
    [_announcer managedVideoDataSource:self didOutputSampleBuffer:sampleBuffer devicePosition:_devicePosition];
    if (shouldLog) {
        SCLogVideoStreamerInfo(@"end annoucing sampleBuffer:%p", sampleBuffer);
    }
}

- (void)didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
{
    if (_performingConfigurations) {
        return;
    }
    SC_GUARD_ELSE_RETURN([_performer isCurrentPerformer]);
    NSTimeInterval currentProcessingTime = CACurrentMediaTime();
    NSTimeInterval currentSampleTime = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer));
    // Only logging it when sticky tweak is on, which means sticky time is too long, and AVFoundation have to drop the
    // sampleBuffer
    if (_keepLateFrames) {
        SCLogVideoStreamerInfo(@"didDropSampleBuffer:%p timestamp:%f latency:%f", sampleBuffer, currentProcessingTime,
                               currentSampleTime);
    }
    [_announcer managedVideoDataSource:self didDropSampleBuffer:sampleBuffer devicePosition:_devicePosition];
}

#pragma mark - AVCaptureVideoDataOutputSampleBufferDelegate

- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
       fromConnection:(AVCaptureConnection *)connection NS_AVAILABLE_IOS(11_0)
{
    // Sticky video tweak is off, i.e. lenses is on,
    // we use same queue for callback and processing, and let AVFoundation decide which frame should be dropped
    if (!_keepLateFrames) {
        [self didOutputSampleBuffer:sampleBuffer];
    }
    // Sticky video tweak is on
    else {
        if ([_performer isCurrentPerformer]) {
            // Note: there might be one frame callbacked in processing queue when switching callback queue,
            // it should be fine. But if following log appears too much, it is not our design.
            SCLogVideoStreamerWarning(@"The callback queue should be a separated queue when sticky tweak is on");
        }
        // TODO: In sticky video v2, we should consider check free memory
        if (_processingBuffersCount >= kSCManagedVideoStreamerMaxProcessingBuffers - 1) {
            SCLogVideoStreamerWarning(@"processingBuffersCount reached to the max. current count:%d",
                                      _processingBuffersCount);
            [self didDropSampleBuffer:sampleBuffer];
            return;
        }
        atomic_fetch_add(&_processingBuffersCount, 1);
        CFRetain(sampleBuffer);
        // _performer should always be the processing queue
        [_performer perform:^{
            [self didOutputSampleBuffer:sampleBuffer];
            CFRelease(sampleBuffer);
            atomic_fetch_sub(&_processingBuffersCount, 1);
        }];
    }
}

- (void)captureOutput:(AVCaptureOutput *)captureOutput
  didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
       fromConnection:(AVCaptureConnection *)connection
{
    [self didDropSampleBuffer:sampleBuffer];
}

#pragma mark - AVCaptureDataOutputSynchronizer (Video + Depth)

- (void)dataOutputSynchronizer:(AVCaptureDataOutputSynchronizer *)synchronizer
    didOutputSynchronizedDataCollection:(AVCaptureSynchronizedDataCollection *)synchronizedDataCollection
    NS_AVAILABLE_IOS(11_0)
{
    AVCaptureSynchronizedDepthData *syncedDepthData = (AVCaptureSynchronizedDepthData *)[synchronizedDataCollection
        synchronizedDataForCaptureOutput:_depthDataOutput];
    AVDepthData *depthData = nil;
    if (syncedDepthData && !syncedDepthData.depthDataWasDropped) {
        depthData = syncedDepthData.depthData;
    }

    AVCaptureSynchronizedSampleBufferData *syncedVideoData =
        (AVCaptureSynchronizedSampleBufferData *)[synchronizedDataCollection
            synchronizedDataForCaptureOutput:_videoDataOutput];
    if (syncedVideoData && !syncedVideoData.sampleBufferWasDropped) {
        CMSampleBufferRef videoSampleBuffer = syncedVideoData.sampleBuffer;
        [self didOutputSampleBuffer:videoSampleBuffer depthData:depthData ? depthData.depthDataMap : nil];
    }
}

#pragma mark - ARSessionDelegate

- (void)session:(ARSession *)session cameraDidChangeTrackingState:(ARCamera *)camera NS_AVAILABLE_IOS(11_0)
{
    NSString *state = nil;
    NSString *reason = nil;
    switch (camera.trackingState) {
    case ARTrackingStateNormal:
        state = @"Normal";
        break;
    case ARTrackingStateLimited:
        state = @"Limited";
        break;
    case ARTrackingStateNotAvailable:
        state = @"Not Available";
        break;
    }
    switch (camera.trackingStateReason) {
    case ARTrackingStateReasonNone:
        reason = @"None";
        break;
    case ARTrackingStateReasonInitializing:
        reason = @"Initializing";
        break;
    case ARTrackingStateReasonExcessiveMotion:
        reason = @"Excessive Motion";
        break;
    case ARTrackingStateReasonInsufficientFeatures:
        reason = @"Insufficient Features";
        break;
#if SC_AT_LEAST_SDK_11_3
    case ARTrackingStateReasonRelocalizing:
        reason = @"Relocalizing";
        break;
#endif
    }
    SCLogVideoStreamerInfo(@"ARKit changed tracking state - %@ (reason: %@)", state, reason);
}

- (void)session:(ARSession *)session didUpdateFrame:(ARFrame *)frame NS_AVAILABLE_IOS(11_0)
{
#ifdef SC_USE_ARKIT_FACE
    // This is extremely weird, but LOOK-10251 indicates that despite the class having it defined, on some specific
    // devices there are ARFrame instances that don't respond to `capturedDepthData`.
    // (note: this was discovered to be due to some people staying on iOS 11 betas).
    AVDepthData *depth = nil;
    if ([frame respondsToSelector:@selector(capturedDepthData)]) {
        depth = frame.capturedDepthData;
    }
#endif

    CGFloat timeSince = frame.timestamp - _lastDisplayedFrameTimestamp;
    // Don't deliver more than 30 frames per sec
    BOOL framerateMinimumElapsed = timeSince >= kSCManagedVideoStreamerARSessionFramerateCap;

#ifdef SC_USE_ARKIT_FACE
    if (depth) {
        CGFloat timeSince = frame.timestamp - _lastDisplayedDepthFrameTimestamp;
        framerateMinimumElapsed |= timeSince >= kSCManagedVideoStreamerARSessionFramerateCap;
    }

#endif

    SC_GUARD_ELSE_RETURN(framerateMinimumElapsed);

#ifdef SC_USE_ARKIT_FACE
    if (depth) {
        self.lastDepthData = depth;
        _lastDisplayedDepthFrameTimestamp = frame.timestamp;
    }
#endif

    // Make sure that current frame is no longer being used, otherwise drop current frame.
    SC_GUARD_ELSE_RETURN(self.currentFrame == nil);

    CVPixelBufferRef pixelBuffer = frame.capturedImage;
    CVPixelBufferLockBaseAddress(pixelBuffer, 0);
    CMTime time = CMTimeMakeWithSeconds(frame.timestamp, 1000000);
    CMSampleTimingInfo timing = {kCMTimeInvalid, time, kCMTimeInvalid};

    CMVideoFormatDescriptionRef videoInfo;
    CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &videoInfo);

    CMSampleBufferRef buffer;
    CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, YES, nil, nil, videoInfo, &timing, &buffer);
    CFRelease(videoInfo);
    CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);

    self.currentFrame = frame;
    [self didOutputSampleBuffer:buffer];
    [self _updateFieldOfViewWithARFrame:frame];

    CFRelease(buffer);
}

- (void)session:(ARSession *)session didAddAnchors:(NSArray<ARAnchor *> *)anchors NS_AVAILABLE_IOS(11_0)
{
    for (ARAnchor *anchor in anchors) {
        if ([anchor isKindOfClass:[ARPlaneAnchor class]]) {
            SCLogVideoStreamerInfo(@"ARKit added plane anchor");
            return;
        }
    }
}

- (void)session:(ARSession *)session didFailWithError:(NSError *)error NS_AVAILABLE_IOS(11_0)
{
    SCLogVideoStreamerError(@"ARKit session failed with error: %@. Resetting", error);
    [session runWithConfiguration:[ARConfiguration sc_configurationForDevicePosition:_devicePosition]];
}

- (void)sessionWasInterrupted:(ARSession *)session NS_AVAILABLE_IOS(11_0)
{
    SCLogVideoStreamerWarning(@"ARKit session interrupted");
}

- (void)sessionInterruptionEnded:(ARSession *)session NS_AVAILABLE_IOS(11_0)
{
    SCLogVideoStreamerInfo(@"ARKit interruption ended");
}

#pragma mark - Private methods

- (void)_performCompletionHandlersForWaitUntilSampleBufferDisplayed
{
    for (NSArray *completion in _waitUntilSampleBufferDisplayedBlocks) {
        // Call the completion handlers.
        dispatch_async(completion[0], completion[1]);
    }
    [_waitUntilSampleBufferDisplayedBlocks removeAllObjects];
}

// This is the magic that ensures the VideoDataOutput will have the correct
// orientation.
- (void)_enableVideoMirrorForDevicePosition:(SCManagedCaptureDevicePosition)devicePosition
{
    SCLogVideoStreamerInfo(@"enable video mirror for device position:%lu", (unsigned long)devicePosition);
    AVCaptureConnection *connection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
    connection.videoOrientation = _videoOrientation;
    if (devicePosition == SCManagedCaptureDevicePositionFront) {
        connection.videoMirrored = YES;
    }
}

- (void)_enableVideoStabilizationIfSupported
{
    SCTraceStart();
    if (!SCCameraTweaksEnableVideoStabilization()) {
        SCLogVideoStreamerWarning(@"SCCameraTweaksEnableVideoStabilization is NO, won't enable video stabilization");
        return;
    }

    AVCaptureConnection *videoConnection = [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
    if (!videoConnection) {
        SCLogVideoStreamerError(@"cannot get videoConnection from videoDataOutput:%@", videoConnection);
        return;
    }
    // Set the video stabilization mode to auto. Default is off.
    if ([videoConnection isVideoStabilizationSupported]) {
        videoConnection.preferredVideoStabilizationMode = _videoStabilizationEnabledIfSupported
                                                              ? AVCaptureVideoStabilizationModeStandard
                                                              : AVCaptureVideoStabilizationModeOff;
        NSDictionary *params = @{ @"iOS8_Mode" : @(videoConnection.activeVideoStabilizationMode) };
        [[SCLogger sharedInstance] logEvent:@"VIDEO_STABILIZATION_MODE" parameters:params];
        SCLogVideoStreamerInfo(@"set video stabilization mode:%ld to videoConnection:%@",
                               (long)videoConnection.preferredVideoStabilizationMode, videoConnection);
    } else {
        SCLogVideoStreamerInfo(@"video stabilization isn't supported on videoConnection:%@", videoConnection);
    }
}

- (void)setVideoStabilizationEnabledIfSupported:(BOOL)videoStabilizationIfSupported
{
    SCLogVideoStreamerInfo(@"setVideoStabilizationEnabledIfSupported:%d", videoStabilizationIfSupported);
    _videoStabilizationEnabledIfSupported = videoStabilizationIfSupported;
    [self _enableVideoStabilizationIfSupported];
}

- (void)_updateFieldOfViewWithARFrame:(ARFrame *)frame NS_AVAILABLE_IOS(11_0)
{
    SC_GUARD_ELSE_RETURN(frame.camera);
    CGSize imageResolution = frame.camera.imageResolution;
    matrix_float3x3 intrinsics = frame.camera.intrinsics;
    float xFovDegrees = 2 * atan(imageResolution.width / (2 * intrinsics.columns[0][0])) * 180 / M_PI;
    if (_fieldOfView != xFovDegrees) {
        self.fieldOfView = xFovDegrees;
    }
}

- (NSString *)description
{
    return [self debugDescription];
}

- (NSString *)debugDescription
{
    NSDictionary *debugDict = @{
        @"_sampleBufferDisplayEnabled" : _sampleBufferDisplayEnabled ? @"Yes" : @"No",
        @"_videoStabilizationEnabledIfSupported" : _videoStabilizationEnabledIfSupported ? @"Yes" : @"No",
        @"_performingConfigurations" : _performingConfigurations ? @"Yes" : @"No",
        @"alwaysDiscardLateVideoFrames" : _videoDataOutput.alwaysDiscardsLateVideoFrames ? @"Yes" : @"No"
    };
    return [NSString sc_stringWithFormat:@"%@", debugDict];
}

@end