// // SCCapturerBufferedVideoWriter.m // Snapchat // // Created by Chao Pang on 12/5/17. // #import "SCCapturerBufferedVideoWriter.h" #import "SCAudioCaptureSession.h" #import "SCCaptureCommon.h" #import "SCManagedCapturerUtils.h" #import #import #import #import #import #import @implementation SCCapturerBufferedVideoWriter { SCQueuePerformer *_performer; __weak id _delegate; FBKVOController *_observeController; AVAssetWriter *_assetWriter; AVAssetWriterInput *_audioWriterInput; AVAssetWriterInput *_videoWriterInput; AVAssetWriterInputPixelBufferAdaptor *_pixelBufferAdaptor; CVPixelBufferPoolRef _defaultPixelBufferPool; CVPixelBufferPoolRef _nightPixelBufferPool; CVPixelBufferPoolRef _lensesPixelBufferPool; CMBufferQueueRef _videoBufferQueue; CMBufferQueueRef _audioBufferQueue; } - (instancetype)initWithPerformer:(id)performer outputURL:(NSURL *)outputURL delegate:(id)delegate error:(NSError **)error { self = [super init]; if (self) { _performer = performer; _delegate = delegate; _observeController = [[FBKVOController alloc] initWithObserver:self]; CMBufferQueueCreate(kCFAllocatorDefault, 0, CMBufferQueueGetCallbacksForUnsortedSampleBuffers(), &_videoBufferQueue); CMBufferQueueCreate(kCFAllocatorDefault, 0, CMBufferQueueGetCallbacksForUnsortedSampleBuffers(), &_audioBufferQueue); _assetWriter = [[AVAssetWriter alloc] initWithURL:outputURL fileType:AVFileTypeMPEG4 error:error]; if (*error) { self = nil; return self; } } return self; } - (BOOL)prepareWritingWithOutputSettings:(SCManagedVideoCapturerOutputSettings *)outputSettings { SCTraceStart(); SCAssert([_performer isCurrentPerformer], @""); SCAssert(outputSettings, @"empty output setting"); // Audio SCTraceSignal(@"Derive audio output setting"); NSDictionary *audioOutputSettings = @{ AVFormatIDKey : @(kAudioFormatMPEG4AAC), AVNumberOfChannelsKey : @(1), AVSampleRateKey : @(kSCAudioCaptureSessionDefaultSampleRate), AVEncoderBitRateKey : @(outputSettings.audioBitRate) }; _audioWriterInput = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings]; _audioWriterInput.expectsMediaDataInRealTime = YES; // Video SCTraceSignal(@"Derive video output setting"); size_t outputWidth = outputSettings.width; size_t outputHeight = outputSettings.height; SCAssert(outputWidth > 0 && outputHeight > 0 && (outputWidth % 2 == 0) && (outputHeight % 2 == 0), @"invalid output size"); NSDictionary *videoCompressionSettings = @{ AVVideoAverageBitRateKey : @(outputSettings.videoBitRate), AVVideoMaxKeyFrameIntervalKey : @(outputSettings.keyFrameInterval) }; NSDictionary *videoOutputSettings = @{ AVVideoCodecKey : AVVideoCodecH264, AVVideoWidthKey : @(outputWidth), AVVideoHeightKey : @(outputHeight), AVVideoScalingModeKey : AVVideoScalingModeResizeAspectFill, AVVideoCompressionPropertiesKey : videoCompressionSettings }; _videoWriterInput = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo outputSettings:videoOutputSettings]; _videoWriterInput.expectsMediaDataInRealTime = YES; CGAffineTransform transform = CGAffineTransformMakeTranslation(outputHeight, 0); _videoWriterInput.transform = CGAffineTransformRotate(transform, M_PI_2); _pixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc] initWithAssetWriterInput:_videoWriterInput sourcePixelBufferAttributes:@{ (NSString *) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), (NSString *) kCVPixelBufferWidthKey : @(outputWidth), (NSString *) kCVPixelBufferHeightKey : @(outputHeight) }]; SCTraceSignal(@"Setup video writer input"); if ([_assetWriter canAddInput:_videoWriterInput]) { [_assetWriter addInput:_videoWriterInput]; } else { return NO; } SCTraceSignal(@"Setup audio writer input"); if ([_assetWriter canAddInput:_audioWriterInput]) { [_assetWriter addInput:_audioWriterInput]; } else { return NO; } return YES; } - (void)appendVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer { SCAssert([_performer isCurrentPerformer], @""); SC_GUARD_ELSE_RETURN(sampleBuffer); if (!CMBufferQueueIsEmpty(_videoBufferQueue)) { // We need to drain the buffer queue in this case while (_videoWriterInput.readyForMoreMediaData) { // TODO: also need to break out in case of errors CMSampleBufferRef dequeuedSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_videoBufferQueue); if (dequeuedSampleBuffer == NULL) { break; } [self _appendVideoSampleBuffer:dequeuedSampleBuffer]; CFRelease(dequeuedSampleBuffer); } } // Fast path, just append this sample buffer if ready if (_videoWriterInput.readyForMoreMediaData) { [self _appendVideoSampleBuffer:sampleBuffer]; } else { // It is not ready, queuing the sample buffer CMBufferQueueEnqueue(_videoBufferQueue, sampleBuffer); } } - (void)appendAudioSampleBuffer:(CMSampleBufferRef)sampleBuffer { SCAssert([_performer isCurrentPerformer], @""); SC_GUARD_ELSE_RETURN(sampleBuffer); if (!CMBufferQueueIsEmpty(_audioBufferQueue)) { // We need to drain the buffer queue in this case while (_audioWriterInput.readyForMoreMediaData) { CMSampleBufferRef dequeuedSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_audioBufferQueue); if (dequeuedSampleBuffer == NULL) { break; } [_audioWriterInput appendSampleBuffer:sampleBuffer]; CFRelease(dequeuedSampleBuffer); } } // fast path, just append this sample buffer if ready if ((_audioWriterInput.readyForMoreMediaData)) { [_audioWriterInput appendSampleBuffer:sampleBuffer]; } else { // it is not ready, queuing the sample buffer CMBufferQueueEnqueue(_audioBufferQueue, sampleBuffer); } } - (void)startWritingAtSourceTime:(CMTime)sourceTime { SCTraceStart(); SCAssert([_performer isCurrentPerformer], @""); // To observe the status change on assetWriter because when assetWriter errors out, it only changes the // status, no further delegate callbacks etc. [_observeController observe:_assetWriter keyPath:@keypath(_assetWriter, status) options:NSKeyValueObservingOptionNew action:@selector(assetWriterStatusChanged:)]; [_assetWriter startWriting]; [_assetWriter startSessionAtSourceTime:sourceTime]; } - (void)cancelWriting { SCTraceStart(); SCAssert([_performer isCurrentPerformer], @""); CMBufferQueueReset(_videoBufferQueue); CMBufferQueueReset(_audioBufferQueue); [_assetWriter cancelWriting]; } - (void)finishWritingAtSourceTime:(CMTime)sourceTime withCompletionHanlder:(dispatch_block_t)completionBlock { SCTraceStart(); SCAssert([_performer isCurrentPerformer], @""); while (_audioWriterInput.readyForMoreMediaData && !CMBufferQueueIsEmpty(_audioBufferQueue)) { CMSampleBufferRef audioSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_audioBufferQueue); if (audioSampleBuffer == NULL) { break; } [_audioWriterInput appendSampleBuffer:audioSampleBuffer]; CFRelease(audioSampleBuffer); } while (_videoWriterInput.readyForMoreMediaData && !CMBufferQueueIsEmpty(_videoBufferQueue)) { CMSampleBufferRef videoSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_videoBufferQueue); if (videoSampleBuffer == NULL) { break; } [_videoWriterInput appendSampleBuffer:videoSampleBuffer]; CFRelease(videoSampleBuffer); } dispatch_block_t finishWritingBlock = ^() { [_assetWriter endSessionAtSourceTime:sourceTime]; [_audioWriterInput markAsFinished]; [_videoWriterInput markAsFinished]; [_assetWriter finishWritingWithCompletionHandler:^{ if (completionBlock) { completionBlock(); } }]; }; if (CMBufferQueueIsEmpty(_audioBufferQueue) && CMBufferQueueIsEmpty(_videoBufferQueue)) { finishWritingBlock(); } else { // We need to drain the samples from the queues before finish writing __block BOOL isAudioDone = NO; __block BOOL isVideoDone = NO; // Audio [_audioWriterInput requestMediaDataWhenReadyOnQueue:_performer.queue usingBlock:^{ if (!CMBufferQueueIsEmpty(_audioBufferQueue) && _assetWriter.status == AVAssetWriterStatusWriting) { CMSampleBufferRef audioSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_audioBufferQueue); if (audioSampleBuffer) { [_audioWriterInput appendSampleBuffer:audioSampleBuffer]; CFRelease(audioSampleBuffer); } } else if (!isAudioDone) { isAudioDone = YES; } if (isAudioDone && isVideoDone) { finishWritingBlock(); } }]; // Video [_videoWriterInput requestMediaDataWhenReadyOnQueue:_performer.queue usingBlock:^{ if (!CMBufferQueueIsEmpty(_videoBufferQueue) && _assetWriter.status == AVAssetWriterStatusWriting) { CMSampleBufferRef videoSampleBuffer = (CMSampleBufferRef)CMBufferQueueDequeueAndRetain(_videoBufferQueue); if (videoSampleBuffer) { [_videoWriterInput appendSampleBuffer:videoSampleBuffer]; CFRelease(videoSampleBuffer); } } else if (!isVideoDone) { isVideoDone = YES; } if (isAudioDone && isVideoDone) { finishWritingBlock(); } }]; } } - (void)cleanUp { _assetWriter = nil; _videoWriterInput = nil; _audioWriterInput = nil; _pixelBufferAdaptor = nil; } - (void)dealloc { CFRelease(_videoBufferQueue); CFRelease(_audioBufferQueue); CVPixelBufferPoolRelease(_defaultPixelBufferPool); CVPixelBufferPoolRelease(_nightPixelBufferPool); CVPixelBufferPoolRelease(_lensesPixelBufferPool); [_observeController unobserveAll]; } - (void)assetWriterStatusChanged:(NSDictionary *)change { SCTraceStart(); if (_assetWriter.status == AVAssetWriterStatusFailed) { SCTraceSignal(@"Asset writer status failed %@, error %@", change, _assetWriter.error); [_delegate videoWriterDidFailWritingWithError:[_assetWriter.error copy]]; } } #pragma - Private methods - (CVImageBufferRef)_croppedPixelBufferWithInputPixelBuffer:(CVImageBufferRef)inputPixelBuffer { SCAssertTrue([SCDeviceName isIphoneX]); const size_t inputBufferWidth = CVPixelBufferGetWidth(inputPixelBuffer); const size_t inputBufferHeight = CVPixelBufferGetHeight(inputPixelBuffer); const size_t croppedBufferWidth = (size_t)(inputBufferWidth * kSCIPhoneXCapturedImageVideoCropRatio) / 2 * 2; const size_t croppedBufferHeight = (size_t)(croppedBufferWidth * SCManagedCapturedImageAndVideoAspectRatio()) / 2 * 2; const size_t offsetPointX = inputBufferWidth - croppedBufferWidth; const size_t offsetPointY = (inputBufferHeight - croppedBufferHeight) / 4 * 2; SC_GUARD_ELSE_RUN_AND_RETURN_VALUE((inputBufferWidth >= croppedBufferWidth) && (inputBufferHeight >= croppedBufferHeight) && (offsetPointX % 2 == 0) && (offsetPointY % 2 == 0) && (inputBufferWidth >= croppedBufferWidth + offsetPointX) && (inputBufferHeight >= croppedBufferHeight + offsetPointY), SCLogGeneralError(@"Invalid cropping configuration"), NULL); CVPixelBufferRef croppedPixelBuffer = NULL; CVPixelBufferPoolRef pixelBufferPool = [self _pixelBufferPoolWithInputSize:CGSizeMake(inputBufferWidth, inputBufferHeight) croppedSize:CGSizeMake(croppedBufferWidth, croppedBufferHeight)]; if (pixelBufferPool) { CVReturn result = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixelBufferPool, &croppedPixelBuffer); if ((result != kCVReturnSuccess) || (croppedPixelBuffer == NULL)) { SCLogGeneralError(@"[SCCapturerVideoWriterInput] Error creating croppedPixelBuffer"); return NULL; } } else { SCAssertFail(@"[SCCapturerVideoWriterInput] PixelBufferPool is NULL with inputBufferWidth:%@, " @"inputBufferHeight:%@, croppedBufferWidth:%@, croppedBufferHeight:%@", @(inputBufferWidth), @(inputBufferHeight), @(croppedBufferWidth), @(croppedBufferHeight)); return NULL; } CVPixelBufferLockBaseAddress(inputPixelBuffer, kCVPixelBufferLock_ReadOnly); CVPixelBufferLockBaseAddress(croppedPixelBuffer, 0); const size_t planesCount = CVPixelBufferGetPlaneCount(inputPixelBuffer); for (int planeIndex = 0; planeIndex < planesCount; planeIndex++) { size_t inPlaneHeight = CVPixelBufferGetHeightOfPlane(inputPixelBuffer, planeIndex); size_t inPlaneBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(inputPixelBuffer, planeIndex); uint8_t *inPlaneAdress = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(inputPixelBuffer, planeIndex); size_t croppedPlaneHeight = CVPixelBufferGetHeightOfPlane(croppedPixelBuffer, planeIndex); size_t croppedPlaneBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(croppedPixelBuffer, planeIndex); uint8_t *croppedPlaneAdress = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(croppedPixelBuffer, planeIndex); // Note that inPlaneBytesPerRow is not strictly 2x of inPlaneWidth for some devices (e.g. iPhone X). // However, since UV are packed together in memory, we can use offsetPointX for all planes size_t offsetPlaneBytesX = offsetPointX; size_t offsetPlaneBytesY = offsetPointY * inPlaneHeight / inputBufferHeight; inPlaneAdress = inPlaneAdress + offsetPlaneBytesY * inPlaneBytesPerRow + offsetPlaneBytesX; size_t bytesToCopyPerRow = MIN(inPlaneBytesPerRow - offsetPlaneBytesX, croppedPlaneBytesPerRow); for (int i = 0; i < croppedPlaneHeight; i++) { memcpy(croppedPlaneAdress, inPlaneAdress, bytesToCopyPerRow); inPlaneAdress += inPlaneBytesPerRow; croppedPlaneAdress += croppedPlaneBytesPerRow; } } CVPixelBufferUnlockBaseAddress(inputPixelBuffer, kCVPixelBufferLock_ReadOnly); CVPixelBufferUnlockBaseAddress(croppedPixelBuffer, 0); return croppedPixelBuffer; } - (CVPixelBufferPoolRef)_pixelBufferPoolWithInputSize:(CGSize)inputSize croppedSize:(CGSize)croppedSize { if (CGSizeEqualToSize(inputSize, [SCManagedCaptureDevice defaultActiveFormatResolution])) { if (_defaultPixelBufferPool == NULL) { _defaultPixelBufferPool = [self _newPixelBufferPoolWithWidth:croppedSize.width height:croppedSize.height]; } return _defaultPixelBufferPool; } else if (CGSizeEqualToSize(inputSize, [SCManagedCaptureDevice nightModeActiveFormatResolution])) { if (_nightPixelBufferPool == NULL) { _nightPixelBufferPool = [self _newPixelBufferPoolWithWidth:croppedSize.width height:croppedSize.height]; } return _nightPixelBufferPool; } else { if (_lensesPixelBufferPool == NULL) { _lensesPixelBufferPool = [self _newPixelBufferPoolWithWidth:croppedSize.width height:croppedSize.height]; } return _lensesPixelBufferPool; } } - (CVPixelBufferPoolRef)_newPixelBufferPoolWithWidth:(size_t)width height:(size_t)height { NSDictionary *attributes = @{ (NSString *) kCVPixelBufferIOSurfacePropertiesKey : @{}, (NSString *) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), (NSString *) kCVPixelBufferWidthKey : @(width), (NSString *) kCVPixelBufferHeightKey : @(height) }; CVPixelBufferPoolRef pixelBufferPool = NULL; CVReturn result = CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL, (__bridge CFDictionaryRef _Nullable)(attributes), &pixelBufferPool); if (result != kCVReturnSuccess) { SCLogGeneralError(@"[SCCapturerBufferredVideoWriter] Error creating pixel buffer pool %i", result); return NULL; } return pixelBufferPool; } - (void)_appendVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer { SCAssert([_performer isCurrentPerformer], @""); CMTime presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); CVImageBufferRef inputPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); if ([SCDeviceName isIphoneX]) { CVImageBufferRef croppedPixelBuffer = [self _croppedPixelBufferWithInputPixelBuffer:inputPixelBuffer]; if (croppedPixelBuffer) { [_pixelBufferAdaptor appendPixelBuffer:croppedPixelBuffer withPresentationTime:presentationTime]; CVPixelBufferRelease(croppedPixelBuffer); } } else { [_pixelBufferAdaptor appendPixelBuffer:inputPixelBuffer withPresentationTime:presentationTime]; } } @end