// // SCManagedDeviceCapacityAnalyzer.m // Snapchat // // Created by Liu Liu on 5/1/15. // Copyright (c) 2015 Liu Liu. All rights reserved. // #import "SCManagedDeviceCapacityAnalyzer.h" #import "SCCameraSettingUtils.h" #import "SCCameraTweaks.h" #import "SCManagedCaptureDevice+SCManagedDeviceCapacityAnalyzer.h" #import "SCManagedCaptureDevice.h" #import "SCManagedDeviceCapacityAnalyzerListenerAnnouncer.h" #import #import #import #import #import #import @import ImageIO; @import QuartzCore; NSInteger const kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor6WithHRSI = 500; NSInteger const kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor6S = 800; NSInteger const kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor7 = 640; NSInteger const kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor8 = 800; // After this much frames we haven't changed exposure time or ISO, we will assume that the adjustingExposure is ended. static NSInteger const kExposureUnchangedHighWatermark = 5; // If deadline reached, and we still haven't reached high watermark yet, we will consult the low watermark and at least // give the system a chance to take not-so-great pictures. static NSInteger const kExposureUnchangedLowWatermark = 1; static NSTimeInterval const kExposureUnchangedDeadline = 0.2; // It seems that between ISO 500 to 640, the brightness value is always somewhere around -0.4 to -0.5. // Therefore, this threshold probably will work fine. static float const kBrightnessValueThreshold = -2.25; // Give some margins between recognized as bright enough and not enough light. // If the brightness is lower than kBrightnessValueThreshold - kBrightnessValueThresholdConfidenceInterval, // and then we count the frame as low light frame. Only if the brightness is higher than // kBrightnessValueThreshold + kBrightnessValueThresholdConfidenceInterval, we think that we // have enough light, and reset low light frame count to 0. 0.5 is choosing because in dark // environment, the brightness value changes +-0.3 with minor orientation changes. static float const kBrightnessValueThresholdConfidenceInterval = 0.5; // If we are at good light condition for 5 frames, ready to change back static NSInteger const kLowLightBoostUnchangedLowWatermark = 7; // Requires we are at low light condition for ~2 seconds (assuming 20~30fps) static NSInteger const kLowLightBoostUnchangedHighWatermark = 25; static NSInteger const kSCLightingConditionDecisionWatermark = 15; // For 30 fps, it is 0.5 second static float const kSCLightingConditionNormalThreshold = 0; static float const kSCLightingConditionDarkThreshold = -3; @implementation SCManagedDeviceCapacityAnalyzer { float _lastExposureTime; int _lastISOSpeedRating; NSTimeInterval _lastAdjustingExposureStartTime; NSInteger _lowLightBoostLowLightCount; NSInteger _lowLightBoostEnoughLightCount; NSInteger _exposureUnchangedCount; NSInteger _maxISOPresetHigh; NSInteger _normalLightingConditionCount; NSInteger _darkLightingConditionCount; NSInteger _extremeDarkLightingConditionCount; SCCapturerLightingConditionType _lightingCondition; BOOL _lowLightCondition; BOOL _adjustingExposure; SCManagedDeviceCapacityAnalyzerListenerAnnouncer *_announcer; FBKVOController *_observeController; id _performer; float _lastBrightnessToLog; // Remember last logged brightness, only log again if it changes greater than a threshold } - (instancetype)initWithPerformer:(id)performer { SCTraceStart(); self = [super init]; if (self) { _performer = performer; _maxISOPresetHigh = kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor6WithHRSI; if ([SCDeviceName isIphone] && [SCDeviceName isSimilarToIphone8orNewer]) { _maxISOPresetHigh = kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor8; } else if ([SCDeviceName isIphone] && [SCDeviceName isSimilarToIphone7orNewer]) { _maxISOPresetHigh = kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor7; } else if ([SCDeviceName isIphone] && [SCDeviceName isSimilarToIphone6SorNewer]) { // iPhone 6S supports higher ISO rate for video recording, accommadating that. _maxISOPresetHigh = kSCManagedDeviceCapacityAnalyzerMaxISOPresetHighFor6S; } _announcer = [[SCManagedDeviceCapacityAnalyzerListenerAnnouncer alloc] init]; _observeController = [[FBKVOController alloc] initWithObserver:self]; } return self; } - (void)addListener:(id)listener { SCTraceStart(); [_announcer addListener:listener]; } - (void)removeListener:(id)listener { SCTraceStart(); [_announcer removeListener:listener]; } - (void)setLowLightConditionEnabled:(BOOL)lowLightConditionEnabled { SCTraceStart(); if (_lowLightConditionEnabled != lowLightConditionEnabled) { _lowLightConditionEnabled = lowLightConditionEnabled; if (!lowLightConditionEnabled) { _lowLightBoostLowLightCount = 0; _lowLightBoostEnoughLightCount = 0; _lowLightCondition = NO; [_announcer managedDeviceCapacityAnalyzer:self didChangeLowLightCondition:_lowLightCondition]; } } } - (void)managedVideoDataSource:(id)managedVideoDataSource didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer devicePosition:(SCManagedCaptureDevicePosition)devicePosition { SCTraceStart(); SampleBufferMetadata metadata = { .isoSpeedRating = _lastISOSpeedRating, .brightness = 0, .exposureTime = _lastExposureTime, }; retrieveSampleBufferMetadata(sampleBuffer, &metadata); if ((SCIsDebugBuild() || SCIsMasterBuild()) // Enable this on internal build only (excluding alpha) && fabs(metadata.brightness - _lastBrightnessToLog) > 0.5f) { // Log only when brightness change is greater than 0.5 _lastBrightnessToLog = metadata.brightness; SCLogCoreCameraInfo(@"ExposureTime: %f, ISO: %ld, Brightness: %f", metadata.exposureTime, (long)metadata.isoSpeedRating, metadata.brightness); } [self _automaticallyDetectAdjustingExposure:metadata.exposureTime ISOSpeedRating:metadata.isoSpeedRating]; _lastExposureTime = metadata.exposureTime; _lastISOSpeedRating = metadata.isoSpeedRating; if (!_adjustingExposure && _lastISOSpeedRating <= _maxISOPresetHigh && _lowLightConditionEnabled) { // If we are not recording, we are not at ISO higher than we needed [self _automaticallyDetectLowLightCondition:metadata.brightness]; } [self _automaticallyDetectLightingConditionWithBrightness:metadata.brightness]; [_announcer managedDeviceCapacityAnalyzer:self didChangeBrightness:metadata.brightness]; } - (void)setAsFocusListenerForDevice:(SCManagedCaptureDevice *)captureDevice { SCTraceStart(); [_observeController observe:captureDevice.device keyPath:@keypath(captureDevice.device, adjustingFocus) options:NSKeyValueObservingOptionNew action:@selector(_adjustingFocusingChanged:)]; } - (void)removeFocusListener { SCTraceStart(); [_observeController unobserveAll]; } #pragma mark - Private methods - (void)_automaticallyDetectAdjustingExposure:(float)currentExposureTime ISOSpeedRating:(NSInteger)currentISOSpeedRating { SCTraceStart(); if (currentISOSpeedRating != _lastISOSpeedRating || fabsf(currentExposureTime - _lastExposureTime) > FLT_MIN) { _exposureUnchangedCount = 0; } else { ++_exposureUnchangedCount; } NSTimeInterval currentTime = CACurrentMediaTime(); if (_exposureUnchangedCount >= kExposureUnchangedHighWatermark || (currentTime - _lastAdjustingExposureStartTime > kExposureUnchangedDeadline && _exposureUnchangedCount >= kExposureUnchangedLowWatermark)) { // The exposure values haven't changed for kExposureUnchangedHighWatermark times, considering the adjustment // as done. Otherwise, if we waited long enough, and the exposure unchange count at least reached low // watermark, we will call it done and give it a shot. if (_adjustingExposure) { _adjustingExposure = NO; SCLogGeneralInfo(@"Adjusting exposure is done, unchanged count: %zd", _exposureUnchangedCount); [_announcer managedDeviceCapacityAnalyzer:self didChangeAdjustingExposure:_adjustingExposure]; } } else { // Otherwise signal that we have adjustments on exposure if (!_adjustingExposure) { _adjustingExposure = YES; _lastAdjustingExposureStartTime = currentTime; [_announcer managedDeviceCapacityAnalyzer:self didChangeAdjustingExposure:_adjustingExposure]; } } } - (void)_automaticallyDetectLowLightCondition:(float)brightness { SCTraceStart(); if (!_lowLightCondition && _lastISOSpeedRating == _maxISOPresetHigh) { // If we are at the stage that we need to use higher ISO (because current ISO is maxed out) // and the brightness is lower than the threshold if (brightness < kBrightnessValueThreshold - kBrightnessValueThresholdConfidenceInterval) { // Either count how many frames like this continuously we encountered // Or if reached the watermark, change the low light boost mode if (_lowLightBoostLowLightCount >= kLowLightBoostUnchangedHighWatermark) { _lowLightCondition = YES; [_announcer managedDeviceCapacityAnalyzer:self didChangeLowLightCondition:_lowLightCondition]; } else { ++_lowLightBoostLowLightCount; } } else if (brightness >= kBrightnessValueThreshold + kBrightnessValueThresholdConfidenceInterval) { // If the brightness is consistently better, reset the low light boost unchanged count to 0 _lowLightBoostLowLightCount = 0; } } else if (_lowLightCondition) { // Check the current ISO to see if we can disable low light boost if (_lastISOSpeedRating <= _maxISOPresetHigh && brightness >= kBrightnessValueThreshold + kBrightnessValueThresholdConfidenceInterval) { if (_lowLightBoostEnoughLightCount >= kLowLightBoostUnchangedLowWatermark) { _lowLightCondition = NO; [_announcer managedDeviceCapacityAnalyzer:self didChangeLowLightCondition:_lowLightCondition]; _lowLightBoostEnoughLightCount = 0; } else { ++_lowLightBoostEnoughLightCount; } } } } - (void)_adjustingFocusingChanged:(NSDictionary *)change { SCTraceStart(); BOOL adjustingFocus = [change[NSKeyValueChangeNewKey] boolValue]; [_performer perform:^{ [_announcer managedDeviceCapacityAnalyzer:self didChangeAdjustingFocus:adjustingFocus]; }]; } - (void)_automaticallyDetectLightingConditionWithBrightness:(float)brightness { if (brightness >= kSCLightingConditionNormalThreshold) { if (_normalLightingConditionCount > kSCLightingConditionDecisionWatermark) { if (_lightingCondition != SCCapturerLightingConditionTypeNormal) { _lightingCondition = SCCapturerLightingConditionTypeNormal; [_announcer managedDeviceCapacityAnalyzer:self didChangeLightingCondition:SCCapturerLightingConditionTypeNormal]; } } else { _normalLightingConditionCount++; } _darkLightingConditionCount = 0; _extremeDarkLightingConditionCount = 0; } else if (brightness >= kSCLightingConditionDarkThreshold) { if (_darkLightingConditionCount > kSCLightingConditionDecisionWatermark) { if (_lightingCondition != SCCapturerLightingConditionTypeDark) { _lightingCondition = SCCapturerLightingConditionTypeDark; [_announcer managedDeviceCapacityAnalyzer:self didChangeLightingCondition:SCCapturerLightingConditionTypeDark]; } } else { _darkLightingConditionCount++; } _normalLightingConditionCount = 0; _extremeDarkLightingConditionCount = 0; } else { if (_extremeDarkLightingConditionCount > kSCLightingConditionDecisionWatermark) { if (_lightingCondition != SCCapturerLightingConditionTypeExtremeDark) { _lightingCondition = SCCapturerLightingConditionTypeExtremeDark; [_announcer managedDeviceCapacityAnalyzer:self didChangeLightingCondition:SCCapturerLightingConditionTypeExtremeDark]; } } else { _extremeDarkLightingConditionCount++; } _normalLightingConditionCount = 0; _darkLightingConditionCount = 0; } } @end