diff --git a/SmartDeviceLink/SDLProxy.m b/SmartDeviceLink/SDLProxy.m index 4470381ab..0ebe5ede2 100644 --- a/SmartDeviceLink/SDLProxy.m +++ b/SmartDeviceLink/SDLProxy.m @@ -29,7 +29,7 @@ #import "SDLProtocolMessage.h" #import "SDLProtocolMessage.h" #import "SDLPutFile.h" -#import "SDLRPCPayload.h" +#import "SDLRegisterAppInterfaceResponse.h" #import "SDLRPCPayload.h" #import "SDLRPCRequestFactory.h" #import "SDLRPCResponse.h" @@ -56,7 +56,8 @@ @interface SDLProxy () { } @property (strong, nonatomic) NSMutableSet *mutableProxyListeners; -@property (nonatomic, strong, readwrite) SDLStreamingMediaManager *streamingMediaManager; +@property (nonatomic, strong, readwrite, nullable) SDLStreamingMediaManager *streamingMediaManager; +@property (nonatomic, strong, nullable) SDLDisplayCapabilities* displayCapabilities; @end @@ -101,6 +102,8 @@ - (void)destructObjects { _transport = nil; _protocol = nil; _mutableProxyListeners = nil; + _streamingMediaManager = nil; + _displayCapabilities = nil; } } @@ -166,7 +169,10 @@ - (NSString *)proxyVersion { - (SDLStreamingMediaManager *)streamingMediaManager { if (_streamingMediaManager == nil) { - _streamingMediaManager = [[SDLStreamingMediaManager alloc] initWithProtocol:self.protocol]; + if (self.displayCapabilities == nil) { + @throw [NSException exceptionWithName:NSInvalidArgumentException reason:@"SDLStreamingMediaManager must be accessed only after a successful RegisterAppInterfaceResponse" userInfo:nil]; + } + _streamingMediaManager = [[SDLStreamingMediaManager alloc] initWithProtocol:self.protocol displayCapabilities:self.displayCapabilities]; [self.protocol.protocolDelegateTable addObject:_streamingMediaManager]; [self.mutableProxyListeners addObject:_streamingMediaManager.touchManager]; } @@ -328,6 +334,10 @@ - (void)handleRegisterAppInterfaceResponse:(SDLRPCResponse *)response { [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sendMobileHMIState) name:UIApplicationDidBecomeActiveNotification object:nil]; [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sendMobileHMIState) name:UIApplicationDidEnterBackgroundNotification object:nil]; } + + // Extract the display capabilties to successfully build SDLStreamingMediaManager's video encoder. + SDLRegisterAppInterfaceResponse* registerResponse = (SDLRegisterAppInterfaceResponse*)response; + self.displayCapabilities = registerResponse.displayCapabilities; } - (void)handleSyncPData:(SDLRPCMessage *)message { diff --git a/SmartDeviceLink/SDLStreamingMediaManager.h b/SmartDeviceLink/SDLStreamingMediaManager.h index 7e0b47dac..dc6434ca1 100644 --- a/SmartDeviceLink/SDLStreamingMediaManager.h +++ b/SmartDeviceLink/SDLStreamingMediaManager.h @@ -12,16 +12,19 @@ #import "SDLProtocolListener.h" @class SDLAbstractProtocol; +@class SDLDisplayCapabilities; @class SDLTouchManager; + NS_ASSUME_NONNULL_BEGIN typedef NS_ENUM(NSInteger, SDLStreamingVideoError) { - SDLStreamingVideoErrorHeadUnitNACK, - SDLSTreamingVideoErrorInvalidOperatingSystemVersion, - SDLStreamingVideoErrorConfigurationCompressionSessionCreationFailure, - SDLStreamingVideoErrorConfigurationAllocationFailure, - SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure + SDLStreamingVideoErrorHeadUnitNACK = 0, + SDLSTreamingVideoErrorInvalidOperatingSystemVersion __deprecated_enum_msg("Use SDLStreamingVideoErrorInvalidOperatingSystemVersion instead") = 1, + SDLStreamingVideoErrorInvalidOperatingSystemVersion = 1, + SDLStreamingVideoErrorConfigurationCompressionSessionCreationFailure = 2, + SDLStreamingVideoErrorConfigurationAllocationFailure = 3, + SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure = 4 }; typedef NS_ENUM(NSInteger, SDLStreamingAudioError) { @@ -36,7 +39,9 @@ typedef void (^SDLStreamingStartBlock)(BOOL success, NSError *__nullable error); @interface SDLStreamingMediaManager : NSObject -- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol; +- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol __deprecated_msg(("Please use initWithProtocol:displayCapabilities: instead")); + +- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol displayCapabilities:(SDLDisplayCapabilities*)displayCapabilities; /** * This method will attempt to start a streaming video session. It will set up iOS's video encoder, and call out to the head unit asking if it will start a video session. @@ -90,6 +95,22 @@ typedef void (^SDLStreamingStartBlock)(BOOL success, NSError *__nullable error); */ @property (nonatomic, strong, readonly) SDLTouchManager* touchManager; +/** + * The settings used in a VTCompressionSessionRef encoder. These will be verified when the video stream is started. Acceptable properties for this are located in VTCompressionProperties. If set to nil, the defaultVideoEncoderSettings will be used. + * + * @warning Video streaming must not be connected to update the encoder properties. If it is running, issue a stopVideoSession before updating. + */ +@property (strong, nonatomic, null_resettable) NSDictionary* videoEncoderSettings; + +/** + * Provides default video encoder settings used. + */ +@property (strong, nonatomic, readonly) NSDictionary* defaultVideoEncoderSettings; + +/** + * This is the current screen size of a connected display. This will be the size the video encoder uses to encode the raw image data. + */ +@property (assign, nonatomic, readonly) CGSize screenSize; @end diff --git a/SmartDeviceLink/SDLStreamingMediaManager.m b/SmartDeviceLink/SDLStreamingMediaManager.m index 0159c5f70..68aaf55c0 100644 --- a/SmartDeviceLink/SDLStreamingMediaManager.m +++ b/SmartDeviceLink/SDLStreamingMediaManager.m @@ -11,9 +11,13 @@ @import UIKit; #import "SDLAbstractProtocol.h" +#import "SDLDisplayCapabilities.h" #import "SDLGlobals.h" +#import "SDLImageResolution.h" +#import "SDLScreenParams.h" #import "SDLTouchManager.h" + NSString *const SDLErrorDomainStreamingMediaVideo = @"com.sdl.streamingmediamanager.video"; NSString *const SDLErrorDomainStreamingMediaAudio = @"com.sdl.streamingmediamanager.audio"; @@ -43,23 +47,57 @@ @implementation SDLStreamingMediaManager #pragma mark - Class Lifecycle +- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol displayCapabilities:(SDLDisplayCapabilities*)displayCapabilities { + self = [self init]; + if (!self) { + return nil; + } + + _protocol = protocol; + + SDLImageResolution* resolution = displayCapabilities.screenParams.resolution; + if (resolution != nil) { + _screenSize = CGSizeMake(resolution.resolutionWidth.floatValue, + resolution.resolutionHeight.floatValue); + } else { + NSLog(@"Could not retrieve screen size. Defaulting to 640 x 480."); + _screenSize = CGSizeMake(640, + 480); + } + + return self; + +} + - (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol { - self = [super init]; + self = [self init]; if (!self) { return nil; } - _compressionSession = NULL; + _protocol = protocol; + return self; +} + +- (instancetype)init { + self = [super init]; + if (!self) { + return nil; + } + + _compressionSession = NULL; + _currentFrameNumber = 0; _videoSessionConnected = NO; _audioSessionConnected = NO; - _protocol = protocol; - + _videoStartBlock = nil; _audioStartBlock = nil; - - _touchManager = [[SDLTouchManager alloc] init]; + + _screenSize = CGSizeMake(640, 480); + _videoEncoderSettings = self.defaultVideoEncoderSettings; + _touchManager = [[SDLTouchManager alloc] init]; [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sdl_applicationDidEnterBackground:) @@ -80,7 +118,7 @@ - (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol { - (void)startVideoSessionWithStartBlock:(SDLStreamingStartBlock)startBlock { if (SDL_SYSTEM_VERSION_LESS_THAN(@"8.0")) { NSAssert(NO, @"SDL Video Sessions can only be run on iOS 8+ devices"); - startBlock(NO, [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLSTreamingVideoErrorInvalidOperatingSystemVersion userInfo:nil]); + startBlock(NO, [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorInvalidOperatingSystemVersion userInfo:nil]); return; } @@ -141,6 +179,31 @@ - (BOOL)sendAudioData:(NSData *)pcmAudioData { return YES; } +#pragma mark - Update video encoder + +- (void)setVideoEncoderSettings:(NSDictionary * _Nullable)videoEncoderSettings { + if (self.videoSessionConnected) { + @throw [NSException exceptionWithName:SDLErrorDomainStreamingMediaVideo reason:@"Cannot update video encoder settings while video session is connected." userInfo:nil]; + return; + } + + if (videoEncoderSettings) { + _videoEncoderSettings = videoEncoderSettings; + } else { + _videoEncoderSettings = self.defaultVideoEncoderSettings; + } +} + +- (NSDictionary*)defaultVideoEncoderSettings { + static NSDictionary* defaultVideoEncoderSettings = nil; + if (defaultVideoEncoderSettings == nil) { + defaultVideoEncoderSettings = @{ + (__bridge NSString*)kVTCompressionPropertyKey_ProfileLevel : (__bridge NSString*)kVTProfileLevel_H264_Baseline_AutoLevel, + (__bridge NSString*)kVTCompressionPropertyKey_RealTime : @YES + }; + } + return defaultVideoEncoderSettings; +} #pragma mark - SDLProtocolListener Methods @@ -238,8 +301,7 @@ - (BOOL)sdl_configureVideoEncoderWithError:(NSError *__autoreleasing *)error { OSStatus status; // Create a compression session - // TODO (Joel F.)[2015-08-18]: Dimensions should be from the Head Unit - status = VTCompressionSessionCreate(NULL, 640, 480, kCMVideoCodecType_H264, NULL, NULL, NULL, &sdl_videoEncoderOutputCallback, (__bridge void *)self, &_compressionSession); + status = VTCompressionSessionCreate(NULL, self.screenSize.width, self.screenSize.height, kCMVideoCodecType_H264, NULL, NULL, NULL, &sdl_videoEncoderOutputCallback, (__bridge void *)self, &_compressionSession); if (status != noErr) { // TODO: Log the error @@ -250,24 +312,41 @@ - (BOOL)sdl_configureVideoEncoderWithError:(NSError *__autoreleasing *)error { return NO; } - // Set the profile level of the video stream - status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel); + // Validate that the video encoder properties are valid. + CFDictionaryRef supportedProperties; + status = VTSessionCopySupportedPropertyDictionary(self.compressionSession, &supportedProperties); if (status != noErr) { if (*error != nil) { *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{ @"OSStatus" : @(status) }]; } - + return NO; } - - // Set the session to compress in real time - status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); - if (status != noErr) { - if (*error != nil) { - *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{ @"OSStatus" : @(status) }]; + + for (NSString* key in self.videoEncoderSettings.allKeys) { + if (CFDictionaryContainsKey(supportedProperties, (__bridge CFStringRef)key) == false) { + if (*error != nil) { + NSString* description = [NSString stringWithFormat:@"\"%@\" is not a supported key.", key]; + *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{NSLocalizedDescriptionKey : description}]; + } + CFRelease(supportedProperties); + return NO; + } + } + CFRelease(supportedProperties); + + // Populate the video encoder settings from provided dictionary. + for (NSString* key in self.videoEncoderSettings.allKeys) { + id value = self.videoEncoderSettings[key]; + + status = VTSessionSetProperty(self.compressionSession, (__bridge CFStringRef)key, (__bridge CFTypeRef)value); + if (status != noErr) { + if (*error != nil) { + *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{ @"OSStatus" : @(status) }]; + } + + return NO; } - - return NO; } return YES;