Custom video encoder settings and dynamic screen sizes

SmartDeviceLink/SDLProxy.m

@@ -29,7 +29,7 @@
 #import "SDLProtocolMessage.h"
 #import "SDLProtocolMessage.h"
 #import "SDLPutFile.h"
-#import "SDLRPCPayload.h"
+#import "SDLRegisterAppInterfaceResponse.h"
 #import "SDLRPCPayload.h"
 #import "SDLRPCRequestFactory.h"
 #import "SDLRPCResponse.h"
@@ -56,7 +56,8 @@ @interface SDLProxy () {
 }
 
 @property (strong, nonatomic) NSMutableSet *mutableProxyListeners;
-@property (nonatomic, strong, readwrite) SDLStreamingMediaManager *streamingMediaManager;
+@property (nonatomic, strong, readwrite, nullable) SDLStreamingMediaManager *streamingMediaManager;
+@property (nonatomic, strong, nullable) SDLDisplayCapabilities* displayCapabilities;
 
 @end
 
@@ -101,6 +102,8 @@ - (void)destructObjects {
         _transport = nil;
         _protocol = nil;
         _mutableProxyListeners = nil;
+        _streamingMediaManager = nil;
+        _displayCapabilities = nil;
     }
 }
 
@@ -166,7 +169,10 @@ - (NSString *)proxyVersion {
 
 - (SDLStreamingMediaManager *)streamingMediaManager {
     if (_streamingMediaManager == nil) {
-        _streamingMediaManager = [[SDLStreamingMediaManager alloc] initWithProtocol:self.protocol];
+        if (self.displayCapabilities == nil) {
+            @throw [NSException exceptionWithName:NSInvalidArgumentException reason:@"SDLStreamingMediaManager must be accessed only after a successful RegisterAppInterfaceResponse" userInfo:nil];
+        }
+        _streamingMediaManager = [[SDLStreamingMediaManager alloc] initWithProtocol:self.protocol displayCapabilities:self.displayCapabilities];
         [self.protocol.protocolDelegateTable addObject:_streamingMediaManager];
     }
 
@@ -328,6 +334,10 @@ - (void)handleRegisterAppInterfaceResponse:(SDLRPCResponse *)response {
         [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sendMobileHMIState) name:UIApplicationDidBecomeActiveNotification object:nil];
         [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sendMobileHMIState) name:UIApplicationDidEnterBackgroundNotification object:nil];
     }
+
+    // Extract the display capabilties to successfully build SDLStreamingMediaManager's video encoder.
+    SDLRegisterAppInterfaceResponse* registerResponse = (SDLRegisterAppInterfaceResponse*)response;
+    self.displayCapabilities = registerResponse.displayCapabilities;
 }
 
 - (void)handleSyncPData:(SDLRPCMessage *)message {

SmartDeviceLink/SDLStreamingMediaManager.h

@@ -12,6 +12,7 @@
 #import "SDLProtocolListener.h"
 
 @class SDLAbstractProtocol;
+@class SDLDisplayCapabilities;
 
 
 NS_ASSUME_NONNULL_BEGIN
@@ -36,7 +37,9 @@ typedef void (^SDLStreamingStartBlock)(BOOL success, NSError *__nullable error);
 
 @interface SDLStreamingMediaManager : NSObject <SDLProtocolListener>
 
-- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol;
+- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol __deprecated_msg(("Please use initWithProtocol:displayCapabilities: instead"));
+
+- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol displayCapabilities:(SDLDisplayCapabilities*)displayCapabilities;
 
 /**
  *  This method will attempt to start a streaming video session. It will set up iOS's video encoder,  and call out to the head unit asking if it will start a video session.
@@ -85,6 +88,22 @@ typedef void (^SDLStreamingStartBlock)(BOOL success, NSError *__nullable error);
 @property (assign, nonatomic, readonly) BOOL videoSessionConnected;
 @property (assign, nonatomic, readonly) BOOL audioSessionConnected;
 
+/**
+ *  The settings used in a VTCompressionSessionRef encoder. These will be verified when the video stream is started. Acceptable properties for this are located in VTCompressionProperties. If set to nil, the defaultVideoEncoderSettings will be used.
+ *
+ *  @warning Video streaming must not be connected to update the encoder properties. If it is running, issue a stopVideoSession before updating.
+ */
+@property (strong, nonatomic, null_resettable) NSDictionary* videoEncoderSettings;
+
+/**
+ *  Provides default video encoder settings used.
+ */
+@property (strong, nonatomic, readonly) NSDictionary* defaultVideoEncoderSettings;
+
+/**
+ *  This is the current screen size of a connected display. This will be the size the video encoder uses to encode the raw image data.
+ */
+@property (assign, nonatomic, readonly) CGSize screenSize;
 
 @end
 

SmartDeviceLink/SDLStreamingMediaManager.m

@@ -11,8 +11,10 @@
 @import UIKit;
 
 #import "SDLAbstractProtocol.h"
+#import "SDLDisplayCapabilities.h"
 #import "SDLGlobals.h"
-
+#import "SDLImageResolution.h"
+#import "SDLScreenParams.h"
 
 NSString *const SDLErrorDomainStreamingMediaVideo = @"com.sdl.streamingmediamanager.video";
 NSString *const SDLErrorDomainStreamingMediaAudio = @"com.sdl.streamingmediamanager.audio";
@@ -41,21 +43,57 @@ @implementation SDLStreamingMediaManager
 
 #pragma mark - Class Lifecycle
 
+- (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol displayCapabilities:(SDLDisplayCapabilities*)displayCapabilities {
+    self = [self init];
+    if (!self) {
+        return nil;
+    }
+
+    _protocol = protocol;
+
+    SDLImageResolution* resolution = displayCapabilities.screenParams.resolution;
+    if (resolution != nil) {
+        _screenSize = CGSizeMake(resolution.resolutionWidth.floatValue,
+                                 resolution.resolutionHeight.floatValue);
+    } else {
+        NSLog(@"Could not retrieve screen size. Defaulting to 640 x 480.");
+        _screenSize = CGSizeMake(640,
+                                 480);
+    }
+
+    return self;
+
+}
+
 - (instancetype)initWithProtocol:(SDLAbstractProtocol *)protocol {
-    self = [super init];
+    self = [self init];
     if (!self) {
         return nil;
     }
 
-    _compressionSession = NULL;
+    _protocol = protocol;
+
+    return self;
+}
 
+- (instancetype)init {
+    self = [super init];
+    if (!self) {
+        return nil;
+    }
+
+    _compressionSession = NULL;
+
     _currentFrameNumber = 0;
     _videoSessionConnected = NO;
     _audioSessionConnected = NO;
-    _protocol = protocol;
-
+
     _videoStartBlock = nil;
     _audioStartBlock = nil;
+
+    _screenSize = CGSizeMake(640, 480);
+
+    _videoEncoderSettings = self.defaultVideoEncoderSettings;
 
     return self;
 }
@@ -127,6 +165,31 @@ - (BOOL)sendAudioData:(NSData *)pcmAudioData {
     return YES;
 }
 
+#pragma mark - Update video encoder
+
+- (void)setVideoEncoderSettings:(NSDictionary * _Nullable)videoEncoderSettings {
+    if (self.videoSessionConnected) {
+        @throw [NSException exceptionWithName:SDLErrorDomainStreamingMediaVideo reason:@"Cannot update video encoder settings while video session is connected." userInfo:nil];
+        return;
+    }
+
+    if (videoEncoderSettings) {
+        _videoEncoderSettings = videoEncoderSettings;
+    } else {
+        _videoEncoderSettings = self.defaultVideoEncoderSettings;
+    }
+}
+
+- (NSDictionary*)defaultVideoEncoderSettings {
+    static NSDictionary* defaultVideoEncoderSettings = nil;
+    if (defaultVideoEncoderSettings == nil) {
+        defaultVideoEncoderSettings = @{
+                                        (__bridge NSString*)kVTCompressionPropertyKey_ProfileLevel : (__bridge NSString*)kVTProfileLevel_H264_Baseline_AutoLevel,
+                                        (__bridge NSString*)kVTCompressionPropertyKey_RealTime : @YES
+                                        };
+    }
+    return defaultVideoEncoderSettings;
+}
 
 #pragma mark - SDLProtocolListener Methods
 
@@ -224,8 +287,7 @@ - (BOOL)sdl_configureVideoEncoderWithError:(NSError *__autoreleasing *)error {
     OSStatus status;
 
     // Create a compression session
-    // TODO (Joel F.)[2015-08-18]: Dimensions should be from the Head Unit
-    status = VTCompressionSessionCreate(NULL, 640, 480, kCMVideoCodecType_H264, NULL, NULL, NULL, &sdl_videoEncoderOutputCallback, (__bridge void *)self, &_compressionSession);
+    status = VTCompressionSessionCreate(NULL, self.screenSize.width, self.screenSize.height, kCMVideoCodecType_H264, NULL, NULL, NULL, &sdl_videoEncoderOutputCallback, (__bridge void *)self, &_compressionSession);
 
     if (status != noErr) {
         // TODO: Log the error
@@ -236,24 +298,41 @@ - (BOOL)sdl_configureVideoEncoderWithError:(NSError *__autoreleasing *)error {
         return NO;
     }
 
-    // Set the profile level of the video stream
-    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
+    // Validate that the video encoder properties are valid.
+    CFDictionaryRef supportedProperties;
+    status = VTSessionCopySupportedPropertyDictionary(self.compressionSession, &supportedProperties);
     if (status != noErr) {
         if (*error != nil) {
             *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{ @"OSStatus" : @(status) }];
         }
-
+        
         return NO;
     }
-
-    // Set the session to compress in real time
-    status = VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
-    if (status != noErr) {
-        if (*error != nil) {
-            *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{ @"OSStatus" : @(status) }];
+
+    for (NSString* key in self.videoEncoderSettings.allKeys) {
+        if (CFDictionaryContainsKey(supportedProperties, (__bridge CFStringRef)key) == false) {
+            if (*error != nil) {
+                NSString* description = [NSString stringWithFormat:@"\"%@\" is not a supported key.", key];
+                *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{NSLocalizedDescriptionKey : description}];
+            }
+            CFRelease(supportedProperties);
+            return NO;
+        }
+    }
+    CFRelease(supportedProperties);
+
+    // Populate the video encoder settings from provided dictionary.
+    for (NSString* key in self.videoEncoderSettings.allKeys) {
+        id value = self.videoEncoderSettings[key];
+
+        status = VTSessionSetProperty(self.compressionSession, (__bridge CFStringRef)key, (__bridge CFTypeRef)value);
+        if (status != noErr) {
+            if (*error != nil) {
+                *error = [NSError errorWithDomain:SDLErrorDomainStreamingMediaVideo code:SDLStreamingVideoErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{ @"OSStatus" : @(status) }];
+            }
+
+            return NO;
         }
-
-        return NO;
     }
 
     return YES;