[release/11.7.1] Release 11.7.1 (#88)

* - OCDAVRawResponse: container for capturing raw WebDAV PROPFIND responses
- add update scan timing logging
- add support for update scan interval (and disable it for now)
- add support for infinite PROPFIND requests, introduce OCPropfindDepth
- OCSQLiteDB: add method to queue blocks on the DB thread
- OCXMLParser: add support for live processing of parsing results and add further abstraction for error and parse result handling
- OCBookmark, OCVault: add methods to prepopulate account databases from infinite PROPFIND responses
- various smaller improvements

* - OCCore+ItemList: scan for changes no longer uses the background URL session to avoid APM redirect issues

* - OCHTTP: add support for streaming responses as NSInputStream (+ prepare for more variants)
- OCConnection: add support for streaming PROPFIND responses
- OCVault+Prepopulation: add methods for streaming metadata and parsing the stream at the same time
- OCBookmark+Prepopulation: add method for streaming prepopulation

* - OCCapabilities: prepare support for new "supportsInfinitePropfind" capability
- OCHTTPStatus: correct spelling of Method Not Allowedc status code

* - add localizable strings for account prepopulation progress messages

* - OCClassSettings improvements
	- extend -keysForClass from documentation to --keysForClass:options: for wider use
	- settingsSnapshotForClasses now uses -keysForClass:options: instead of trying to compile a definitive list of keys itself
	- adapt calling code accordingly

* - fix DAVRawResponseTests comment error

* - support for custom poll interval for changes:
	- through capabilities: mirroring changes from owncloud/client#8777
	- through MDM via `core.scan-for-changes-interval`
	- in milliseconds
	- defaulting to 10 seconds
	- enforcing (and logging a warning) for intervals below a minimum of 5 seconds
	- logging a warning for intervals greater than 60 seconds

* - improved configurable scan for changes poll interval
	- determine effective poll interval only once / initialized core
	- limit logging of warnings and errors to once / initialized core
	- improve wording and verbosity of log messages
	- clean up code

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 11.7.1 version
+
+- support for streaming, infinite PROPFIND to prepopulate accounts and speed up initial discovery
+- minimum interval between two scans for changes can now be configured via MDM and serverside via capabilities
+- fix crash happening during class settings discovery
+- add streaming support to OCXMLParser
+
 ## 11.7 version
 
 - Scan for changes no longer uses the background URL session, so redirects can be fully managed by the SDK

ownCloudSDK/Authentication/Browser Session/MIBrowser/OCAuthenticationBrowserSessionMIBrowser.m

@@ -6,6 +6,16 @@
 //  Copyright © 2021 ownCloud GmbH. All rights reserved.
 //
 
+/*
+ * Copyright (C) 2021, ownCloud GmbH.
+ *
+ * This code is covered by the GNU Public License Version 3.
+ *
+ * For distribution utilizing Apple mechanisms please see https://owncloud.org/contribute/iOS-license-exception/
+ * You should have received a copy of this license along with this program. If not, see <http://www.gnu.org/licenses/gpl-3.0.en.html>.
+ *
+ */
+
 #import "OCAuthenticationBrowserSessionMIBrowser.h"
 
 @implementation OCAuthenticationBrowserSessionMIBrowser

ownCloudSDK/Connection/Capabilities/OCCapabilities.h

@@ -73,6 +73,7 @@ typedef NSNumber* OCCapabilityBool;
 @property(readonly,nullable,nonatomic) NSArray<NSString *> *blacklistedFiles;
 @property(readonly,nullable,nonatomic) OCCapabilityBool supportsUndelete;
 @property(readonly,nullable,nonatomic) OCCapabilityBool supportsVersioning;
+@property(readonly,nullable,nonatomic) OCCapabilityBool supportsInfinitePropfind;
 
 #pragma mark - Sharing
 @property(readonly,nullable,nonatomic) OCCapabilityBool sharingAPIEnabled;

ownCloudSDK/Connection/Capabilities/OCCapabilities.m

@@ -78,6 +78,7 @@ @implementation OCCapabilities
 @dynamic blacklistedFiles;
 @dynamic supportsUndelete;
 @dynamic supportsVersioning;
+@dynamic supportsInfinitePropfind;
 
 #pragma mark - Sharing
 @dynamic sharingAPIEnabled;
@@ -343,6 +344,12 @@ - (OCCapabilityBool)supportsVersioning
 	return (OCTypedCast(_capabilities[@"files"][@"versioning"], NSNumber));
 }
 
+- (OCCapabilityBool)supportsInfinitePropfind
+{
+	// Prepared for future server-side capability
+	return (nil);
+}
+
 #pragma mark - Sharing
 - (OCCapabilityBool)sharingAPIEnabled
 {

ownCloudSDK/Connection/DAVResponse/OCDAVRawResponse.h

@@ -0,0 +1,30 @@
+//
+//  OCDAVRawResponse.h
+//  ownCloudSDK
+//
+//  Created by Felix Schwarz on 24.06.21.
+//  Copyright © 2021 ownCloud GmbH. All rights reserved.
+//
+
+/*
+ * Copyright (C) 2021, ownCloud GmbH.
+ *
+ * This code is covered by the GNU Public License Version 3.
+ *
+ * For distribution utilizing Apple mechanisms please see https://owncloud.org/contribute/iOS-license-exception/
+ * You should have received a copy of this license along with this program. If not, see <http://www.gnu.org/licenses/gpl-3.0.en.html>.
+ *
+ */
+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface OCDAVRawResponse : NSObject <NSSecureCoding>
+
+@property(strong,nullable) NSURL *responseDataURL;
+@property(strong,nullable) NSString *basePath;
+
+@end
+
+NS_ASSUME_NONNULL_END

ownCloudSDK/Connection/DAVResponse/OCDAVRawResponse.m

@@ -0,0 +1,47 @@
+//
+//  OCDAVRawResponse.m
+//  ownCloudSDK
+//
+//  Created by Felix Schwarz on 24.06.21.
+//  Copyright © 2021 ownCloud GmbH. All rights reserved.
+//
+
+/*
+ * Copyright (C) 2021, ownCloud GmbH.
+ *
+ * This code is covered by the GNU Public License Version 3.
+ *
+ * For distribution utilizing Apple mechanisms please see https://owncloud.org/contribute/iOS-license-exception/
+ * You should have received a copy of this license along with this program. If not, see <http://www.gnu.org/licenses/gpl-3.0.en.html>.
+ *
+ */
+
+#import "OCDAVRawResponse.h"
+
+@implementation OCDAVRawResponse
+
+#pragma mark - NSSecureCoding
++ (BOOL)supportsSecureCoding
+{
+	return(YES);
+}
+
+- (instancetype)initWithCoder:(NSCoder *)decoder
+{
+	if ((self = [self init]) != nil)
+	{
+		_responseDataURL = [decoder decodeObjectOfClass:NSURL.class forKey:@"responseDataURL"];
+		_basePath = [decoder decodeObjectOfClass:NSString.class forKey:@"basePath"];
+	}
+
+	return (self);
+}
+
+- (void)encodeWithCoder:(NSCoder *)coder
+{
+	[coder encodeObject:_responseDataURL forKey:@"responseDataURL"];
+	[coder encodeObject:_basePath forKey:@"basePath"];
+}
+
+@end
+

ownCloudSDK/Connection/OCConnection.h

@@ -421,6 +421,8 @@ extern OCConnectionOptionKey OCConnectionOptionRequiredSignalsKey; //!< NSSet<OC
 extern OCConnectionOptionKey OCConnectionOptionRequiredCellularSwitchKey; //!< OCCellularSwitchIdentifier to require for the requests.
 extern OCConnectionOptionKey OCConnectionOptionTemporarySegmentFolderURLKey; //!< NSURL of the temporary folder to store file segments in when performing uploads via TUS
 extern OCConnectionOptionKey OCConnectionOptionForceReplaceKey; //!< If YES, force replace existing items.
+extern OCConnectionOptionKey OCConnectionOptionResponseDestinationURL; //!< NSURL of where to store a (raw) response
+extern OCConnectionOptionKey OCConnectionOptionResponseStreamHandler; //!< Response stream handler (OCHTTPRequestEphermalStreamHandler) to receive the response body stream
 
 extern OCConnectionSignalID OCConnectionSignalIDAuthenticationAvailable; //!< Signal indicating that authentication is required for this request
 

ownCloudSDK/Connection/OCConnection.m

@@ -48,6 +48,7 @@
 #import "OCHTTPPolicyBookmark.h"
 #import "OCHTTPRequest.h"
 #import "NSURL+OCURLNormalization.h"
+#import "OCDAVRawResponse.h"
 
 // Imported to use the identifiers in OCConnectionPreferredAuthenticationMethodIDs only
 #import "OCAuthenticationMethodOpenIDConnect.h"
@@ -1786,6 +1787,15 @@ - (NSProgress *)retrieveItemListAtPath:(OCPath)path depth:(NSUInteger)depth opti
 	{
 		if ((davRequest = [self _propfindDAVRequestForPath:path endpointURL:endpointURL depth:depth]) != nil)
 		{
+			OCHTTPRequestEphermalStreamHandler ephermalStreamHandler = nil;
+
+			if ((ephermalStreamHandler = options[OCConnectionOptionResponseStreamHandler]) != nil)
+			{
+				// Remove block from options as it can't be serialized otherwise
+				options = [options mutableCopy];
+				[(NSMutableDictionary *)options removeObjectForKey:OCConnectionOptionResponseStreamHandler];
+			}
+
 			// davRequest.requiredSignals = self.actionSignals;
 			davRequest.resultHandlerAction = @selector(_handleRetrieveItemListAtPathResult:error:);
 			davRequest.userInfo = @{
@@ -1819,6 +1829,17 @@ - (NSProgress *)retrieveItemListAtPath:(OCPath)path depth:(NSUInteger)depth opti
 				davRequest.requiredSignals = options[OCConnectionOptionRequiredSignalsKey];
 			}
 
+			if (options[OCConnectionOptionResponseDestinationURL] != nil)
+			{
+				davRequest.downloadedFileURL = options[OCConnectionOptionResponseDestinationURL];
+			}
+
+			if (ephermalStreamHandler != nil)
+			{
+				davRequest.ephermalStreamHandler = ephermalStreamHandler;
+				davRequest.downloadRequest = NO;
+			}
+
 			// Attach to pipelines
 			[self attachToPipelines];
 
@@ -1862,6 +1883,7 @@ - (void)_handleRetrieveItemListAtPathResult:(OCHTTPRequest *)request error:(NSEr
 	OCEvent *event;
 	NSDictionary<OCConnectionOptionKey,id> *options = [request.userInfo[@"options"] isKindOfClass:[NSDictionary class]] ? request.userInfo[@"options"] : nil;
 	OCEventType eventType = OCEventTypeRetrieveItemList;
+	NSURL *responseDestinationURL = options[OCConnectionOptionResponseDestinationURL];
 
 	if ((options!=nil) && (options[@"alternativeEventType"]!=nil))
 	{
@@ -1870,6 +1892,8 @@ - (void)_handleRetrieveItemListAtPathResult:(OCHTTPRequest *)request error:(NSEr
 
 	if ((event = [OCEvent eventForEventTarget:request.eventTarget type:eventType uuid:request.identifier attributes:nil]) != nil)
 	{
+		NSURL *endpointURL = request.userInfo[@"endpointURL"];
+
 		if (error != nil)
 		{
 			event.error = error;
@@ -1885,9 +1909,25 @@ - (void)_handleRetrieveItemListAtPathResult:(OCHTTPRequest *)request error:(NSEr
 			event.error = request.httpResponse.status.error;
 		}
 
-		if (event.error == nil)
+		if ((event.error == nil) && (responseDestinationURL != nil))
+		{
+			if ([NSFileManager.defaultManager fileExistsAtPath:responseDestinationURL.path])
+			{
+				OCDAVRawResponse *rawResponse = [OCDAVRawResponse new];
+
+				rawResponse.responseDataURL = responseDestinationURL;
+				rawResponse.basePath = endpointURL.path;
+
+				event.result = rawResponse;
+			}
+			else
+			{
+				event.error = OCError(OCErrorFileNotFound);
+			}
+		}
+
+		if ((event.error == nil) && (responseDestinationURL == nil))
 		{
-			NSURL *endpointURL = request.userInfo[@"endpointURL"];
 			NSArray <NSError *> *errors = nil;
 			NSArray <OCItem *> *items = nil;
 
@@ -3147,6 +3187,8 @@ - (NSError *)sendSynchronousRequest:(OCHTTPRequest *)request
 OCConnectionOptionKey OCConnectionOptionRequiredCellularSwitchKey = @"required-cellular-switch";
 OCConnectionOptionKey OCConnectionOptionTemporarySegmentFolderURLKey = @"temporary-segment-folder-url";
 OCConnectionOptionKey OCConnectionOptionForceReplaceKey = @"force-replace";
+OCConnectionOptionKey OCConnectionOptionResponseDestinationURL = @"response-destination-url";
+OCConnectionOptionKey OCConnectionOptionResponseStreamHandler = @"response-stream-handler";
 
 OCConnectionSignalID OCConnectionSignalIDAuthenticationAvailable = @"authAvailable";
 

ownCloudSDK/Core/ItemList/OCCore+ItemList.m

@@ -1204,6 +1204,12 @@ - (void)queueRequestJob:(OCAsyncSequentialQueueJob)requestJob
 - (void)startCheckingForUpdates
 {
 	[self queueBlock:^{
+		if (self->_directoryUpdateStartTime == 0)
+		{
+			OCTLog(@[@"UpdateScan"], @"Starting update scan");
+			self->_directoryUpdateStartTime = NSDate.timeIntervalSinceReferenceDate;
+		}
+
 		[self _checkForUpdatesNotBefore:nil inBackground:NO completionHandler:nil];
 	}];
 }
@@ -1445,25 +1451,95 @@ - (void)_handleRetrieveItemListEvent:(OCEvent *)event sender:(id)sender
 	// Schedule next
 	if ((event.depth == 0) && ([event.path isEqual:@"/"]))
 	{
-		// Check again after configured time interval (with fall back to 10 seconds)
-		NSNumber *configuredInterval = [self classSettingForOCClassSettingsKey:OCCoreScanForChangesInterval];
-		NSTimeInterval minimumTimeInterval = (configuredInterval.integerValue > 0) ? configuredInterval.doubleValue : 10;
+		// Check again after configured time interval
+		NSTimeInterval pollInterval = self.effectivePollForChangesInterval;
 
 		if (self.state == OCCoreStateRunning)
 		{
 			@synchronized([OCCoreItemList class])
 			{
-				if ((_lastScheduledItemListUpdateDate==nil) || ([_lastScheduledItemListUpdateDate timeIntervalSinceNow]<-(minimumTimeInterval-1)))
+				if ((_lastScheduledItemListUpdateDate==nil) || (_lastScheduledItemListUpdateDate.timeIntervalSinceNow < -(pollInterval-1.0)))
 				{
 					_lastScheduledItemListUpdateDate = [NSDate date];
 
-					[self _checkForUpdatesNotBefore:[NSDate dateWithTimeIntervalSinceNow:minimumTimeInterval] inBackground:NO completionHandler:nil];
+					[self _checkForUpdatesNotBefore:[NSDate dateWithTimeIntervalSinceNow:pollInterval] inBackground:NO completionHandler:nil];
 				}
 			}
 		}
 	}
 }
 
+- (NSTimeInterval)effectivePollForChangesInterval
+{
+	if (_effectivePollForChangesInterval == 0)
+	{
+		const NSTimeInterval defaultMinimumPollInterval = 10.0, minimumAllowedPollInterval = 5.0, warnPollIntervalThreshold = 60.0;
+		NSTimeInterval effectivePollInterval = defaultMinimumPollInterval;
+		NSString *effectivePollIntervalSource = @"default";
+		BOOL loggedPollIntervalWarning = NO;
+
+		// Capabilities
+		if (self.connection.capabilities.pollInterval != nil)
+		{
+			// Server default is 60 seconds, but iOS default is 10 seconds
+			// also the capability is no longer in seconds, but milliseconds,
+			// so ignore anything less than 5 seconds, warn for anything greater
+			// than 60 seconds
+
+			NSTimeInterval configuredTimeInterval = self.connection.capabilities.pollInterval.doubleValue / 1000.0;
+
+			if (configuredTimeInterval < minimumAllowedPollInterval)
+			{
+				if (self.connection.capabilities.pollInterval.integerValue != 60)
+				{
+					OCTLogError(@[@"PollForChanges"], @"Poll interval in capabilities (%@) not server legacy default (60 (sec)), and - as milliseconds - less than minimum allowed poll interval (%.02f sec). Ignoring value.", self.connection.capabilities.pollInterval, minimumAllowedPollInterval);
+					loggedPollIntervalWarning = YES;
+				}
+			}
+			else
+			{
+				effectivePollInterval = configuredTimeInterval;
+				effectivePollIntervalSource = @"capabilities";
+			}
+		}
+
+		// Class Settings
+		NSNumber *classSettingsInterval;
+
+		if ((classSettingsInterval = [self classSettingForOCClassSettingsKey:OCCoreScanForChangesInterval]) != nil)
+		{
+			NSTimeInterval configuredTimeInterval = classSettingsInterval.doubleValue / 1000.0;
+
+			if (configuredTimeInterval < minimumAllowedPollInterval)
+			{
+				OCTLogError(@[@"PollForChanges"], @"MDM/Branding: poll interval %.03f less than minimum allowed poll interval (%.02f sec). Ignoring value.", configuredTimeInterval, minimumAllowedPollInterval);
+				loggedPollIntervalWarning = YES;
+			}
+			else
+			{
+				effectivePollInterval = configuredTimeInterval;
+				effectivePollIntervalSource = @"ClassSettings";
+			}
+		}
+
+		// Log warning when exceeding threshold
+		if (effectivePollInterval > warnPollIntervalThreshold)
+		{
+			OCTLogWarning(@[@"PollForChanges"], @"Poll interval (%@) of %.02f sec > %.02f sec", effectivePollIntervalSource, effectivePollInterval, warnPollIntervalThreshold);
+			loggedPollIntervalWarning = YES;
+		}
+
+		if (loggedPollIntervalWarning)
+		{
+			OCTLog(@[@"PollForChanges"], @"Using poll interval of %.02f sec (%@)", effectivePollInterval, effectivePollIntervalSource);
+		}
+
+		_effectivePollForChangesInterval = effectivePollInterval;
+	}
+
+	return (_effectivePollForChangesInterval);
+}
+
 #pragma mark - Update Scan finish
 - (void)_finishedUpdateScanWithError:(nullable NSError *)error foundChanges:(BOOL)foundChanges
 {
@@ -1486,6 +1562,12 @@ - (void)_finishedUpdateScanWithError:(nullable NSError *)error foundChanges:(BOO
 		[self runProtectedPolicyProcessorsForTrigger:OCItemPolicyProcessorTriggerItemListUpdateCompletedWithoutChanges];
 	}
 
+	if (_directoryUpdateStartTime != 0)
+	{
+		OCTLog(@[@"UpdateScan"], @"Finished update scan in %.1f sec", NSDate.timeIntervalSinceReferenceDate - _directoryUpdateStartTime);
+		_directoryUpdateStartTime = 0;
+	}
+
 	for (OCCoreItemListFetchUpdatesCompletionHandler completionHandler in completionHandlers)
 	{
 		completionHandler(error, foundChanges);

ownCloudSDK/Core/OCCore.h

@@ -178,6 +178,8 @@ typedef id<NSObject> OCCoreItemTracking;
 
 	NSDate *_nextSchedulingDate;
 
+	NSTimeInterval _effectivePollForChangesInterval;
+
 	OCActivityManager *_activityManager;
 	NSMutableSet <OCSyncRecordID> *_publishedActivitySyncRecordIDs;
 	BOOL _needsToBroadcastSyncRecordActivityUpdates;
@@ -199,6 +201,7 @@ typedef id<NSObject> OCCoreItemTracking;
 	NSUInteger _pendingScheduledDirectoryUpdateJobs;
 	OCAsyncSequentialQueue *_itemListTasksRequestQueue;
 	BOOL _itemListTaskRunning;
+	NSTimeInterval _directoryUpdateStartTime;
 	NSMutableArray<OCCoreItemListFetchUpdatesCompletionHandler> *_fetchUpdatesCompletionHandlers;
 
 	NSMutableArray <OCItemPolicy *> *_itemPolicies;