Logic Tests - Collect and send log files over to client

Summary: Tests standard outputs (e.g. NSLogs and/or printfs) are collected and reported back to idb client.

Reviewed By: phyllipecesar

Differential Revision: D27367868

fbshipit-source-id: d02541ed4b26918c38364108076de409d3b22512

XCTestBootstrap/Configuration/FBXCTestConfiguration.h

@@ -186,10 +186,15 @@ typedef NS_OPTIONS(NSUInteger, FBLogicTestMirrorLogs) {
  */
 @property (nonatomic, readonly) FBLogicTestMirrorLogs mirroring;
 
+/**
+ The Directory to use for storing logs generated during the execution of the test run.
+ */
+@property (nonatomic, nullable, copy, readonly) NSString *logDirectoryPath;
+
 /**
  The Designated Initializer.
  */
-+ (instancetype)configurationWithShims:(FBXCTestShimConfiguration *)shims environment:(NSDictionary<NSString *, NSString *> *)environment workingDirectory:(NSString *)workingDirectory testBundlePath:(NSString *)testBundlePath waitForDebugger:(BOOL)waitForDebugger timeout:(NSTimeInterval)timeout testFilter:(nullable NSString *)testFilter mirroring:(FBLogicTestMirrorLogs)mirroring;
++ (instancetype)configurationWithShims:(FBXCTestShimConfiguration *)shims environment:(NSDictionary<NSString *, NSString *> *)environment workingDirectory:(NSString *)workingDirectory logDirectoryPath:(nullable NSString *)logDirectoryPath testBundlePath:(NSString *)testBundlePath waitForDebugger:(BOOL)waitForDebugger timeout:(NSTimeInterval)timeout testFilter:(nullable NSString *)testFilter mirroring:(FBLogicTestMirrorLogs)mirroring;
 
 @end
 

XCTestBootstrap/Configuration/FBXCTestConfiguration.m

@@ -247,12 +247,12 @@ @implementation FBLogicTestConfiguration
 
 #pragma mark Initializers
 
-+ (instancetype)configurationWithShims:(FBXCTestShimConfiguration *)shims environment:(NSDictionary<NSString *, NSString *> *)environment workingDirectory:(NSString *)workingDirectory testBundlePath:(NSString *)testBundlePath waitForDebugger:(BOOL)waitForDebugger timeout:(NSTimeInterval)timeout testFilter:(NSString *)testFilter mirroring:(FBLogicTestMirrorLogs)mirroring
++ (instancetype)configurationWithShims:(FBXCTestShimConfiguration *)shims environment:(NSDictionary<NSString *, NSString *> *)environment workingDirectory:(NSString *)workingDirectory logDirectoryPath:(nullable NSString *)logDirectoryPath testBundlePath:(NSString *)testBundlePath waitForDebugger:(BOOL)waitForDebugger timeout:(NSTimeInterval)timeout testFilter:(NSString *)testFilter mirroring:(FBLogicTestMirrorLogs)mirroring
 {
-  return [[FBLogicTestConfiguration alloc] initWithShims:shims environment:environment workingDirectory:workingDirectory testBundlePath:testBundlePath waitForDebugger:waitForDebugger timeout:timeout testFilter:testFilter  mirroring:mirroring];
+  return [[FBLogicTestConfiguration alloc] initWithShims:shims environment:environment workingDirectory:workingDirectory logDirectoryPath:logDirectoryPath testBundlePath:testBundlePath waitForDebugger:waitForDebugger timeout:timeout testFilter:testFilter  mirroring:mirroring];
 }
 
-- (instancetype)initWithShims:(FBXCTestShimConfiguration *)shims environment:(NSDictionary<NSString *, NSString *> *)environment workingDirectory:(NSString *)workingDirectory testBundlePath:(NSString *)testBundlePath waitForDebugger:(BOOL)waitForDebugger timeout:(NSTimeInterval)timeout testFilter:(NSString *)testFilter mirroring:(FBLogicTestMirrorLogs)mirroring
+- (instancetype)initWithShims:(FBXCTestShimConfiguration *)shims environment:(NSDictionary<NSString *, NSString *> *)environment workingDirectory:(NSString *)workingDirectory logDirectoryPath:(NSString *)logDirectoryPath testBundlePath:(NSString *)testBundlePath waitForDebugger:(BOOL)waitForDebugger timeout:(NSTimeInterval)timeout testFilter:(NSString *)testFilter mirroring:(FBLogicTestMirrorLogs)mirroring
 {
   self = [super initWithShims:shims environment:environment workingDirectory:workingDirectory testBundlePath:testBundlePath waitForDebugger:waitForDebugger timeout:timeout];
   if (!self) {
@@ -261,6 +261,7 @@ - (instancetype)initWithShims:(FBXCTestShimConfiguration *)shims environment:(NS
 
   _testFilter = testFilter;
   _mirroring = mirroring;
+  _logDirectoryPath = logDirectoryPath;
 
   return self;
 }

XCTestBootstrap/Reporters/FBJSONTestReporter.h

@@ -19,7 +19,7 @@ NS_ASSUME_NONNULL_BEGIN
 /**
  A Reporter using xctool's linewise-json output.
  */
-@interface FBJSONTestReporter : NSObject <FBXCTestReporter>
+@interface FBJSONTestReporter : NSObject <FBXCTestReporter, FBXCTestReporterWithFiles>
 
 /**
  The Designated Initializer.

XCTestBootstrap/Reporters/FBJSONTestReporter.m

@@ -332,6 +332,10 @@ - (NSString *)noStartOfTestPlanErrorMessage
   };
 }
 
+- (void)setLogDirectoryPath:(NSString *)logDirectoryPath
+{
+}
+
 @end
 
 static inline NSString *FBFullyFormattedXCTestName(NSString *className, NSString *methodName)

XCTestBootstrap/Reporters/FBLogicReporterAdapter.h

@@ -7,17 +7,17 @@
 
 #import <Foundation/Foundation.h>
 #import <XCTestBootstrap/FBLogicXCTestReporter.h>
+#import <XCTestBootstrap/FBXCTestReporter.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
-@protocol FBXCTestReporter;
 @protocol FBControlCoreLogger;
 
 /**
  This adapter parses streams of events in JSON and invokes
  the corresponding methods in the provided FBXCTestReporter
  */
-@interface FBLogicReporterAdapter : NSObject <FBLogicXCTestReporter>
+@interface FBLogicReporterAdapter : NSObject <FBLogicXCTestReporter, FBXCTestReporterWithFiles>
 
 /**
  The Designated Initializer.
@@ -26,7 +26,7 @@ NS_ASSUME_NONNULL_BEGIN
  @param logger an optional logger to log to,
  @return a new FBLogicXCTestReporter instance.
  */
-- (instancetype)initWithReporter:(id<FBXCTestReporter>)reporter logger:(nullable id<FBControlCoreLogger>)logger;
+- (instancetype)initWithReporter:(id<FBXCTestReporter, FBXCTestReporterWithFiles>)reporter logger:(nullable id<FBControlCoreLogger>)logger;
 
 @end
 

XCTestBootstrap/Reporters/FBLogicReporterAdapter.m

@@ -13,14 +13,14 @@
 
 @interface FBLogicReporterAdapter ()
 
-@property (nonatomic, readonly) id<FBXCTestReporter> reporter;
+@property (nonatomic, readonly) id<FBXCTestReporter, FBXCTestReporterWithFiles> reporter;
 @property (nonatomic, readonly) FBXCTestLogger *logger;
 
 @end
 
 @implementation FBLogicReporterAdapter
 
-- (instancetype)initWithReporter:(id<FBXCTestReporter>)reporter logger:(nullable id<FBControlCoreLogger>)logger
+- (instancetype)initWithReporter:(id<FBXCTestReporter, FBXCTestReporterWithFiles>)reporter logger:(nullable id<FBControlCoreLogger>)logger
 {
   self = [self init];
   if (!self) {
@@ -138,4 +138,8 @@ - (void)didCrashDuringTest:(NSError *)error
   }
 }
 
+- (void)setLogDirectoryPath:(NSString *)logDirectoryPath {
+    [self.reporter setLogDirectoryPath:logDirectoryPath];
+}
+
 @end

XCTestBootstrap/Reporters/FBXCTestReporter.h

@@ -170,4 +170,21 @@ NS_ASSUME_NONNULL_BEGIN
 
 @end
 
+
+
+/**
+ fbxtest's reporting protocol to handle log files.
+ */
+@protocol FBXCTestReporterWithFiles <NSObject>
+
+/**
+ Add log files contained in the directory to the report.
+ @param logDirectoryPath the path to the log directory
+ */
+- (void)setLogDirectoryPath:(NSString *)logDirectoryPath;
+
+@end
+
+
+
 NS_ASSUME_NONNULL_END

XCTestBootstrap/Strategies/FBLogicTestRunStrategy.m

@@ -217,7 +217,6 @@ - (instancetype)initWithExecutor:(id<FBXCTestProcessExecutor>)executor configura
   id<FBLogicXCTestReporter> reporter = self.reporter;
   id<FBControlCoreLogger> logger = self.logger;
   dispatch_queue_t queue = self.executor.workQueue;
-  FBXCTestLogger *mirrorLogger = [FBXCTestLogger defaultLoggerInDefaultDirectory];
   BOOL mirrorToLogger = (self.configuration.mirroring & FBLogicTestMirrorLogger) != 0;
   BOOL mirrorToFiles = (self.configuration.mirroring & FBLogicTestMirrorFileLogs) != 0;
 
@@ -256,6 +255,8 @@ - (instancetype)initWithExecutor:(id<FBXCTestProcessExecutor>)executor configura
   FBFuture<id<FBDataConsumer, FBDataConsumerLifecycle>> *stdErrFuture = [FBFuture futureWithResult:stdErrConsumer];
   FBFuture<id<FBDataConsumer, FBDataConsumerLifecycle>> *shimFuture = [FBFuture futureWithResult:shimConsumer];
   if (mirrorToFiles) {
+    FBXCTestLogger *mirrorLogger = self.configuration.logDirectoryPath ? [FBXCTestLogger defaultLoggerInDirectory:self.configuration.logDirectoryPath] : [FBXCTestLogger defaultLoggerInDefaultDirectory];
+
     stdOutFuture = [mirrorLogger logConsumptionToFile:stdOutConsumer outputKind:@"out" udid:udid logger:logger];
     stdErrFuture = [mirrorLogger logConsumptionToFile:stdErrConsumer outputKind:@"err" udid:udid logger:logger];
     shimFuture = [mirrorLogger logConsumptionToFile:shimConsumer outputKind:@"shim" udid:udid logger:logger];

XCTestBootstrapTests/Fixtures/FBXCTestReporterDouble.h

@@ -14,7 +14,7 @@ NS_ASSUME_NONNULL_BEGIN
 /**
  A Double for verifiying callers of FBXCTestReporter
  */
-@interface FBXCTestReporterDouble : NSObject <FBXCTestReporter>
+@interface FBXCTestReporterDouble : NSObject <FBXCTestReporter, FBXCTestReporterWithFiles>
 
 /**
  An array of the started test suites.
@@ -46,6 +46,11 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @property (nonatomic, assign, readonly) BOOL printReportWasCalled;
 
+/**
+ Path to logs directory
+ */
+@property (nonatomic, nullable, copy) NSString *logDirectoryPath;
+
 /**
  Get events by name that were recieved from `-[FBXCTestReporter handleExternalEvent:]`
  */

fbxctest/FBXCTestKit/Configuration/FBXCTestCommandLine.m

@@ -75,6 +75,7 @@ + (nullable instancetype)commandLineFromArguments:(NSArray<NSString *> *)argumen
       configurationWithShims:shims
       environment:environment
       workingDirectory:workingDirectory
+      logDirectoryPath:nil
       testBundlePath:testBundlePath
       waitForDebugger:waitForDebugger
       timeout:timeout

fbxctest/FBXCTestKit/Configuration/FBXCTestContext.h

@@ -15,6 +15,7 @@ NS_ASSUME_NONNULL_BEGIN
 @class FBXCTestCommandLine;
 @class FBXCTestLogger;
 @protocol FBXCTestReporter;
+@protocol FBXCTestReporterWithFiles;
 
 /**
  Context for the Test Run.
@@ -32,7 +33,7 @@ NS_ASSUME_NONNULL_BEGIN
  @param logger the logger to log with.
  @return a new context.
  */
-+ (instancetype)contextWithReporter:(nullable id<FBXCTestReporter>)reporter logger:(nullable FBXCTestLogger *)logger;
++ (instancetype)contextWithReporter:(nullable id<FBXCTestReporter,FBXCTestReporterWithFiles>)reporter logger:(nullable FBXCTestLogger *)logger;
 
 #pragma mark Properties
 
@@ -44,7 +45,7 @@ NS_ASSUME_NONNULL_BEGIN
 /**
  The Reporter to report to.
  */
-@property (nonatomic, strong, readonly, nullable) id<FBXCTestReporter> reporter;
+@property (nonatomic, strong, readonly, nullable) id<FBXCTestReporter,FBXCTestReporterWithFiles> reporter;
 
 #pragma mark Public Methods
 

fbxctest/FBXCTestKit/Configuration/FBXCTestContext.m

@@ -22,13 +22,13 @@ @implementation FBXCTestContext
 
 #pragma mark Initializers
 
-+ (instancetype)contextWithReporter:(nullable id<FBXCTestReporter>)reporter logger:(nullable FBXCTestLogger *)logger
++ (instancetype)contextWithReporter:(nullable id<FBXCTestReporter, FBXCTestReporterWithFiles>)reporter logger:(nullable FBXCTestLogger *)logger
 {
   return [[FBXCTestContext alloc] initWithReporter:reporter logger:logger];
 }
 
 
-- (instancetype)initWithReporter:(nullable id<FBXCTestReporter>)reporter logger:(nullable FBXCTestLogger *)logger
+- (instancetype)initWithReporter:(nullable id<FBXCTestReporter,FBXCTestReporterWithFiles>)reporter logger:(nullable FBXCTestLogger *)logger
 {
   self = [super init];
   if (!self) {

fbxctest/FBXCTestKitTests/Tests/Unit/FBOSXLogicTestConfigurationTests.m

@@ -59,6 +59,7 @@ - (void)testMacLogicTests
       configurationWithShims:configuration.shims
       environment:processEnvironment
       workingDirectory:workingDirectory
+      logDirectoryPath:nil
       testBundlePath:testBundlePath
       waitForDebugger:NO
       timeout:0
@@ -94,6 +95,7 @@ - (void)testMacLogicTestsIgnoresDestination
       configurationWithShims:configuration.shims
       environment:processEnvironment
       workingDirectory:workingDirectory
+      logDirectoryPath:nil
       testBundlePath:testBundlePath
       waitForDebugger:NO
       timeout:0

fbxctest/FBXCTestKitTests/Tests/Unit/FBiOSLogicTestConfigurationTests.m

@@ -60,6 +60,7 @@ - (void)testiOSLogicTestsWithDestinationAndSDK
       configurationWithShims:configuration.shims
       environment:processEnvironment
       workingDirectory:workingDirectory
+      logDirectoryPath:nil
       testBundlePath:self.iOSUnitTestBundlePath
       waitForDebugger:NO
       timeout:0
@@ -98,6 +99,7 @@ - (void)testiOSLogicTestsWithDestinationWithoutSDK
       configurationWithShims:configuration.shims
       environment:processEnvironment
       workingDirectory:workingDirectory
+      logDirectoryPath:nil
       testBundlePath:self.iOSUnitTestBundlePath
       waitForDebugger:NO
       timeout:0
@@ -138,6 +140,7 @@ - (void)testiOSLogicTestsWithSDKWithoutDestination
       configurationWithShims:configuration.shims
       environment:processEnvironment
       workingDirectory:workingDirectory
+      logDirectoryPath:nil
       testBundlePath:self.iOSUnitTestBundlePath
       waitForDebugger:NO
       timeout:0

idb_companion/Server/FBIDBCommandExecutor.h

@@ -321,7 +321,7 @@ This allows to avoid the permission popup the first time we open a deeplink
  @param logger the logger to log to.
  @return a Future that resolves with the xctest operation.
  */
-- (FBFuture<FBIDBTestOperation *> *)xctest_run:(FBXCTestRunRequest *)request reporter:(id<FBXCTestReporter>)reporter logger:(id<FBControlCoreLogger>)logger;
+- (FBFuture<FBIDBTestOperation *> *)xctest_run:(FBXCTestRunRequest *)request reporter:(id<FBXCTestReporter, FBXCTestReporterWithFiles>)reporter logger:(id<FBControlCoreLogger>)logger;
 
 /**
  Starts the debugserver

idb_companion/Server/FBIDBCommandExecutor.m

@@ -238,7 +238,7 @@ - (instancetype)initWithTarget:(id<FBiOSTarget>)target storageManager:(FBIDBStor
   if (self.target.state == FBiOSTargetStateShutdown) {
     return [self remove_all_storage_and_clear_keychain];
   }
-  
+
   return [[self uninstall_all_applications] onQueue:self.target.workQueue fmap:^FBFuture<NSNull *> *(id _) {
     return [self remove_all_storage_and_clear_keychain];
   }];
@@ -314,7 +314,7 @@ - (instancetype)initWithTarget:(id<FBiOSTarget>)target storageManager:(FBIDBStor
   [replacements addEntriesFromDictionary:self.storageManager.replacementMapping];
   [replacements addEntriesFromDictionary:self.target.replacementMapping];
   NSDictionary<NSString *, NSString *> *environment = [self applyEnvironmentReplacements:configuration.environment replacements:replacements];
-  
+
   FBApplicationLaunchConfiguration *derived = [[FBApplicationLaunchConfiguration alloc]
     initWithBundleID:configuration.bundleID
     bundleName:configuration.bundleName
@@ -343,7 +343,7 @@ - (instancetype)initWithTarget:(id<FBiOSTarget>)target storageManager:(FBIDBStor
   return interpolatedEnvironment;
 }
 
-- (FBFuture<FBIDBTestOperation *> *)xctest_run:(FBXCTestRunRequest *)request reporter:(id<FBXCTestReporter>)reporter logger:(id<FBControlCoreLogger>)logger
+- (FBFuture<FBIDBTestOperation *> *)xctest_run:(FBXCTestRunRequest *)request reporter:(id<FBXCTestReporter, FBXCTestReporterWithFiles>)reporter logger:(id<FBControlCoreLogger>)logger
 {
   return [request startWithBundleStorageManager:self.storageManager.xctest target:self.target reporter:reporter logger:logger temporaryDirectory:self.temporaryDirectory];
 }

idb_companion/Server/FBIDBServiceHandler.mm

@@ -262,6 +262,7 @@ static Status respond_file_path(NSURL *source, NSString *destination, grpc::inte
   NSString *testBundleID = nsstring_from_c_string(request->test_bundle_id());
   BOOL reportActivities = request->report_activities();
   BOOL collectCoverage = request->collect_coverage();
+  BOOL collectLogs = request->collect_logs();
 
   if (request->tests_to_run_size() > 0) {
     testsToRun = NSMutableSet.set;
@@ -278,7 +279,7 @@ static Status respond_file_path(NSURL *source, NSString *destination, grpc::inte
 
   switch (request->mode().mode_case()) {
     case idb::XctestRunRequest_Mode::kLogic: {
-      return [FBXCTestRunRequest logicTestWithTestBundleID:testBundleID environment:environment arguments:arguments testsToRun:testsToRun testsToSkip:testsToSkip testTimeout:testTimeout reportActivities:reportActivities collectCoverage:collectCoverage];
+      return [FBXCTestRunRequest logicTestWithTestBundleID:testBundleID environment:environment arguments:arguments testsToRun:testsToRun testsToSkip:testsToSkip testTimeout:testTimeout reportActivities:reportActivities collectCoverage:collectCoverage collectLogs:collectLogs];
     }
     case idb::XctestRunRequest_Mode::kApplication: {
       const idb::XctestRunRequest::Application application = request->mode().application();
@@ -1023,12 +1024,12 @@ static void populate_companion_info(idb::CompanionInfo *info, id<FBEventReporter
   NSError *error = nil;
   FBIDBXCTestReporter *reporter = [[FBIDBXCTestReporter alloc] initWithResponseWriter:response reportAttachments:request->report_attachments() queue:_target.workQueue logger:_target.logger];
   FBIDBTestOperation *operation = [[_commandExecutor xctest_run:xctestRunRequest reporter:reporter logger:[FBControlCoreLogger loggerToConsumer:reporter]] block:&error];
-  reporter.resultBundlePath = operation.resultBundlePath;
-  reporter.coveragePath = operation.coveragePath;
-  reporter.binaryPath = operation.binaryPath;
   if (!operation) {
     return Status(grpc::StatusCode::INTERNAL, error.localizedDescription.UTF8String);
   }
+  reporter.resultBundlePath = operation.resultBundlePath;
+  reporter.coveragePath = operation.coveragePath;
+  reporter.binaryPath = operation.binaryPath;
   // First wait for the test operation to finish
   [operation.completed block:&error];
   // Then make sure we've reported everything, otherwise we could write in the background (use-after-free)

idb_companion/Utility/FBIDBXCTestReporter.h

@@ -20,7 +20,7 @@ NS_ASSUME_NONNULL_BEGIN
  Bridges from the FBXCTestReporter protocol to a GRPC result writer.
  This also keeps track of the terminal condition of the reporter, so this can be used to know when reporting has fully terminated.
  */
-@interface FBIDBXCTestReporter : NSObject <FBXCTestReporter, FBDataConsumer>
+@interface FBIDBXCTestReporter : NSObject <FBXCTestReporter, FBXCTestReporterWithFiles, FBDataConsumer>
 
 #pragma mark Initializers
 
@@ -51,6 +51,11 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @property (nonatomic, copy, nullable, readwrite) NSString *coveragePath;
 
+/**
+ Log directory path
+ */
+@property (nonatomic, copy, nullable, readwrite) NSString *logDirectoryPath;
+
 /**
  App binary path
  */

idb_companion/Utility/FBIDBXCTestReporter.mm

@@ -347,6 +347,19 @@ - (void)insertFinalDataThenWriteResponse:(const idb::XctestRunResponse &)respons
       return [FBFuture futureWithResult:NSNull.null];
     }]];
   }
+  if (self.logDirectoryPath) {
+      [futures addObject:[[self getLogDirectoryData] onQueue:self.queue chain:^FBFuture<NSNull *> *(FBFuture<NSData *> *future) {
+        NSData *data = future.result;
+        if (data) {
+          idb::Payload *payload = responseCaptured.mutable_log_directory();
+          payload->set_data(data.bytes, data.length);
+        } else {
+          [self.logger.info logFormat:@"Failed to get log drectory %@", future];
+        }
+        return [FBFuture futureWithResult:NSNull.null];
+      }]];
+
+  }
   if (futures.count == 0) {
     [self writeResponseFinal:responseCaptured];
     return;
@@ -363,7 +376,7 @@ - (void)writeResponseFinal:(const idb::XctestRunResponse &)response
   {
     // Break out if the terminating condition happens twice.
     if (self.reportingTerminated.hasCompleted || self.writer == nil) {
-      [self.logger.error log:@"writeResponse called, but the last response has already be written!!"];
+      [self.logger.error log:@"writeResponse called, but the last response has already been written!!"];
       return;
     }
 
@@ -406,5 +419,10 @@ - (void)writeResponseFinal:(const idb::XctestRunResponse &)response
   return [FBArchiveOperations createGzippedTarDataForPath:self.resultBundlePath queue:self.queue logger:self.logger];
 }
 
+-(FBFuture<NSData *> *)getLogDirectoryData
+{
+    return [FBArchiveOperations createGzippedTarDataForPath:self.logDirectoryPath queue:self.queue logger:self.logger];
+
+}
 
 @end