[ci] Upload screenshots, logs, and Xcode test results for drive and integration_test runs

script/tool/lib/src/common/core.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'package:file/file.dart';
+import 'package:platform/platform.dart';
 import 'package:pub_semver/pub_semver.dart';
 
 /// The signature for a print handler for commands that allow overriding the
@@ -127,3 +128,13 @@ const int exitCommandFoundErrors = 1;
 
 /// A exit code for [ToolExit] for a failure to run due to invalid arguments.
 const int exitInvalidArguments = 2;
+
+/// The directory to which to write logs and other artifacts, if set in CI.
+Directory? ciLogsDirectory(Platform platform, FileSystem fileSystem) {
+  final String? logsDirectoryPath = platform.environment['FLUTTER_LOGS_DIR'];
+  Directory? logsDirectory;
+  if (logsDirectoryPath != null) {
+    logsDirectory = fileSystem.directory(logsDirectoryPath);
+  }
+  return logsDirectory;
+}

script/tool/lib/src/common/xcode.dart

@@ -6,7 +6,9 @@ import 'dart:convert';
 import 'dart:io' as io;
 
 import 'package:file/file.dart';
+import 'package:platform/platform.dart';
 
+import 'core.dart';
 import 'output_utils.dart';
 import 'process_runner.dart';
 
@@ -33,17 +35,29 @@ class Xcode {
   /// Runs an `xcodebuild` in [directory] with the given parameters.
   Future<int> runXcodeBuild(
     Directory exampleDirectory,
-    String platform, {
+    String targetPlatform, {
     List<String> actions = const <String>['build'],
     required String workspace,
     required String scheme,
     String? configuration,
     List<String> extraFlags = const <String>[],
-  }) {
+    required Platform platform,
+  }) async {
+    final FileSystem fileSystem = exampleDirectory.fileSystem;
+    String? resultBundlePath;
+    final Directory? logsDirectory = ciLogsDirectory(platform, fileSystem);
+    Directory? resultBundleTemp;
+    if (logsDirectory != null) {
+      resultBundleTemp = fileSystem.systemTempDirectory
+          .createTempSync('flutter_xcresult.');
+      resultBundlePath = resultBundleTemp
+          .childDirectory('result')
+          .path;
+    }
     File? disabledSandboxEntitlementFile;
-    if (actions.contains('test') && platform.toLowerCase() == 'macos') {
+    if (actions.contains('test') && targetPlatform.toLowerCase() == 'macos') {
       disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
-        exampleDirectory.childDirectory(platform.toLowerCase()),
+        exampleDirectory.childDirectory(targetPlatform.toLowerCase()),
         configuration ?? 'Debug',
       );
     }
@@ -52,6 +66,8 @@ class Xcode {
       ...actions,
       ...<String>['-workspace', workspace],
       ...<String>['-scheme', scheme],
+      if (resultBundlePath != null)
+        ...<String>['-resultBundlePath', resultBundlePath],
       if (configuration != null) ...<String>['-configuration', configuration],
       ...extraFlags,
       if (disabledSandboxEntitlementFile != null)
@@ -61,8 +77,36 @@ class Xcode {
     if (log) {
       print(completeTestCommand);
     }
-    return processRunner.runAndStream(_xcRunCommand, args,
+    final int resultExit = await processRunner.runAndStream(_xcRunCommand, args,
         workingDir: exampleDirectory);
+
+    if (resultExit != 0 && resultBundleTemp != null) {
+      final Directory xcresultBundle = resultBundleTemp.childDirectory(
+          'result.xcresult');
+      if (logsDirectory != null) {
+        if (xcresultBundle.existsSync()) {
+          // Zip the test results to the artifacts directory for upload.
+          final File zipPath = logsDirectory.childFile(
+              'xcodebuild-${DateTime.now().toLocal().toIso8601String()}.zip');
+          await processRunner.run(
+            'zip',
+            <String>[
+              '-r',
+              '-9',
+              '-q',
+              zipPath.path,
+              xcresultBundle.basename,
+            ],
+            workingDir: resultBundleTemp,
+          );
+        } else {
+          print('xcresult bundle ${xcresultBundle
+              .path} does not exist, skipping upload');
+        }
+      }
+    }
+    return resultExit;
+
   }
 
   /// Returns true if [project], which should be an .xcodeproj directory,

script/tool/lib/src/drive_examples_command.dart

@@ -228,8 +228,12 @@ class DriveExamplesCommand extends PackageLoopingCommand {
         }
         for (final File driver in drivers) {
           final List<File> failingTargets = await _driveTests(
-              example, driver, testTargets,
-              deviceFlags: deviceFlags);
+            example,
+            driver,
+            testTargets,
+            deviceFlags: deviceFlags,
+            exampleName: exampleName,
+          );
           for (final File failingTarget in failingTargets) {
             errors.add(
                 getRelativePosixPath(failingTarget, from: package.directory));
@@ -376,19 +380,27 @@ class DriveExamplesCommand extends PackageLoopingCommand {
     File driver,
     List<File> targets, {
     required List<String> deviceFlags,
+    required String exampleName,
   }) async {
     final List<File> failures = <File>[];
 
     final String enableExperiment = getStringArg(kEnableExperiment);
+    final Directory? logsDirectory = ciLogsDirectory(platform, driver.fileSystem);
 
     for (final File target in targets) {
+      Directory? screenshotDirectory;
+      if (logsDirectory != null) {
+        screenshotDirectory = logsDirectory.childDirectory('$exampleName-drive');
+      }
       final int exitCode = await processRunner.runAndStream(
           flutterCommand,
           <String>[
             'drive',
             ...deviceFlags,
             if (enableExperiment.isNotEmpty)
               '--enable-experiment=$enableExperiment',
+            if (screenshotDirectory != null)
+              '--screenshot=${screenshotDirectory.path}',
             '--driver',
             getRelativePosixPath(driver, from: example.directory),
             '--target',
@@ -416,6 +428,7 @@ class DriveExamplesCommand extends PackageLoopingCommand {
     required List<File> testFiles,
   }) async {
     final String enableExperiment = getStringArg(kEnableExperiment);
+    final Directory? logsDirectory = testFiles.isNotEmpty ? ciLogsDirectory(platform, testFiles.first.fileSystem) : null;
 
     // Workaround for https://github.com/flutter/flutter/issues/135673
     // Once that is fixed on stable, this logic can be removed and the command
@@ -438,6 +451,8 @@ class DriveExamplesCommand extends PackageLoopingCommand {
             ...deviceFlags,
             if (enableExperiment.isNotEmpty)
               '--enable-experiment=$enableExperiment',
+            if (logsDirectory != null)
+              '--debug-logs-dir=${logsDirectory.path}',
             target,
           ],
           workingDir: example.directory);

script/tool/lib/src/native_test_command.dart

@@ -431,7 +431,7 @@ this command.
   /// usually at "example/{ios,macos}/Runner.xcworkspace".
   Future<_PlatformResult> _runXcodeTests(
     RepositoryPackage plugin,
-    String platform,
+    String targetPlatform,
     _TestMode mode, {
     List<String> extraFlags = const <String>[],
   }) async {
@@ -456,7 +456,7 @@ this command.
       final String? targetToCheck =
           testTarget ?? (mode.unit ? unitTestTarget : null);
       final Directory xcodeProject = example.directory
-          .childDirectory(platform.toLowerCase())
+          .childDirectory(targetPlatform.toLowerCase())
           .childDirectory('Runner.xcodeproj');
       if (targetToCheck != null) {
         final bool? hasTarget =
@@ -473,16 +473,17 @@ this command.
         }
       }
 
-      _printRunningExampleTestsMessage(example, platform);
+      _printRunningExampleTestsMessage(example, targetPlatform);
       final int exitCode = await _xcode.runXcodeBuild(
         example.directory,
-        platform,
+        targetPlatform,
         // Clean before testing to remove cached swiftmodules from previous
         // runs, which can cause conflicts.
         actions: <String>['clean', 'test'],
-        workspace: '${platform.toLowerCase()}/Runner.xcworkspace',
+        workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace',
         scheme: 'Runner',
         configuration: 'Debug',
+        platform: platform,
         extraFlags: <String>[
           if (testTarget != null) '-only-testing:$testTarget',
           ...extraFlags,
@@ -494,9 +495,9 @@ this command.
       const int xcodebuildNoTestExitCode = 66;
       switch (exitCode) {
         case xcodebuildNoTestExitCode:
-          _printNoExampleTestsMessage(example, platform);
+          _printNoExampleTestsMessage(example, targetPlatform);
         case 0:
-          printSuccess('Successfully ran $platform xctest for $exampleName');
+          printSuccess('Successfully ran $targetPlatform xctest for $exampleName');
           // If this is the first test, assume success until something fails.
           if (overallResult == RunState.skipped) {
             overallResult = RunState.succeeded;

script/tool/lib/src/xcode_analyze_command.dart

@@ -97,36 +97,37 @@ class XcodeAnalyzeCommand extends PackageLoopingCommand {
             multiplePlatformsRequested ? failures : <String>[]);
   }
 
-  /// Analyzes [plugin] for [platform], returning true if it passed analysis.
+  /// Analyzes [plugin] for [targetPlatform], returning true if it passed analysis.
   Future<bool> _analyzePlugin(
     RepositoryPackage plugin,
-    String platform, {
+    String targetPlatform, {
     List<String> extraFlags = const <String>[],
   }) async {
     bool passing = true;
     for (final RepositoryPackage example in plugin.getExamples()) {
       // Running tests and static analyzer.
       final String examplePath = getRelativePosixPath(example.directory,
           from: plugin.directory.parent);
-      print('Running $platform tests and analyzer for $examplePath...');
+      print('Running $targetPlatform tests and analyzer for $examplePath...');
       final int exitCode = await _xcode.runXcodeBuild(
         example.directory,
-        platform,
+        targetPlatform,
         // Clean before analyzing to remove cached swiftmodules from previous
         // runs, which can cause conflicts.
         actions: <String>['clean', 'analyze'],
-        workspace: '${platform.toLowerCase()}/Runner.xcworkspace',
+        workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace',
         scheme: 'Runner',
         configuration: 'Debug',
+        platform: platform,
         extraFlags: <String>[
           ...extraFlags,
           'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
         ],
       );
       if (exitCode == 0) {
-        printSuccess('$examplePath ($platform) passed analysis.');
+        printSuccess('$examplePath ($targetPlatform) passed analysis.');
       } else {
-        printError('$examplePath ($platform) failed analysis.');
+        printError('$examplePath ($targetPlatform) failed analysis.');
         passing = false;
       }
     }

script/tool/test/common/xcode_test.dart

@@ -165,6 +165,7 @@ void main() {
         'ios',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
+        platform: MockPlatform(),
       );
 
       expect(exitCode, 0);
@@ -193,6 +194,7 @@ void main() {
           workspace: 'A.xcworkspace',
           scheme: 'AScheme',
           configuration: 'Debug',
+          platform: MockPlatform(),
           extraFlags: <String>['-a', '-b', 'c=d']);
 
       expect(exitCode, 0);
@@ -230,6 +232,7 @@ void main() {
         'ios',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
+        platform: MockPlatform(),
       );
 
       expect(exitCode, 1);
@@ -264,6 +267,7 @@ void main() {
         'macos',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
+        platform: MockPlatform(),
         actions: <String>['test'],
       );