Skip to content
This repository was archived by the owner on Feb 22, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[tool] Check integration tests for test
Ensures that tests in integration test files are written with
`testWidgets` instead of `test`, as the latter won't correctly report
failures in an integration test.

See flutter/flutter#105055
  • Loading branch information
stuartmorgan-g committed Jun 8, 2022
commit eb2ebc4b2363b36ef9103c5efb6fce3cc6f544a4
2 changes: 2 additions & 0 deletions script/tool/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
## NEXT

- Supports empty custom analysis allow list files.
- `drive-examples` now validates files to ensure that they don't accidentally
use `test(...)`.

## 0.8.6

Expand Down
30 changes: 29 additions & 1 deletion script/tool/lib/src/drive_examples_command.dart
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,16 @@ class DriveExamplesCommand extends PackageLoopingCommand {
if (legacyTestFile != null) {
testTargets.add(legacyTestFile);
} else {
(await _getIntegrationTests(example)).forEach(testTargets.add);
for (final File testFile in await _getIntegrationTests(example)) {
// Check files for known problematic patterns.
final bool passesValidation = _validateIntegrationTest(testFile);
if (!passesValidation) {
// Report the issue, but continue with the test as the validation
// errors don't prevent running.
errors.add('${testFile.basename} failed validation');
}
testTargets.add(testFile);
}
}

if (testTargets.isEmpty) {
Expand Down Expand Up @@ -310,6 +319,25 @@ class DriveExamplesCommand extends PackageLoopingCommand {
return tests;
}

/// Checks [testFile] for known bad patterns in integration tests, logging
/// any issues.
///
/// Returns true if the file passes validation without issues.
bool _validateIntegrationTest(File testFile) {
final List<String> lines = testFile.readAsLinesSync();

final RegExp badTestPattern = RegExp(r'\s*test\(');
if (lines.any((String line) => line.startsWith(badTestPattern))) {
final String filename = testFile.basename;
printError(
'$filename uses "test", which will not report failures correctly. '
'Use testWidgets instead.');
return false;
}

return true;
}

/// For each file in [targets], uses
/// `flutter drive --driver [driver] --target <target>`
/// to drive [example], returning a list of any failing test targets.
Expand Down
41 changes: 41 additions & 0 deletions script/tool/test/drive_examples_command_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,47 @@ void main() {
);
});

test('integration tests using test(...) fail validation', () async {
setMockFlutterDevicesOutput();
final RepositoryPackage package = createFakePlugin(
'plugin',
packagesDir,
extraFiles: <String>[
'example/test_driver/integration_test.dart',
'example/integration_test/foo_test.dart',
'example/android/android.java',
],
platformSupport: <String, PlatformDetails>{
platformAndroid: const PlatformDetails(PlatformSupport.inline),
platformIOS: const PlatformDetails(PlatformSupport.inline),
},
);
package.directory
.childDirectory('example')
.childDirectory('integration_test')
.childFile('foo_test.dart')
.writeAsStringSync('''
test('this is the wrong kind of test!'), () {
...
}
''');

Error? commandError;
final List<String> output = await runCapturingPrint(
runner, <String>['drive-examples', '--android'],
errorHandler: (Error e) {
commandError = e;
});

expect(commandError, isA<ToolExit>());
expect(
output,
containsAllInOrder(<Matcher>[
contains('foo_test.dart failed validation'),
]),
);
});

test(
'driving under folder "test_driver" when targets are under "integration_test"',
() async {
Expand Down