Skip to content

[ci] Add LUCI version of Android FTL tests in bringup mode #4571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
auto-submit merged 11 commits into from
Jul 28, 2023
Merged

[ci] Add LUCI version of Android FTL tests in bringup mode #4571

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
b54b403
Remove default service key from gcloud, and add explicit key to Cirru…
stuartmorgan-g Jul 26, 2023
49459fa
Add FTL
stuartmorgan-g Jul 26, 2023
bda7015
Disable everything else for testing
stuartmorgan-g Jul 26, 2023
c9a3c75
Add project
stuartmorgan-g Jul 26, 2023
b575876
Fix results bucket
stuartmorgan-g Jul 26, 2023
eca166b
Remove default bucket, update unit tests
stuartmorgan-g Jul 27, 2023
69667ce
Analysis fix
stuartmorgan-g Jul 27, 2023
4fade20
Another analysis fix
stuartmorgan-g Jul 27, 2023
8a5c80c
Move FTL tests to new bringup tasks
stuartmorgan-g Jul 27, 2023
cb5ea47
Merge branch 'main' into luci-ftl-initial
reidbaker Jul 28, 2023
d779d5b
Improve comment per review feedback
stuartmorgan-g Jul 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove default bucket, update unit tests
  • Loading branch information
@stuartmorgan-g
stuartmorgan-g committed Jul 27, 2023
commit eca166b562c400aa03dbf1254431bf9405715649
2 changes: 1 addition & 1 deletion .cirrus.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ task:
firebase_test_lab_script:
- if [[ -n "$GCLOUD_FIREBASE_TESTLAB_KEY" ]]; then
- echo $GCLOUD_FIREBASE_TESTLAB_KEY > "${GCLOUD_KEY_PATH}"
- ./script/tool_runner.sh firebase-test-lab --device model=redfin,version=30 --exclude=script/configs/exclude_integration_android.yaml --project=flutter-cirrus --service-key="${GCLOUD_KEY_PATH}"
- ./script/tool_runner.sh firebase-test-lab --device model=redfin,version=30 --exclude=script/configs/exclude_integration_android.yaml --project=flutter-cirrus --results-bucket=flutter_cirrus_testlab --service-key="${GCLOUD_KEY_PATH}"
- else
- echo "This user does not have permission to run Firebase Test Lab tests."
- fi
9 changes: 5 additions & 4 deletions script/tool/lib/src/firebase_test_lab_command.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import 'common/package_looping_command.dart';
import 'common/plugin_utils.dart';
import 'common/repository_package.dart';

const int _exitGcloudAuthFailed = 2;
const int _exitGcloudAuthFailed = 3;
const int _exitNoResultsBucket = 3;

/// A command to run tests via Firebase test lab.
class FirebaseTestLabCommand extends PackageLoopingCommand {
Expand Down Expand Up @@ -52,8 +53,7 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
],
help:
'Device model(s) to test. See https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run for more info');
argParser.addOption('results-bucket',
defaultsTo: 'gs://flutter_cirrus_testlab');
argParser.addOption(_gCloudResultsBucketArg, mandatory: true);
argParser.addOption(
kEnableExperiment,
defaultsTo: '',
Expand All @@ -63,6 +63,7 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {

static const String _gCloudServiceKeyArg = 'service-key';
static const String _gCloudProjectArg = 'project';
static const String _gCloudResultsBucketArg = 'results-bucket';

@override
final String name = 'firebase-test-lab';
Expand Down Expand Up @@ -288,7 +289,7 @@ class FirebaseTestLabCommand extends PackageLoopingCommand {
'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk',
'--timeout',
'7m',
'--results-bucket=${getStringArg('results-bucket')}',
'--results-bucket=gs://${getStringArg(_gCloudResultsBucketArg)}',
'--results-dir=$resultsDir',
for (final String device in getStringListArg('device')) ...<String>[
'--device',
Expand Down
79 changes: 38 additions & 41 deletions script/tool/test/firebase_test_lab_command_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,11 @@ public class MainActivityTest {
writeJavaTestFile(plugin, javaTestFileRelativePath);

Error? commandError;
final List<String> output = await runCapturingPrint(
runner, <String>['firebase-test-lab'], errorHandler: (Error e) {
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--service-key=/path/to/key',
], errorHandler: (Error e) {
commandError = e;
});

Expand Down Expand Up @@ -99,8 +102,12 @@ public class MainActivityTest {
]);
writeJavaTestFile(plugin, javaTestFileRelativePath);

final List<String> output =
await runCapturingPrint(runner, <String>['firebase-test-lab']);
final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--service-key=/path/to/key',
'--project=a-project'
]);

expect(
output,
Expand Down Expand Up @@ -132,6 +139,9 @@ public class MainActivityTest {

final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--project=a-project',
'--service-key=/path/to/key',
'--device',
'model=redfin,version=30',
'--device',
Expand All @@ -158,11 +168,11 @@ public class MainActivityTest {
orderedEquals(<ProcessCall>[
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
'auth activate-service-account --key-file=/path/to/key'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
'gcloud', 'config set project a-project'.split(' '), null),
ProcessCall(
'/packages/plugin1/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true'.split(' '),
Expand All @@ -174,7 +184,7 @@ public class MainActivityTest {
'/packages/plugin1/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin1/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin1/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin1/example'),
ProcessCall(
Expand All @@ -188,7 +198,7 @@ public class MainActivityTest {
'/packages/plugin2/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin2/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin2/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin2/example'),
]),
Expand All @@ -211,6 +221,7 @@ public class MainActivityTest {

final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--device',
Expand All @@ -225,7 +236,6 @@ public class MainActivityTest {
output,
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Firebase project configured.'),
contains('Testing example/integration_test/bar_test.dart...'),
contains('Testing example/integration_test/foo_test.dart...'),
]),
Expand All @@ -237,13 +247,6 @@ public class MainActivityTest {
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true'.split(' '),
Expand All @@ -255,7 +258,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example'),
ProcessCall(
Expand All @@ -265,7 +268,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/1/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/1/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example'),
]),
Expand All @@ -292,6 +295,7 @@ public class MainActivityTest {

final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--device',
Expand Down Expand Up @@ -320,7 +324,7 @@ public class MainActivityTest {
'/packages/plugin/example/example1/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example1/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example1/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example/example1'),
ProcessCall(
Expand All @@ -330,7 +334,7 @@ public class MainActivityTest {
'/packages/plugin/example/example2/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example2/0/ --device model=redfin,version=30 --device model=seoul,version=26'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example2/0/ --device model=redfin,version=30 --device model=seoul,version=26'
.split(' '),
'/packages/plugin/example/example2'),
]),
Expand All @@ -350,8 +354,6 @@ public class MainActivityTest {
writeJavaTestFile(plugin, javaTestFileRelativePath);

processRunner.mockProcessesForExecutable['gcloud'] = <FakeProcessInfo>[
FakeProcessInfo(MockProcess(), <String>['auth']),
FakeProcessInfo(MockProcess(), <String>['config']),
FakeProcessInfo(MockProcess(exitCode: 1),
<String>['firebase', 'test']), // integration test #1
FakeProcessInfo(MockProcess(exitCode: 1),
Expand All @@ -365,6 +367,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
Expand Down Expand Up @@ -399,8 +402,6 @@ public class MainActivityTest {
writeJavaTestFile(plugin, javaTestFileRelativePath);

processRunner.mockProcessesForExecutable['gcloud'] = <FakeProcessInfo>[
FakeProcessInfo(MockProcess(), <String>['auth']),
FakeProcessInfo(MockProcess(), <String>['config']),
FakeProcessInfo(MockProcess(exitCode: 1),
<String>['firebase', 'test']), // integration test #1
FakeProcessInfo(MockProcess(),
Expand All @@ -411,6 +412,7 @@ public class MainActivityTest {

final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
]);
Expand All @@ -437,6 +439,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
Expand Down Expand Up @@ -467,6 +470,7 @@ public class MainActivityTest {

final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
]);
Expand Down Expand Up @@ -497,6 +501,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
Expand Down Expand Up @@ -539,6 +544,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
Expand Down Expand Up @@ -567,6 +573,7 @@ public class MainActivityTest {

final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
]);
Expand Down Expand Up @@ -599,6 +606,7 @@ public class MainActivityTest {

final List<String> output = await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--test-run-id',
Expand All @@ -612,7 +620,6 @@ public class MainActivityTest {
containsAllInOrder(<Matcher>[
contains('Running for plugin'),
contains('Running flutter build apk...'),
contains('Firebase project configured.'),
contains('Testing example/integration_test/foo_test.dart...'),
]),
);
Expand All @@ -625,13 +632,6 @@ public class MainActivityTest {
'build apk --config-only'.split(' '),
'/packages/plugin/example/android',
),
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true'.split(' '),
Expand All @@ -643,7 +643,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
.split(' '),
'/packages/plugin/example'),
]),
Expand All @@ -669,6 +669,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
Expand Down Expand Up @@ -711,6 +712,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
Expand Down Expand Up @@ -753,6 +755,7 @@ public class MainActivityTest {
runner,
<String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
],
Expand Down Expand Up @@ -785,6 +788,7 @@ public class MainActivityTest {

await runCapturingPrint(runner, <String>[
'firebase-test-lab',
'--results-bucket=a_bucket',
'--device',
'model=redfin,version=30',
'--test-run-id',
Expand All @@ -797,13 +801,6 @@ public class MainActivityTest {
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
'gcloud',
'auth activate-service-account --key-file=${Platform.environment['HOME']}/gcloud-service-key.json'
.split(' '),
null),
ProcessCall(
'gcloud', 'config set project flutter-cirrus'.split(' '), null),
ProcessCall(
'/packages/plugin/example/android/gradlew',
'app:assembleAndroidTest -Pverbose=true -Pextra-front-end-options=--enable-experiment%3Dexp1 -Pextra-gen-snapshot-options=--enable-experiment%3Dexp1'
Expand All @@ -816,7 +813,7 @@ public class MainActivityTest {
'/packages/plugin/example/android'),
ProcessCall(
'gcloud',
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://flutter_cirrus_testlab --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
'firebase test android run --type instrumentation --app build/app/outputs/apk/debug/app-debug.apk --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk --timeout 7m --results-bucket=gs://a_bucket --results-dir=plugins_android_test/plugin/buildId/testRunId/example/0/ --device model=redfin,version=30'
.split(' '),
'/packages/plugin/example'),
]),
Expand Down