Skip to content

Commit d63987f

Browse files
JasguerreroJasguerrero
authored
Parser machine logs (#118707)
* remove file reporter optional * decouple stack trace from metric collections * update tests; add collect metrics option * add failed tests property * add test for multiple stack failed * change path on result * create factory method * throw exception when test file failed to generate * remove catch of file exception * handle when no stacktrace on file reporter
1 parent d4b6898 commit d63987f

File tree

3 files changed

+97
-41
lines changed

3 files changed

+97
-41
lines changed

‎dev/bots/test.dart‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1802,6 +1802,7 @@ Future<void> _runDartTest(String workingDirectory, {
18021802
final List<String> args = <String>[
18031803
'run',
18041804
'test',
1805+
'--file-reporter=json:${metricFile.path}',
18051806
if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
18061807
'-j$cpus',
18071808
if (!hasColor)
@@ -1813,8 +1814,6 @@ Future<void> _runDartTest(String workingDirectory, {
18131814
if (testPaths != null)
18141815
for (final String testPath in testPaths)
18151816
testPath,
1816-
if (collectMetrics)
1817-
'--file-reporter=json:${metricFile.path}',
18181817
];
18191818
final Map<String, String> environment = <String, String>{
18201819
'FLUTTER_ROOT': flutterRoot,
@@ -1840,19 +1839,24 @@ Future<void> _runDartTest(String workingDirectory, {
18401839
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
18411840
);
18421841

1842+
final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
1843+
final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
1844+
info.writeAsStringSync(json.encode(test.errors));
1845+
18431846
if (collectMetrics) {
18441847
try {
18451848
final List<String> testList = <String>[];
1846-
final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile);
1849+
final Map<int, TestSpecs> allTestSpecs = test.allTestSpecs;
18471850
for (final TestSpecs testSpecs in allTestSpecs.values) {
18481851
testList.add(testSpecs.toJson());
18491852
}
18501853
if (testList.isNotEmpty) {
18511854
final String testJson = json.encode(testList);
1852-
final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json'));
1855+
final File testResults = fileSystem.file(
1856+
path.join(flutterRoot, 'test_results.json'));
18531857
testResults.writeAsStringSync(testJson);
18541858
}
1855-
} on fs.FileSystemException catch (e){
1859+
} on fs.FileSystemException catch (e) {
18561860
print('Failed to generate metrics: $e');
18571861
}
18581862
}

‎dev/bots/test/tool_subsharding_test.dart‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ void main() {
2323
{"missing": "entry"}
2424
{"other": true}''';
2525
file.writeAsStringSync(output);
26-
final Map<int, TestSpecs> result = generateMetrics(file);
27-
expect(result, isEmpty);
26+
final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
27+
expect(result.allTestSpecs, isEmpty);
2828
});
2929

3030
test('have metrics', () async {
@@ -47,7 +47,7 @@ void main() {
4747
{"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}
4848
{"success":true,"type":"done","time":4870}''';
4949
file.writeAsStringSync(output);
50-
final Map<int, TestSpecs> result = generateMetrics(file);
50+
final Map<int, TestSpecs> result = TestFileReporterResults.fromFile(file).allTestSpecs;
5151
expect(result, contains(0));
5252
expect(result, contains(1));
5353
expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart');
@@ -62,8 +62,37 @@ void main() {
6262
{"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
6363
{"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}''';
6464
file.writeAsStringSync(output);
65-
final Map<int, TestSpecs> result = generateMetrics(file);
66-
expect(result, isEmpty);
65+
final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
66+
expect(result.hasFailedTests, true);
67+
});
68+
69+
test('has failed stack traces', () async {
70+
final File file = fileSystem.file('success_file');
71+
const String output = '''
72+
{"protocolVersion":"0.1.1","runnerVersion":"1.22.1","pid":47372,"type":"start","time":0}
73+
{"suite":{"id":0,"platform":"vm","path":"test/tool_subsharding_test.dart"},"type":"suite","time":0}
74+
{"test":{"id":1,"name":"loading test/tool_subsharding_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":2}
75+
{"count":1,"time":11,"type":"allSuites"}
76+
{"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1021}
77+
{"group":{"id":2,"suiteID":0,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":null,"column":null,"url":null},"type":"group","time":1026}
78+
{"group":{"id":3,"suiteID":0,"parentID":2,"name":"generateMetrics","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":13,"column":3,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"group","time":1027}
79+
{"test":{"id":4,"name":"generateMetrics empty metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":20,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1027}
80+
{"testID":4,"error":"Expected: <true> Actual: <false>","stackTrace":"package:test_api expect test/tool_subsharding_test.dart 28:7 main.<fn>.<fn> ","isFailure":true,"type":"error","time":1095}
81+
{"testID":4,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1096}
82+
{"test":{"id":5,"name":"generateMetrics have metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":31,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1097}
83+
{"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":1103}
84+
{"test":{"id":6,"name":"generateMetrics missing success entry","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":60,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1103}
85+
{"testID":6,"error":"Expected: <false> Actual: <true>","stackTrace":"package:test_api expect test/tool_subsharding_test.dart 68:7 main.<fn>.<fn> ","isFailure":true,"type":"error","time":1107}
86+
{"testID":6,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1107}
87+
{"testID":6,"error":"my error","isFailure":true,"type":"error","time":1107}
88+
{"success":false,"type":"done","time":1120}''';
89+
file.writeAsStringSync(output);
90+
final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
91+
expect(result.hasFailedTests, true);
92+
expect(result.errors.length == 3, true);
93+
expect(result.errors[0].contains('Expected: <true> Actual: <false>'), true);
94+
expect(result.errors[1].contains('Expected: <false> Actual: <true>'), true);
95+
expect(result.errors[2].contains('my error'), true);
6796
});
6897
});
6998
}

‎dev/bots/tool_subsharding.dart‎

Lines changed: 54 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -38,42 +38,65 @@ class TestSpecs {
3838
}
3939
}
4040

41-
/// Intended to parse the output file of `dart test --file-reporter json:file_name
42-
Map<int, TestSpecs> generateMetrics(File metrics) {
43-
final Map<int, TestSpecs> allTestSpecs = <int, TestSpecs>{};
44-
if (!metrics.existsSync()) {
45-
return allTestSpecs;
46-
}
41+
class TestFileReporterResults {
42+
TestFileReporterResults._({
43+
required this.allTestSpecs,
44+
required this.hasFailedTests,
45+
required this.errors,
46+
});
4747

48-
bool success = false;
49-
for(final String metric in metrics.readAsLinesSync()) {
50-
final Map<String, dynamic> entry = json.decode(metric) as Map<String, dynamic>;
51-
if (entry.containsKey('suite')) {
52-
final Map<dynamic, dynamic> suite = entry['suite'] as Map<dynamic, dynamic>;
53-
allTestSpecs[suite['id'] as int] = TestSpecs(
54-
path: suite['path'] as String,
55-
startTime: entry['time'] as int,
56-
);
57-
} else if (_isMetricDone(entry, allTestSpecs)) {
58-
final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
59-
final int suiteID = group['suiteID'] as int;
60-
final TestSpecs testSpec = allTestSpecs[suiteID]!;
61-
testSpec.endTime = entry['time'] as int;
62-
} else if (entry.containsKey('success') && entry['success'] == true) {
63-
success = true;
48+
/// Intended to parse the output file of `dart test --file-reporter json:file_name
49+
factory TestFileReporterResults.fromFile(File metrics) {
50+
if (!metrics.existsSync()) {
51+
throw Exception('${metrics.path} does not exist');
6452
}
53+
54+
final Map<int, TestSpecs> testSpecs = <int, TestSpecs>{};
55+
bool hasFailedTests = true;
56+
final List<String> errors = <String>[];
57+
58+
for (final String metric in metrics.readAsLinesSync()) {
59+
final Map<String, Object?> entry = json.decode(metric) as Map<String, Object?>;
60+
if (entry.containsKey('suite')) {
61+
final Map<String, Object?> suite = entry['suite']! as Map<String, Object?>;
62+
addTestSpec(suite, entry['time']! as int, testSpecs);
63+
} else if (isMetricDone(entry, testSpecs)) {
64+
final Map<String, Object?> group = entry['group']! as Map<String, Object?>;
65+
final int suiteID = group['suiteID']! as int;
66+
addMetricDone(suiteID, entry['time']! as int, testSpecs);
67+
} else if (entry.containsKey('error')) {
68+
final String stackTrace = entry.containsKey('stackTrace') ? entry['stackTrace']! as String : '';
69+
errors.add('${entry['error']}\n $stackTrace');
70+
} else if (entry.containsKey('success') && entry['success'] == true) {
71+
hasFailedTests = false;
72+
}
73+
}
74+
75+
return TestFileReporterResults._(allTestSpecs: testSpecs, hasFailedTests: hasFailedTests, errors: errors);
6576
}
6677

67-
if (!success) { // means that not all tests succeeded therefore no metrics are stored
68-
return <int, TestSpecs>{};
78+
final Map<int, TestSpecs> allTestSpecs;
79+
final bool hasFailedTests;
80+
final List<String> errors;
81+
82+
83+
static void addTestSpec(Map<String, Object?> suite, int time, Map<int, TestSpecs> allTestSpecs) {
84+
allTestSpecs[suite['id']! as int] = TestSpecs(
85+
path: suite['path']! as String,
86+
startTime: time,
87+
);
6988
}
70-
return allTestSpecs;
71-
}
7289

73-
bool _isMetricDone(Map<String, dynamic> entry, Map<int, TestSpecs> allTestSpecs) {
74-
if (entry.containsKey('group') && entry['type'] as String == 'group') {
75-
final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
76-
return allTestSpecs.containsKey(group['suiteID'] as int);
90+
static void addMetricDone(int suiteID, int time, Map<int, TestSpecs> allTestSpecs) {
91+
final TestSpecs testSpec = allTestSpecs[suiteID]!;
92+
testSpec.endTime = time;
93+
}
94+
95+
static bool isMetricDone(Map<String, Object?> entry, Map<int, TestSpecs> allTestSpecs) {
96+
if (entry.containsKey('group') && entry['type']! as String == 'group') {
97+
final Map<String, Object?> group = entry['group']! as Map<String, Object?>;
98+
return allTestSpecs.containsKey(group['suiteID']! as int);
99+
}
100+
return false;
77101
}
78-
return false;
79102
}

0 commit comments

Comments
 (0)