Skip to content

Commit 745ea32

Browse files
baishengzcopybara-github
baishengz
authored and
copybara-github
committed
Internal change
PiperOrigin-RevId: 950703951
1 parent 6417ab8 commit 745ea32

7 files changed

Lines changed: 459 additions & 96 deletions

File tree

‎src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor/BUILD‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,16 @@ java_library(
4747
"@maven//:com_google_guava_guava",
4848
],
4949
)
50+
51+
java_library(
52+
name = "retry_tests_grouper",
53+
srcs = ["RetryTestsGrouper.java"],
54+
deps = [
55+
"//src/devtools/mobileharness/api/model/proto:test_java_proto",
56+
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/strategy:flaky_test_retry_constants",
57+
"//src/java/com/google/devtools/mobileharness/shared/util/logging:google_logger",
58+
"//src/java/com/google/wireless/qa/mobileharness/shared/constant:property",
59+
"//src/java/com/google/wireless/qa/mobileharness/shared/model/job",
60+
"@maven//:com_google_guava_guava",
61+
],
62+
)

‎src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor/RetryTestsGrouper.java‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.devtools.mobileharness.infra.client.api.controller.job.retry.processor;
18+
19+
import static com.google.common.collect.ImmutableList.toImmutableList;
20+
21+
import com.google.common.collect.ImmutableList;
22+
import com.google.common.collect.ImmutableMap;
23+
import com.google.common.flogger.FluentLogger;
24+
import com.google.devtools.mobileharness.api.model.proto.Test.TestResult;
25+
import com.google.devtools.mobileharness.infra.client.api.controller.job.retry.strategy.FlakyTestRetryConstants;
26+
import com.google.wireless.qa.mobileharness.shared.constant.PropertyName.Test;
27+
import com.google.wireless.qa.mobileharness.shared.model.job.JobInfo;
28+
import com.google.wireless.qa.mobileharness.shared.model.job.TestInfo;
29+
import java.util.Comparator;
30+
import java.util.List;
31+
import java.util.Map;
32+
import java.util.Map.Entry;
33+
import java.util.Optional;
34+
import java.util.TreeMap;
35+
import java.util.stream.Collectors;
36+
37+
/** Utility for grouping test runs. */
38+
public final class RetryTestsGrouper {
39+
40+
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
41+
42+
private RetryTestsGrouper() {}
43+
44+
/**
45+
* Grouped test runs for a single shard by flaky attempt index.
46+
*
47+
* @param testsByFlakyAttemptIndex map of flaky attempt index to test run
48+
*/
49+
public record ShardTestRuns(ImmutableMap<Integer, TestInfo> testsByFlakyAttemptIndex) {}
50+
51+
/**
52+
* Grouped test runs by shard index then by flaky attempt index.
53+
*
54+
* @param testsByShardIndex map of shard index to tests grouped by flaky attempt index for that
55+
* shard
56+
*/
57+
public record GroupedTests(ImmutableMap<Integer, ShardTestRuns> testsByShardIndex) {}
58+
59+
/**
60+
* Groups test runs by shard index then by flaky attempt index.
61+
*
62+
* <p>If a shard has equal to or more than one pass/fail test run, test runs excluding pass/fail
63+
* runs are discarded in the result. If none of the test runs for a shard is pass or fail, returns
64+
* the last run for that shard.
65+
*/
66+
public static GroupedTests groupTestsByShardAndFlakyAttemptIndex(JobInfo jobInfo) {
67+
ImmutableMap.Builder<Integer, ShardTestRuns> groupedTestsByShard = ImmutableMap.builder();
68+
69+
Map<Integer, ImmutableList<TestInfo>> shardIndexToTestInfoList =
70+
jobInfo.tests().getAll().values().stream()
71+
.collect(
72+
Collectors.groupingBy(
73+
t -> t.properties().getInt(Test.SHARD_INDEX).orElse(0),
74+
TreeMap::new,
75+
toImmutableList()));
76+
77+
for (Integer shardIndex : shardIndexToTestInfoList.keySet()) {
78+
ImmutableList<TestInfo> testRunsForShard = shardIndexToTestInfoList.get(shardIndex);
79+
logger.atInfo().log("Found %s test runs for shard %d", testRunsForShard.size(), shardIndex);
80+
boolean hasPassOrFail = testRunsForShard.stream().anyMatch(RetryTestsGrouper::isPassOrFail);
81+
82+
ImmutableMap.Builder<Integer, TestInfo> flakyAttemptIndexToUserVisibleTest =
83+
ImmutableMap.builder();
84+
85+
if (hasPassOrFail) {
86+
Map<Integer, ImmutableList<TestInfo>> flakyAttemptIndexToTestInfoList =
87+
testRunsForShard.stream()
88+
.collect(
89+
Collectors.groupingBy(
90+
t ->
91+
getAttemptIndexProperty(
92+
t, FlakyTestRetryConstants.TEST_PROP_FLAKY_ATTEMPT_INDEX),
93+
TreeMap::new,
94+
toImmutableList()));
95+
96+
for (Entry<Integer, ImmutableList<TestInfo>> flakyAttemptIndexToTestInfoListEntry :
97+
flakyAttemptIndexToTestInfoList.entrySet()) {
98+
int flakyAttemptIndex = flakyAttemptIndexToTestInfoListEntry.getKey();
99+
ImmutableList<TestInfo> testsFromSameFlakyAttempt =
100+
flakyAttemptIndexToTestInfoListEntry.getValue();
101+
findLastPassOrFailRun(testsFromSameFlakyAttempt)
102+
.ifPresent(
103+
passOrFailTest ->
104+
flakyAttemptIndexToUserVisibleTest.put(flakyAttemptIndex, passOrFailTest));
105+
}
106+
} else {
107+
findLastRun(testRunsForShard)
108+
.ifPresent(
109+
lastRun ->
110+
flakyAttemptIndexToUserVisibleTest.put(
111+
getAttemptIndexProperty(
112+
lastRun, FlakyTestRetryConstants.TEST_PROP_FLAKY_ATTEMPT_INDEX),
113+
lastRun));
114+
}
115+
groupedTestsByShard.put(
116+
shardIndex, new ShardTestRuns(flakyAttemptIndexToUserVisibleTest.buildOrThrow()));
117+
}
118+
119+
return new GroupedTests(groupedTestsByShard.buildOrThrow());
120+
}
121+
122+
private static Optional<TestInfo> findLastPassOrFailRun(List<TestInfo> attempts) {
123+
return attempts.stream()
124+
.filter(RetryTestsGrouper::isPassOrFail)
125+
.max(
126+
Comparator.comparingInt(
127+
t ->
128+
getAttemptIndexProperty(
129+
t, FlakyTestRetryConstants.TEST_PROP_ERROR_ATTEMPT_INDEX)));
130+
}
131+
132+
private static Optional<TestInfo> findLastRun(List<TestInfo> attempts) {
133+
return attempts.stream()
134+
.max(
135+
Comparator.comparingInt(
136+
(TestInfo t) ->
137+
getAttemptIndexProperty(
138+
t, FlakyTestRetryConstants.TEST_PROP_FLAKY_ATTEMPT_INDEX))
139+
.thenComparingInt(
140+
t ->
141+
getAttemptIndexProperty(
142+
t, FlakyTestRetryConstants.TEST_PROP_ERROR_ATTEMPT_INDEX)));
143+
}
144+
145+
private static boolean isPassOrFail(TestInfo testInfo) {
146+
if (testInfo.resultWithCause() == null || testInfo.resultWithCause().get() == null) {
147+
return false;
148+
}
149+
TestResult type = testInfo.resultWithCause().get().type();
150+
return type == TestResult.PASS || type == TestResult.FAIL;
151+
}
152+
153+
private static int getAttemptIndexProperty(TestInfo testInfo, String propertyName) {
154+
String value = testInfo.properties().get(propertyName);
155+
if (value == null) {
156+
return 0;
157+
}
158+
try {
159+
return Integer.parseInt(value);
160+
} catch (NumberFormatException e) {
161+
return 0;
162+
}
163+
}
164+
}

‎src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/strategy/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ java_library(
4343
"FlakyTestRetryConstants.java",
4444
],
4545
visibility = [
46+
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor:__pkg__",
4647
"//src/java/com/google/devtools/mobileharness/infra/client/api/plugin:__pkg__",
4748
"//src/java/com/google/devtools/mobileharness/shared/util/testresult:__subpackages__",
49+
"//src/javatests/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor:__pkg__",
4850
],
4951
)
5052

‎src/java/com/google/devtools/mobileharness/shared/util/testresult/rollup/BUILD‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,10 @@ java_library(
3737
"//src/devtools/mobileharness/platform/android/instrumentation/result/proto:test_suite_result_java_proto",
3838
"//src/java/com/google/devtools/mobileharness/api/model/error",
3939
"//src/java/com/google/devtools/mobileharness/api/model/job/out",
40-
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/strategy:flaky_test_retry_constants",
40+
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor:retry_tests_grouper",
4141
"//src/java/com/google/devtools/mobileharness/shared/util/file/local",
4242
"//src/java/com/google/devtools/mobileharness/shared/util/logging:google_logger",
4343
"//src/java/com/google/devtools/mobileharness/shared/util/path",
44-
"//src/java/com/google/wireless/qa/mobileharness/shared/constant:property",
4544
"//src/java/com/google/wireless/qa/mobileharness/shared/model/job",
4645
"@maven//:com_google_guava_guava",
4746
"@protobuf//:protobuf_java",

‎src/java/com/google/devtools/mobileharness/shared/util/testresult/rollup/JobResultUtil.java‎

Lines changed: 7 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,19 @@
2424
import com.google.common.flogger.FluentLogger;
2525
import com.google.devtools.mobileharness.api.model.error.MobileHarnessException;
2626
import com.google.devtools.mobileharness.api.model.job.out.Result.ResultTypeWithCause;
27-
import com.google.devtools.mobileharness.infra.client.api.controller.job.retry.strategy.FlakyTestRetryConstants;
27+
import com.google.devtools.mobileharness.infra.client.api.controller.job.retry.processor.RetryTestsGrouper;
28+
import com.google.devtools.mobileharness.infra.client.api.controller.job.retry.processor.RetryTestsGrouper.GroupedTests;
29+
import com.google.devtools.mobileharness.infra.client.api.controller.job.retry.processor.RetryTestsGrouper.ShardTestRuns;
2830
import com.google.devtools.mobileharness.platform.android.instrumentation.result.proto.TestSuiteResult;
2931
import com.google.devtools.mobileharness.shared.util.file.local.LocalFileUtil;
3032
import com.google.devtools.mobileharness.shared.util.path.PathUtil;
3133
import com.google.devtools.mobileharness.shared.util.testresult.rollup.Outcome.OutcomeSummary;
3234
import com.google.protobuf.ExtensionRegistryLite;
3335
import com.google.protobuf.InvalidProtocolBufferException;
34-
import com.google.wireless.qa.mobileharness.shared.constant.PropertyName.Test;
3536
import com.google.wireless.qa.mobileharness.shared.model.job.JobInfo;
3637
import com.google.wireless.qa.mobileharness.shared.model.job.TestInfo;
3738
import java.util.ArrayList;
38-
import java.util.Comparator;
3939
import java.util.List;
40-
import java.util.Map;
41-
import java.util.Optional;
42-
import java.util.TreeMap;
43-
import java.util.stream.Collectors;
4440

4541
/** Utility for computing job-level rolled up test results. */
4642
public final class JobResultUtil {
@@ -54,9 +50,8 @@ private JobResultUtil() {}
5450
* to get a rollup {@link TestResult} for each shard, and then rolling up shards horizontally.
5551
*/
5652
public static TestResult computeJobRunResult(JobInfo jobInfo) {
57-
ImmutableListMultimap<String, TestInfo> groupedTestRunsByShard =
58-
groupTestRunsByShard(ImmutableList.copyOf(jobInfo.tests().getAll().values()));
59-
if (groupedTestRunsByShard.isEmpty()) {
53+
GroupedTests groupedTests = RetryTestsGrouper.groupTestsByShardAndFlakyAttemptIndex(jobInfo);
54+
if (groupedTests.testsByShardIndex().isEmpty()) {
6055
logger.atInfo().log("No valid test runs found for job %s.", jobInfo.locator().getId());
6156
return TestResult.create(
6257
/* testCases= */ ImmutableList.of(),
@@ -66,10 +61,9 @@ public static TestResult computeJobRunResult(JobInfo jobInfo) {
6661
}
6762

6863
ImmutableList.Builder<TestResult> shardResultsOfJobRunBuilder = ImmutableList.builder();
69-
for (String shardIndex : groupedTestRunsByShard.keySet()) {
70-
ImmutableList<TestInfo> attemptsForOneShard = groupedTestRunsByShard.get(shardIndex);
64+
for (ShardTestRuns attemptsForOneShard : groupedTests.testsByShardIndex().values()) {
7165
List<TestResult> rerunShardTestResults = new ArrayList<>();
72-
for (TestInfo attempt : attemptsForOneShard) {
66+
for (TestInfo attempt : attemptsForOneShard.testsByFlakyAttemptIndex().values()) {
7367
rerunShardTestResults.add(loadTestResult(attempt));
7468
}
7569
TestResult rollupShardResult = rollupToShardTestResult(rerunShardTestResults);
@@ -80,75 +74,6 @@ public static TestResult computeJobRunResult(JobInfo jobInfo) {
8074
return rollupShardResultsToJobResult(shardResults);
8175
}
8276

83-
/**
84-
* Groups test runs by shard.
85-
*
86-
* <p>Each entry in the returned map represents all test runs(including original run and reruns,
87-
* only consider pass or fail runs) for a particular shard. If none of the runs for a shard is
88-
* pass or fail, return the last run for that shard.
89-
*/
90-
private static ImmutableListMultimap<String, TestInfo> groupTestRunsByShard(
91-
List<TestInfo> testInfoList) {
92-
ImmutableListMultimap<String, TestInfo> shardIndexToTestInfoList =
93-
Multimaps.index(
94-
testInfoList, t -> t.properties().getOptional(Test.SHARD_INDEX).orElse("0"));
95-
96-
ImmutableListMultimap.Builder<String, TestInfo> testRunsByShard =
97-
ImmutableListMultimap.builder();
98-
99-
for (String shardIndex : shardIndexToTestInfoList.keySet()) {
100-
ImmutableList<TestInfo> testRunsForShard = shardIndexToTestInfoList.get(shardIndex);
101-
logger.atInfo().log("Found %s test runs for shard %s", testRunsForShard.size(), shardIndex);
102-
boolean hasPassOrFail = testRunsForShard.stream().anyMatch(JobResultUtil::isPassOrFail);
103-
104-
if (hasPassOrFail) {
105-
Map<Integer, ImmutableList<TestInfo>> flakyAttemptIndexToTestInfoList =
106-
testRunsForShard.stream()
107-
.collect(
108-
Collectors.groupingBy(
109-
t ->
110-
getAttemptIndexProperty(
111-
t, FlakyTestRetryConstants.TEST_PROP_FLAKY_ATTEMPT_INDEX),
112-
TreeMap::new,
113-
toImmutableList()));
114-
115-
for (ImmutableList<TestInfo> testsFromSameFlakyAttempt :
116-
flakyAttemptIndexToTestInfoList.values()) {
117-
findLastPassOrFailRun(testsFromSameFlakyAttempt)
118-
.ifPresent(passOrFailTest -> testRunsByShard.put(shardIndex, passOrFailTest));
119-
}
120-
} else {
121-
findLastRun(testRunsForShard)
122-
.ifPresent(lastRun -> testRunsByShard.put(shardIndex, lastRun));
123-
}
124-
}
125-
126-
return testRunsByShard.build();
127-
}
128-
129-
private static Optional<TestInfo> findLastPassOrFailRun(List<TestInfo> attempts) {
130-
return attempts.stream()
131-
.filter(JobResultUtil::isPassOrFail)
132-
.max(
133-
Comparator.comparingInt(
134-
t ->
135-
getAttemptIndexProperty(
136-
t, FlakyTestRetryConstants.TEST_PROP_ERROR_ATTEMPT_INDEX)));
137-
}
138-
139-
private static Optional<TestInfo> findLastRun(List<TestInfo> attempts) {
140-
return attempts.stream()
141-
.max(
142-
Comparator.comparingInt(
143-
(TestInfo t) ->
144-
getAttemptIndexProperty(
145-
t, FlakyTestRetryConstants.TEST_PROP_FLAKY_ATTEMPT_INDEX))
146-
.thenComparingInt(
147-
t ->
148-
getAttemptIndexProperty(
149-
t, FlakyTestRetryConstants.TEST_PROP_ERROR_ATTEMPT_INDEX)));
150-
}
151-
15277
private static boolean isPassOrFail(TestInfo testInfo) {
15378
if (testInfo.resultWithCause() == null || testInfo.resultWithCause().get() == null) {
15479
return false;
@@ -159,18 +84,6 @@ private static boolean isPassOrFail(TestInfo testInfo) {
15984
|| type == com.google.devtools.mobileharness.api.model.proto.Test.TestResult.FAIL;
16085
}
16186

162-
private static int getAttemptIndexProperty(TestInfo testInfo, String propertyName) {
163-
String value = testInfo.properties().get(propertyName);
164-
if (value == null) {
165-
return 0;
166-
}
167-
try {
168-
return Integer.parseInt(value);
169-
} catch (NumberFormatException e) {
170-
return 0;
171-
}
172-
}
173-
17487
private static TestResult loadTestResult(TestInfo testInfo) {
17588
if (!isPassOrFail(testInfo)) {
17689
return TestResult.create(

‎src/javatests/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ java_library(
2929
"//src/devtools/mobileharness/platform/android/instrumentation/result/proto:test_suite_result_java_proto",
3030
"//src/java/com/google/devtools/mobileharness/api/model/error",
3131
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor:android_instrumentation_test_retry_processor",
32+
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor:retry_tests_grouper",
3233
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/processor:test_retry_processor",
34+
"//src/java/com/google/devtools/mobileharness/infra/client/api/controller/job/retry/strategy:flaky_test_retry_constants",
3335
"//src/java/com/google/devtools/mobileharness/platform/android/instrumentation/result:test_suite_result_loader",
3436
"//src/java/com/google/wireless/qa/mobileharness/shared/constant:property",
3537
"//src/java/com/google/wireless/qa/mobileharness/shared/model/job",

0 commit comments

Comments
 (0)