Skip to content

Commit 3c1014a

Browse files
OmniLab Teamcopybara-github
OmniLab Team
authored and
copybara-github
committed
Internal change
PiperOrigin-RevId: 915783524
1 parent f872211 commit 3c1014a

5 files changed

Lines changed: 290 additions & 41 deletions

File tree

‎src/java/com/google/devtools/mobileharness/platform/android/shared/emulator/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ java_library(
8181
"//src/java/com/google/devtools/mobileharness/api/model/error",
8282
"//src/java/com/google/devtools/mobileharness/shared/util/auto:auto_value",
8383
"//src/java/com/google/devtools/mobileharness/shared/util/concurrent/retry",
84+
"//src/java/com/google/devtools/mobileharness/shared/util/file/local",
8485
"//src/java/com/google/devtools/mobileharness/shared/util/logging:google_logger",
8586
"@maven//:com_google_code_findbugs_jsr305",
8687
"@maven//:com_google_code_gson_gson",

‎src/java/com/google/devtools/mobileharness/platform/android/shared/emulator/CloudOrchestratorClient.java‎

Lines changed: 143 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.google.devtools.mobileharness.platform.android.shared.emulator;
1818

19+
import static com.google.common.base.Strings.isNullOrEmpty;
1920
import static com.google.common.base.Strings.nullToEmpty;
2021
import static java.nio.charset.StandardCharsets.UTF_8;
2122

@@ -37,6 +38,7 @@
3738
import com.google.common.annotations.VisibleForTesting;
3839
import com.google.common.base.Stopwatch;
3940
import com.google.common.collect.ImmutableList;
41+
import com.google.common.flogger.FluentLogger;
4042
import com.google.common.io.Files;
4143
import com.google.devtools.mobileharness.api.model.error.AndroidErrorId;
4244
import com.google.devtools.mobileharness.api.model.error.BasicErrorId;
@@ -56,22 +58,29 @@
5658
import com.google.devtools.mobileharness.shared.util.concurrent.retry.RetryException;
5759
import com.google.devtools.mobileharness.shared.util.concurrent.retry.RetryStrategy;
5860
import com.google.devtools.mobileharness.shared.util.concurrent.retry.RetryingCallable;
61+
import com.google.devtools.mobileharness.shared.util.file.local.LocalFileUtil;
5962
import com.google.errorprone.annotations.CanIgnoreReturnValue;
6063
import com.google.gson.stream.MalformedJsonException;
6164
import java.io.File;
65+
import java.io.FileOutputStream;
6266
import java.io.IOException;
6367
import java.time.Duration;
6468
import java.util.Base64;
6569
import java.util.HashMap;
6670
import java.util.List;
6771
import java.util.Map;
72+
import java.util.regex.Matcher;
73+
import java.util.regex.Pattern;
6874
import javax.annotation.Nullable;
6975

7076
/** Java client for Cloud Orchestrator API. */
7177
public class CloudOrchestratorClient {
7278

79+
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
7380
private static final JsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance();
7481
private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
82+
private static final Pattern LOG_LINK_PATTERN =
83+
Pattern.compile("<a[^>]*href=\"([^\"]+)\"[^>]*>", Pattern.CASE_INSENSITIVE);
7584

7685
private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30);
7786
private static final Duration WAIT_OPERATION_MAX_TIMEOUT = Duration.ofMinutes(10);
@@ -600,6 +609,139 @@ private <T> T put(String path, @Nullable Object body, Class<T> responseClass)
600609
}
601610
}
602611

612+
/**
613+
* Downloads a file from the given path to the target file.
614+
*
615+
* @param path the path relative to the root endpoint
616+
* @param targetFile the file to save the content to
617+
* @throws MobileHarnessException if the download fails
618+
*/
619+
public void downloadFile(String path, File targetFile) throws MobileHarnessException {
620+
int maxRetries = 3;
621+
LocalFileUtil localFileUtil = new LocalFileUtil();
622+
localFileUtil.prepareParentDir(targetFile.getAbsolutePath());
623+
try {
624+
RetryingCallable.newBuilder(
625+
() -> {
626+
GenericUrl url = new GenericUrl(rootEndpoint + path);
627+
HttpRequest request = requestFactory.buildRequest("GET", url, null);
628+
prepareRequest(request);
629+
HttpResponse response = request.execute();
630+
try (FileOutputStream out = new FileOutputStream(targetFile)) {
631+
response.download(out);
632+
} finally {
633+
response.disconnect();
634+
}
635+
return null;
636+
},
637+
RetryStrategy.exponentialBackoff(requestRetryDelay, 2, maxRetries))
638+
.setPredicate(this::isRetriable)
639+
.build()
640+
.call();
641+
} catch (RetryException e) {
642+
Throwable cause = e.getCause();
643+
if (cause instanceof InterruptedException) {
644+
Thread.currentThread().interrupt();
645+
throw new MobileHarnessException(
646+
BasicErrorId.LOCAL_NETWORK_ERROR,
647+
String.format("Interrupted while downloading file from %s", path),
648+
cause);
649+
}
650+
if (cause instanceof HttpResponseException httpResponseException) {
651+
throw handleHttpException(
652+
String.format("Failed to download file from %s", path), httpResponseException);
653+
}
654+
throw new MobileHarnessException(
655+
BasicErrorId.LOCAL_NETWORK_ERROR,
656+
String.format("Failed to download file from %s", path),
657+
e);
658+
}
659+
}
660+
661+
/**
662+
* Fetches all logs from the logs directory and saves them to the target directory.
663+
*
664+
* @param hostId the ID of the host
665+
* @param cvdGroup the group of the CVD
666+
* @param cvdName the name of the CVD
667+
* @param targetDir the directory to save the logs to
668+
* @throws MobileHarnessException if the operation fails
669+
*/
670+
public void fetchAllLogs(String hostId, String cvdGroup, String cvdName, File targetDir)
671+
throws MobileHarnessException {
672+
String logsPath = "/hosts/" + hostId + "/cvds/" + cvdGroup + "/" + cvdName + "/logs/";
673+
int maxRetries = 3;
674+
String htmlContent;
675+
try {
676+
htmlContent =
677+
RetryingCallable.newBuilder(
678+
() -> {
679+
GenericUrl url = new GenericUrl(rootEndpoint + logsPath);
680+
HttpRequest request = requestFactory.buildRequest("GET", url, null);
681+
prepareRequest(request);
682+
HttpResponse response = request.execute();
683+
try {
684+
return response.parseAsString();
685+
} finally {
686+
response.disconnect();
687+
}
688+
},
689+
RetryStrategy.exponentialBackoff(requestRetryDelay, 2, maxRetries))
690+
.setPredicate(this::isRetriable)
691+
.build()
692+
.call();
693+
} catch (RetryException e) {
694+
Throwable cause = e.getCause();
695+
if (cause instanceof InterruptedException) {
696+
Thread.currentThread().interrupt();
697+
throw new MobileHarnessException(
698+
BasicErrorId.LOCAL_NETWORK_ERROR,
699+
String.format("Interrupted while fetching logs directory listing from %s", logsPath),
700+
cause);
701+
}
702+
if (cause instanceof HttpResponseException httpResponseException) {
703+
throw handleHttpException(
704+
String.format("Failed to fetch logs directory listing from %s", logsPath),
705+
httpResponseException);
706+
}
707+
throw new MobileHarnessException(
708+
BasicErrorId.LOCAL_NETWORK_ERROR,
709+
String.format("Failed to fetch logs directory listing from %s", logsPath),
710+
e);
711+
}
712+
713+
if (isNullOrEmpty(htmlContent)) {
714+
return;
715+
}
716+
717+
// Parse HTML to find links
718+
Matcher matcher = LOG_LINK_PATTERN.matcher(htmlContent);
719+
720+
while (matcher.find()) {
721+
String filename = matcher.group(1);
722+
if (filename.endsWith("/") || filename.equals("..")) {
723+
continue;
724+
}
725+
File targetFile = new File(targetDir, new File(filename).getName());
726+
try {
727+
logger.atInfo().log(
728+
"Downloading log file %s to %s", filename, targetFile.getAbsolutePath());
729+
downloadFile(logsPath + filename, targetFile);
730+
} catch (MobileHarnessException e) {
731+
logger.atWarning().withCause(e).log("Failed to download log file %s", filename);
732+
}
733+
}
734+
}
735+
736+
private void prepareRequest(HttpRequest request) {
737+
if (sessionCookie != null) {
738+
request.getHeaders().set("Cookie", sessionCookie);
739+
}
740+
if (basicAuth != null) {
741+
request.getHeaders().set("Authorization", "Basic " + basicAuth);
742+
}
743+
}
744+
603745
private <T> T executeRequest(
604746
String method,
605747
String path,
@@ -634,12 +776,7 @@ private <T> T executeRequest(
634776
if (buildApiCreds != null) {
635777
request.getHeaders().set("X-Cutf-Host-Orchestrator-BuildAPI-Creds", buildApiCreds);
636778
}
637-
if (sessionCookie != null) {
638-
request.getHeaders().set("Cookie", sessionCookie);
639-
}
640-
if (basicAuth != null) {
641-
request.getHeaders().set("Authorization", "Basic " + basicAuth);
642-
}
779+
prepareRequest(request);
643780
if (timeout != null) {
644781
request.setConnectTimeout((int) timeout.toMillis());
645782
request.setReadTimeout((int) timeout.toMillis());

‎src/java/com/google/wireless/qa/mobileharness/shared/api/device/AndroidJitEmulator.java‎

Lines changed: 69 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.google.devtools.mobileharness.platform.android.shared.emulator.CloudOrchestratorMessages.Cvd;
3535
import com.google.devtools.mobileharness.platform.android.shared.emulator.CloudOrchestratorMessages.DockerInstance;
3636
import com.google.devtools.mobileharness.platform.android.shared.emulator.CloudOrchestratorMessages.HostInstance;
37+
import com.google.devtools.mobileharness.shared.util.file.local.LocalFileUtil;
3738
import com.google.devtools.mobileharness.shared.util.flags.Flags;
3839
import com.google.devtools.mobileharness.shared.util.time.Sleeper;
3940
import com.google.errorprone.annotations.CanIgnoreReturnValue;
@@ -92,9 +93,14 @@ public class AndroidJitEmulator extends AndroidDevice {
9293
@Nullable
9394
private String cvdGroup;
9495

96+
@GuardedBy("cloudOrchestratorLock")
97+
@Nullable
98+
private String cvdName;
99+
95100
@Nullable private AdbWebSocketBridge adbWebSocketBridge;
96101

97102
private final AdbWebSocketBridgeFactory adbWebSocketBridgeFactory;
103+
private final LocalFileUtil localFileUtil;
98104

99105
public AndroidJitEmulator(String deviceId) {
100106
this(
@@ -119,6 +125,7 @@ public AndroidJitEmulator(String deviceId) {
119125
super(deviceId, apiConfig, validatorFactory);
120126
this.deviceId = deviceId;
121127
this.androidAdbUtil = androidAdbUtil;
128+
this.localFileUtil = new LocalFileUtil();
122129
this.cloudOrchestratorClientFactory = cloudOrchestratorClientFactory;
123130
this.adbWebSocketBridgeFactory = adbWebSocketBridgeFactory;
124131
this.sleeper = sleeper;
@@ -171,7 +178,8 @@ public void preRunTest(TestInfo testInfo) throws MobileHarnessException, Interru
171178
} catch (MobileHarnessException | InterruptedException | RuntimeException e) {
172179
logger.atWarning().log("Failed to set up JIT emulator. Cleaning up...");
173180
stopWebSocketBridgeAndDisconnectAdb();
174-
deleteCvd(serviceUrl);
181+
fetchLogsWithClient(serviceUrl, testInfo);
182+
deleteCvdWithClient(serviceUrl);
175183
throw e;
176184
} finally {
177185
cloudOrchestratorLock.unlock();
@@ -203,6 +211,10 @@ private void createCvdWithClient(String serviceUrl, TestInfo testInfo)
203211
hostId = hosts.get(0).name;
204212
}
205213

214+
this.cvdHostId = hostId;
215+
this.cvdGroup = cvdId;
216+
this.cvdName = cvdId;
217+
206218
String branch = testInfo.jobInfo().params().get(PARAM_BRANCH, "aosp-android-latest-release");
207219
String target =
208220
testInfo.jobInfo().params().get(PARAM_TARGET, "aosp_cf_x86_64_only_phone-userdebug");
@@ -246,8 +258,8 @@ private void createCvdWithClient(String serviceUrl, TestInfo testInfo)
246258
}
247259

248260
logger.atInfo().log("Created CVD: %s, ADB Serial: %s", cvd.name, cvd.adbSerial);
249-
this.cvdHostId = hostId;
250261
this.cvdGroup = cvd.group;
262+
this.cvdName = cvd.name;
251263

252264
int localPort = AndroidJitEmulatorUtil.getPortFromDeviceId(deviceId);
253265
if (localPort > 0) {
@@ -281,10 +293,57 @@ public PostTestDeviceOp postRunTest(TestInfo testInfo)
281293
}
282294

283295
stopWebSocketBridgeAndDisconnectAdb();
284-
deleteCvd(Flags.cloudOrchestratorServiceUrl.getNonNull());
296+
297+
String serviceUrl = Flags.cloudOrchestratorServiceUrl.getNonNull();
298+
if (serviceUrl.isEmpty()) {
299+
logger.atWarning().log(
300+
"cloud_orchestrator_service_url flag is NOT set, skipping log fetch and CVD deletion.");
301+
return PostTestDeviceOp.NONE;
302+
}
303+
304+
logger.atInfo().log("Waiting for Cloud Orchestrator lock for cleanup...");
305+
try {
306+
cloudOrchestratorLock.lockInterruptibly();
307+
try {
308+
logger.atInfo().log("Acquired Cloud Orchestrator lock for cleanup.");
309+
fetchLogsWithClient(serviceUrl, testInfo);
310+
deleteCvdWithClient(serviceUrl);
311+
} finally {
312+
cloudOrchestratorLock.unlock();
313+
}
314+
} catch (InterruptedException e) {
315+
logger.atWarning().withCause(e).log("Interrupted during cleanup lock");
316+
Thread.currentThread().interrupt();
317+
}
285318
return PostTestDeviceOp.NONE;
286319
}
287320

321+
@GuardedBy("cloudOrchestratorLock")
322+
private void fetchLogsWithClient(String serviceUrl, TestInfo testInfo) {
323+
if (cvdHostId == null || cvdGroup == null || cvdName == null) {
324+
logger.atWarning().log("Skipping log fetch as cvdHostId, cvdGroup, or cvdName is null");
325+
return;
326+
}
327+
String zone = Flags.cloudOrchestratorZone.getNonNull();
328+
CloudOrchestratorClient client = cloudOrchestratorClientFactory.create(serviceUrl, "v1", zone);
329+
client.setBasicAuth("user", "");
330+
331+
File targetDir;
332+
try {
333+
targetDir = new File(testInfo.getGenFileDir(), deviceId + "/cvd_logs");
334+
localFileUtil.prepareDir(targetDir.getAbsolutePath());
335+
} catch (MobileHarnessException e) {
336+
logger.atWarning().withCause(e).log("Failed to prepare gen file dir, skipping log fetch");
337+
return;
338+
}
339+
340+
try {
341+
client.fetchAllLogs(cvdHostId, cvdGroup, cvdName, targetDir);
342+
} catch (MobileHarnessException e) {
343+
logger.atWarning().withCause(e).log("Failed to fetch all logs");
344+
}
345+
}
346+
288347
private void stopWebSocketBridgeAndDisconnectAdb() {
289348
if (adbWebSocketBridge != null) {
290349
try {
@@ -305,43 +364,12 @@ private void stopWebSocketBridgeAndDisconnectAdb() {
305364
}
306365
}
307366

308-
private void deleteCvd(String serviceUrl) {
309-
if (serviceUrl.isEmpty()) {
310-
logger.atWarning().log(
311-
"cloud_orchestrator_service_url flag is NOT set, skipping CVD deletion.");
312-
return;
313-
}
314-
logger.atInfo().log("Waiting for Cloud Orchestrator lock for cleanup...");
315-
try {
316-
cloudOrchestratorLock.lockInterruptibly();
317-
try {
318-
logger.atInfo().log("Acquired Cloud Orchestrator lock for cleanup.");
319-
deleteCvdWithClient(serviceUrl);
320-
} finally {
321-
cloudOrchestratorLock.unlock();
322-
}
323-
} catch (InterruptedException e) {
324-
logger.atWarning().withCause(e).log("Interrupted during CVD deletion lock");
325-
Thread.currentThread().interrupt();
326-
} catch (MobileHarnessException e) {
327-
logger.atWarning().withCause(e).log("Failed to delete CVD during cleanup");
328-
}
329-
}
330-
331367
@GuardedBy("cloudOrchestratorLock")
332-
private void deleteCvdWithClient(String serviceUrl)
333-
throws MobileHarnessException, InterruptedException {
368+
private void deleteCvdWithClient(String serviceUrl) throws InterruptedException {
334369
String zone = Flags.cloudOrchestratorZone.getNonNull();
335370
CloudOrchestratorClient client = cloudOrchestratorClientFactory.create(serviceUrl, "v1", zone);
336371
client.setBasicAuth("user", "");
337372

338-
List<HostInstance> hosts = client.listHosts();
339-
if (hosts == null || hosts.isEmpty()) {
340-
logger.atWarning().log("No hosts found, skipping CVD deletion.");
341-
return;
342-
}
343-
String hostId = hosts.get(0).name;
344-
345373
if (cvdHostId != null && cvdGroup != null) {
346374
try {
347375
logger.atInfo().log("Deleting CVD %s on host %s", cvdGroup, cvdHostId);
@@ -355,6 +383,12 @@ private void deleteCvdWithClient(String serviceUrl)
355383

356384
// Fallback: list and delete if we don't have the specific CVD info.
357385
try {
386+
List<HostInstance> hosts = client.listHosts();
387+
if (hosts == null || hosts.isEmpty()) {
388+
logger.atWarning().log("No hosts found, skipping CVD deletion.");
389+
return;
390+
}
391+
String hostId = hosts.get(0).name;
358392
List<Cvd> cvds = client.listCvds(hostId);
359393
if (cvds != null) {
360394
for (Cvd cvd : cvds) {

‎src/java/com/google/wireless/qa/mobileharness/shared/api/device/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ java_library(
152152
"//src/java/com/google/devtools/mobileharness/platform/android/shared/emulator:adb_websocket_bridge",
153153
"//src/java/com/google/devtools/mobileharness/platform/android/shared/emulator:android_jit_emulator_util",
154154
"//src/java/com/google/devtools/mobileharness/platform/android/shared/emulator:cloud_orchestrator_client",
155+
"//src/java/com/google/devtools/mobileharness/shared/util/file/local",
155156
"//src/java/com/google/devtools/mobileharness/shared/util/flags",
156157
"//src/java/com/google/devtools/mobileharness/shared/util/logging:google_logger",
157158
"//src/java/com/google/devtools/mobileharness/shared/util/time:sleeper",

0 commit comments

Comments
 (0)