Skip to content

Commit 9e9f144

Browse files
kaxunakaxuna
authored
Refactored Telemetry object (#918)
* changed Telemetry object to follow open to extension closed to modification principle * changed enum names * nit value change * changed some variable names * nit: precommit
1 parent 6c13a98 commit 9e9f144

File tree

14 files changed

+309
-112
lines changed

14 files changed

+309
-112
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/Main.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
public class Main {
3030

3131
private static final Logger logger = LoggerFactory.getLogger(Main.class);
32+
private static final TelemetryProcessor telemetryProcessor = new TelemetryProcessor();
3233

3334
private final MetadataDumper metadataDumper;
3435

@@ -62,9 +63,11 @@ private static void printErrorMessages(Throwable e) {
6263

6364
public static void main(String... args) throws Exception {
6465
try {
65-
StartUpMetainformationPrinter.printMetainfo();
66+
StartUpMetaInfoProcessor.printMetaInfo();
67+
telemetryProcessor.setDumperMetadata(StartUpMetaInfoProcessor.getDumperMetadata());
68+
69+
Main main = new Main(new MetadataDumper(telemetryProcessor));
6670

67-
Main main = new Main(new MetadataDumper(new DumperRunMetricsGenerator()));
6871
if (args.length == 0) {
6972
args = new String[] {"--help"};
7073
}

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,10 @@ public class MetadataDumper {
6161
private static final Pattern GCS_PATH_PATTERN =
6262
Pattern.compile("gs://(?<bucket>[^/]+)/(?<path>.*)");
6363

64-
private final DumperRunMetricsGenerator dumperRunMetricsGenerator;
64+
private final TelemetryProcessor telemetryProcessor;
6565

66-
public MetadataDumper(DumperRunMetricsGenerator dumperRunMetricsGenerator) {
67-
this.dumperRunMetricsGenerator = dumperRunMetricsGenerator;
66+
public MetadataDumper(TelemetryProcessor telemetryProcessor) {
67+
this.telemetryProcessor = telemetryProcessor;
6868
}
6969

7070
public boolean run(String... args) throws Exception {
@@ -192,10 +192,9 @@ protected boolean run(@Nonnull Connector connector, @Nonnull ConnectorArguments
192192

193193
requiredTaskSucceeded = checkRequiredTaskSuccess(summaryPrinter, state, outputFileLocation);
194194

195-
dumperRunMetricsGenerator.generateRunMetrics(
196-
fileSystem, arguments, state, stopwatch, requiredTaskSucceeded);
197-
} catch (IOException e) {
198-
logger.warn("Unable to generate dumper run metrics");
195+
telemetryProcessor.addDumperRunMetricsToPayload(
196+
arguments, state, stopwatch, requiredTaskSucceeded);
197+
telemetryProcessor.processTelemetry(fileSystem);
199198
} finally {
200199
// We must do this in finally after the ZipFileSystem has been closed.
201200
File outputFile = new File(outputFileLocation);
@@ -226,7 +225,7 @@ private String getOutputFileLocation(Connector connector, ConnectorArguments arg
226225
return arguments
227226
.getOutputFile()
228227
.map(file -> getVerifiedFile(defaultFileName, file))
229-
.orElseGet(() -> defaultFileName);
228+
.orElse(defaultFileName);
230229
}
231230

232231
private String getVerifiedFile(String defaultFileName, String fileName) {

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/StartUpMetainformationPrinter.java renamed to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/StartUpMetaInfoProcessor.java

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import static java.util.jar.Attributes.Name.IMPLEMENTATION_TITLE;
2020

21+
import com.google.edwmigration.dumper.application.dumper.metrics.DumperMetadata;
2122
import java.io.BufferedReader;
2223
import java.io.InputStream;
2324
import java.io.InputStreamReader;
@@ -27,20 +28,26 @@
2728
import java.util.jar.Manifest;
2829
import org.springframework.core.io.ClassPathResource;
2930

30-
public class StartUpMetainformationPrinter {
31+
public class StartUpMetaInfoProcessor {
3132

32-
public static void printMetainfo() {
33+
private static final DumperMetadata dumperMetadata = generateDumperMetadata();
34+
35+
public static void printMetaInfo() {
3336
try {
3437
printBanner();
35-
printMetainfFile();
38+
printMetaInfoFile();
3639
} catch (Exception ignore) {
3740
}
3841
}
3942

43+
public static DumperMetadata getDumperMetadata() {
44+
return dumperMetadata;
45+
}
46+
4047
private static void printBanner() {
4148
try {
4249
ClassPathResource classPathResource =
43-
new ClassPathResource("/banner/banner.txt", StartUpMetainformationPrinter.class);
50+
new ClassPathResource("/banner/banner.txt", StartUpMetaInfoProcessor.class);
4451
try (BufferedReader reader =
4552
new BufferedReader(new InputStreamReader(classPathResource.getInputStream()))) {
4653
reader.lines().forEach(System.out::println);
@@ -49,26 +56,40 @@ private static void printBanner() {
4956
}
5057
}
5158

52-
private static void printMetainfFile() {
59+
private static void printMetaInfoFile() {
60+
if (dumperMetadata == null) {
61+
return;
62+
}
63+
64+
System.out.println(
65+
"App version: ["
66+
+ dumperMetadata.getVersion()
67+
+ "], change: ["
68+
+ dumperMetadata.getGitCommit()
69+
+ "]");
70+
System.out.println("Build date: " + dumperMetadata.getBuildDate());
71+
System.out.println();
72+
}
73+
74+
private static DumperMetadata generateDumperMetadata() {
5375
Manifest manifest = loadCurrentClassManifest();
76+
5477
if (manifest == null) {
55-
return;
78+
return null;
5679
}
5780

5881
String buildDate = manifest.getMainAttributes().getValue("Build-Date-UTC");
5982
String change = manifest.getMainAttributes().getValue("Change");
6083
String version = manifest.getMainAttributes().getValue("Implementation-Version");
6184

62-
System.out.println("App version: [" + version + "], change: [" + change + "]");
63-
System.out.println("Build date: " + buildDate);
64-
System.out.println();
85+
return new DumperMetadata(version, change, buildDate);
6586
}
6687

6788
private static Manifest loadCurrentClassManifest() {
6889
try {
6990
final String implementationTitle =
70-
StartUpMetainformationPrinter.class.getPackage().getImplementationTitle();
71-
final ClassLoader classLoader = StartUpMetainformationPrinter.class.getClassLoader();
91+
StartUpMetaInfoProcessor.class.getPackage().getImplementationTitle();
92+
final ClassLoader classLoader = StartUpMetaInfoProcessor.class.getClassLoader();
7293

7394
Enumeration<URL> manifestResources = classLoader.getResources("META-INF/MANIFEST.MF");
7495
while (manifestResources.hasMoreElements()) {

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/TelemetryProcessor.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright 2022-2025 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.edwmigration.dumper.application.dumper;
18+
19+
import com.google.common.base.Stopwatch;
20+
import com.google.edwmigration.dumper.application.dumper.metrics.*;
21+
import com.google.edwmigration.dumper.application.dumper.task.TaskSetState;
22+
import java.nio.file.FileSystem;
23+
import java.time.Duration;
24+
import java.time.ZonedDateTime;
25+
import java.util.List;
26+
import java.util.UUID;
27+
import java.util.stream.Collectors;
28+
import org.slf4j.Logger;
29+
import org.slf4j.LoggerFactory;
30+
31+
public class TelemetryProcessor {
32+
private static final Logger logger = LoggerFactory.getLogger(TelemetryProcessor.class);
33+
34+
private final ClientTelemetry clientTelemetry;
35+
36+
public TelemetryProcessor() {
37+
clientTelemetry = new ClientTelemetry();
38+
}
39+
40+
public void setDumperMetadata(DumperMetadata dumperMetadata) {
41+
clientTelemetry.setDumperMetadata(dumperMetadata);
42+
}
43+
44+
/**
45+
* Generates a list of strings representing the summary for the current run. This summary does NOT
46+
* include the final ZIP file size, as it's generated before the ZIP is closed.
47+
*/
48+
public void addDumperRunMetricsToPayload(
49+
ConnectorArguments arguments, TaskSetState state, Stopwatch stopwatch, boolean success) {
50+
try {
51+
clientTelemetry.setEventType(EventType.DUMPER_RUN_METRICS);
52+
53+
List<TaskExecutionSummary> taskExecutionSummaries =
54+
state.getTasksReports().stream()
55+
.map(
56+
tasksReport ->
57+
new TaskExecutionSummary(tasksReport.count(), tasksReport.state().name()))
58+
.collect(Collectors.toList());
59+
60+
List<TaskDetailedSummary> taskDetailedSummaries =
61+
state.getTaskResultSummaries().stream()
62+
.map(
63+
item ->
64+
new TaskDetailedSummary(
65+
item.getTask().getName(),
66+
item.getTask().getCategory().name(),
67+
item.getTaskState().name(),
68+
item.getThrowable().isPresent()
69+
? item.getThrowable().get().getMessage()
70+
: null))
71+
.collect(Collectors.toList());
72+
73+
Duration elapsed = stopwatch.elapsed();
74+
75+
DumperRunMetrics dumperRunMetrics =
76+
DumperRunMetrics.builder()
77+
.setId(UUID.randomUUID().toString())
78+
.setMeasureStartTime(ZonedDateTime.now().minus(elapsed))
79+
.setRunDurationInMinutes(elapsed.getSeconds() / 60)
80+
.setOverallStatus(success ? "SUCCESS" : "FAILURE")
81+
.setTaskExecutionSummary(taskExecutionSummaries)
82+
.setTaskDetailedSummary(taskDetailedSummaries)
83+
.setArguments(arguments)
84+
.build();
85+
clientTelemetry.addToPayload(dumperRunMetrics);
86+
} catch (Exception e) {
87+
logger.warn("Failed to generate dumperRunMetrics and add it to payload", e);
88+
}
89+
}
90+
91+
public void processTelemetry(FileSystem fileSystem) {
92+
try {
93+
TelemetryWriter.write(fileSystem, clientTelemetry);
94+
} catch (Exception e) {
95+
logger.warn("Failed to write telemetry", e);
96+
}
97+
}
98+
}

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/DumperRunMetricsGenerator.java renamed to dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/TelemetryWriter.java

Lines changed: 7 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -23,29 +23,20 @@
2323
import com.fasterxml.jackson.databind.ObjectMapper;
2424
import com.fasterxml.jackson.databind.SerializationFeature;
2525
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
26-
import com.google.common.base.Stopwatch;
27-
import com.google.edwmigration.dumper.application.dumper.metrics.DumperRunMetrics;
28-
import com.google.edwmigration.dumper.application.dumper.metrics.TaskDetailedSummary;
29-
import com.google.edwmigration.dumper.application.dumper.metrics.TaskExecutionSummary;
30-
import com.google.edwmigration.dumper.application.dumper.task.TaskSetState;
26+
import com.google.edwmigration.dumper.application.dumper.metrics.ClientTelemetry;
3127
import java.io.BufferedWriter;
3228
import java.io.IOException;
3329
import java.io.PrintWriter;
3430
import java.nio.charset.StandardCharsets;
3531
import java.nio.file.*;
36-
import java.time.Duration;
37-
import java.time.LocalDateTime;
38-
import java.util.List;
39-
import java.util.UUID;
40-
import java.util.stream.Collectors;
4132
import net.harawata.appdirs.AppDirs;
4233
import net.harawata.appdirs.AppDirsFactory;
4334
import org.slf4j.Logger;
4435
import org.slf4j.LoggerFactory;
4536

46-
public class DumperRunMetricsGenerator {
37+
public class TelemetryWriter {
4738

48-
private static final Logger logger = LoggerFactory.getLogger(DumperRunMetricsGenerator.class);
39+
private static final Logger logger = LoggerFactory.getLogger(TelemetryWriter.class);
4940
private static final String ALL_DUMPER_RUN_METRICS = "all-dumper-telemetry.jsonl";
5041
private static final String DUMPER_RUN_METRICS = "dumper-telemetry.jsonl";
5142
private static final ObjectMapper MAPPER = createObjectMapper();
@@ -60,31 +51,23 @@ private static ObjectMapper createObjectMapper() {
6051
return mapper;
6152
}
6253

63-
public void generateRunMetrics(
64-
FileSystem fileSystem,
65-
ConnectorArguments arguments,
66-
TaskSetState state,
67-
Stopwatch stopwatch,
68-
boolean requiredTaskSucceeded)
54+
public static void write(FileSystem fileSystem, ClientTelemetry clientTelemetry)
6955
throws IOException {
7056

71-
DumperRunMetrics currentDumperRunMetrics =
72-
generateCurrentDumperRunMetrics(arguments, state, stopwatch, requiredTaskSucceeded);
73-
7457
String cacheDir = createDirPathIfNotExist();
7558

7659
Path PathToCachedCumulativeSummary = Paths.get(cacheDir + ALL_DUMPER_RUN_METRICS);
7760

7861
try {
79-
String serializedMetrics = MAPPER.writeValueAsString(currentDumperRunMetrics);
62+
String serializedMetrics = MAPPER.writeValueAsString(clientTelemetry);
8063
appendToCacheOnDisk(PathToCachedCumulativeSummary, serializedMetrics);
8164
copyCachedAsCurrent(fileSystem, PathToCachedCumulativeSummary);
8265
} catch (JsonProcessingException e) {
8366
logger.warn("Failed to serialize dumperRunMetrics", e);
8467
}
8568
}
8669

87-
private String createDirPathIfNotExist() throws IOException {
70+
private static String createDirPathIfNotExist() throws IOException {
8871
AppDirs appDirs = AppDirsFactory.getInstance();
8972

9073
String appName = "DWH-Dumper";
@@ -122,48 +105,8 @@ private static void copyCachedAsCurrent(FileSystem zipFs, Path externalLogPath)
122105
}
123106
}
124107

125-
/**
126-
* Generates a list of strings representing the summary for the current run. This summary does NOT
127-
* include the final ZIP file size, as it's generated before the ZIP is closed.
128-
*/
129-
private DumperRunMetrics generateCurrentDumperRunMetrics(
130-
ConnectorArguments arguments, TaskSetState state, Stopwatch stopwatch, boolean success) {
131-
132-
List<TaskExecutionSummary> taskExecutionSummaries =
133-
state.getTasksReports().stream()
134-
.map(
135-
tasksReport ->
136-
new TaskExecutionSummary(tasksReport.count(), tasksReport.state().name()))
137-
.collect(Collectors.toList());
138-
139-
List<TaskDetailedSummary> taskDetailedSummaries =
140-
state.getTaskResultSummaries().stream()
141-
.map(
142-
item ->
143-
new TaskDetailedSummary(
144-
item.getTask().getName(),
145-
item.getTask().getCategory().name(),
146-
item.getTaskState().name(),
147-
item.getThrowable().isPresent()
148-
? item.getThrowable().get().getMessage()
149-
: null))
150-
.collect(Collectors.toList());
151-
152-
Duration elapsed = stopwatch.elapsed();
153-
154-
return DumperRunMetrics.builder()
155-
.setId(UUID.randomUUID().toString())
156-
.setRunStartTime(LocalDateTime.now().minus(elapsed))
157-
.setRunDurationInSeconds(elapsed.getSeconds())
158-
.setOverallStatus(success ? "SUCCESS" : "FAILURE")
159-
.setTaskExecutionSummary(taskExecutionSummaries)
160-
.setTaskDetailedSummary(taskDetailedSummaries)
161-
.setArguments(arguments)
162-
.build();
163-
}
164-
165108
/** Appends the given summary lines for the current run to the external log file. */
166-
private void appendToCacheOnDisk(Path externalLogPath, String summaryLines) {
109+
private static void appendToCacheOnDisk(Path externalLogPath, String summaryLines) {
167110
try (BufferedWriter writer =
168111
newBufferedWriter(
169112
externalLogPath,

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/clouddumper/Main.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
import com.google.cloud.kms.v1.CryptoKeyName;
2020
import com.google.cloud.kms.v1.DecryptResponse;
2121
import com.google.cloud.kms.v1.KeyManagementServiceClient;
22-
import com.google.edwmigration.dumper.application.dumper.DumperRunMetricsGenerator;
2322
import com.google.edwmigration.dumper.application.dumper.MetadataDumper;
2423
import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
24+
import com.google.edwmigration.dumper.application.dumper.TelemetryProcessor;
2525
import com.google.gson.Gson;
2626
import com.google.protobuf.ByteString;
2727
import java.io.IOException;
@@ -100,7 +100,7 @@ public static void main(String... args) throws Exception {
100100
/* maxRetries= */ 3, /* defaultRetryInterval= */ TimeValue.ofSeconds(1L)))
101101
.build()) {
102102
new Main(
103-
() -> new MetadataDumper(new DumperRunMetricsGenerator()),
103+
() -> new MetadataDumper(new TelemetryProcessor()),
104104
new HttpClientMetadataRetriever(httpClient),
105105
DriverRetriever.create(httpClient, Files.createTempDirectory("clouddumper")))
106106
.run();

0 commit comments

Comments
 (0)