Skip to content

Commit 52b7139

Browse files
lszydelko-googlelszydelko-google
committed
reshape solution - extract TaskLoggingUtil
1 parent d276f4f commit 52b7139

File tree

5 files changed

+235
-197
lines changed

5 files changed

+235
-197
lines changed

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/TaskLoggingUtil.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Copyright 2022-2025 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.edwmigration.dumper.application.dumper;
18+
19+
import static com.google.edwmigration.dumper.application.dumper.DurationFormatter.formatApproximateDuration;
20+
import static java.lang.Math.max;
21+
22+
import com.google.common.base.Stopwatch;
23+
import java.time.Duration;
24+
import java.util.ArrayDeque;
25+
import java.util.Arrays;
26+
import java.util.Collections;
27+
import java.util.Deque;
28+
import java.util.concurrent.atomic.AtomicInteger;
29+
30+
class TaskLoggingUtil {
31+
private final int TASK_QUEUE_SIZE = 10;
32+
AtomicInteger numberOfCompletedTasks;
33+
final int totalNumberOfTasks;
34+
final Stopwatch stopwatch;
35+
final Deque<Duration> lastTaskDurations = new ArrayDeque<>(TASK_QUEUE_SIZE);
36+
37+
public TaskLoggingUtil(int totalNumberOfTasks) {
38+
this.totalNumberOfTasks = totalNumberOfTasks;
39+
stopwatch = Stopwatch.createStarted();
40+
numberOfCompletedTasks = new AtomicInteger();
41+
}
42+
43+
private Duration getAverageTaskDurationFromAllTasks() {
44+
return stopwatch.elapsed().dividedBy(max(1, numberOfCompletedTasks.get()));
45+
}
46+
47+
private Duration getAverageTaskDurationFromLatestTasks() {
48+
if (lastTaskDurations.isEmpty()) {
49+
return Duration.ZERO;
50+
}
51+
Duration total = lastTaskDurations.getLast().minus(lastTaskDurations.getFirst());
52+
53+
return total.dividedBy(lastTaskDurations.size());
54+
}
55+
56+
private Duration getTaskDuration() {
57+
return Collections.max(
58+
Arrays.asList(
59+
getAverageTaskDurationFromAllTasks(), getAverageTaskDurationFromLatestTasks()));
60+
}
61+
62+
public void recordTaskCompletion() {
63+
numberOfCompletedTasks.getAndIncrement();
64+
65+
Duration taskDuration = stopwatch.elapsed();
66+
if (lastTaskDurations.size() == TASK_QUEUE_SIZE) {
67+
lastTaskDurations.removeFirst();
68+
}
69+
lastTaskDurations.addLast(taskDuration);
70+
}
71+
72+
public String getProgressLog() {
73+
int completedTasks = numberOfCompletedTasks.get();
74+
75+
Duration averageTimePerTask = getTaskDuration();
76+
77+
int percentFinished = completedTasks * 100 / totalNumberOfTasks;
78+
String progressMessage = percentFinished + "% Completed";
79+
80+
int remainingTasks = totalNumberOfTasks - completedTasks;
81+
Duration remainingTime = averageTimePerTask.multipliedBy(remainingTasks);
82+
83+
if (completedTasks > TASK_QUEUE_SIZE && remainingTasks > 0) {
84+
progressMessage += ". ETA: " + formatApproximateDuration(remainingTime);
85+
}
86+
87+
return progressMessage;
88+
}
89+
}

‎dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/TasksRunner.java‎

Lines changed: 4 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,7 @@
1616
*/
1717
package com.google.edwmigration.dumper.application.dumper;
1818

19-
import static com.google.edwmigration.dumper.application.dumper.DurationFormatter.formatApproximateDuration;
20-
import static java.lang.Math.max;
21-
2219
import com.google.common.base.Preconditions;
23-
import com.google.common.base.Stopwatch;
2420
import com.google.edwmigration.dumper.application.dumper.handle.Handle;
2521
import com.google.edwmigration.dumper.application.dumper.io.OutputHandle;
2622
import com.google.edwmigration.dumper.application.dumper.io.OutputHandle.WriteMode;
@@ -34,13 +30,8 @@
3430
import java.io.IOException;
3531
import java.nio.charset.StandardCharsets;
3632
import java.sql.SQLException;
37-
import java.time.Duration;
38-
import java.util.ArrayDeque;
3933
import java.util.Arrays;
40-
import java.util.Collections;
41-
import java.util.Deque;
4234
import java.util.List;
43-
import java.util.concurrent.atomic.AtomicInteger;
4435
import javax.annotation.CheckForNull;
4536
import javax.annotation.Nonnull;
4637
import org.slf4j.Logger;
@@ -52,16 +43,11 @@ public class TasksRunner implements TaskRunContextOps {
5243
private static final Logger logger = LoggerFactory.getLogger(TasksRunner.class);
5344
public static final Logger PROGRESS_LOG = LoggerFactory.getLogger("progress-logger");
5445

55-
private AtomicInteger numberOfCompletedTasks;
56-
private final int totalNumberOfTasks;
57-
private final Stopwatch stopwatch;
58-
5946
private final TaskRunContext context;
6047
private final TaskSetState.Impl state;
6148
private final List<Task<?>> tasks;
6249

63-
// For tracking last 10 finished task durations
64-
private final Deque<Duration> lastTaskDurations = new ArrayDeque<>(10);
50+
private final TaskLoggingUtil taskLoggingUtil;
6551

6652
public TasksRunner(
6753
OutputHandleFactory sinkFactory,
@@ -73,9 +59,7 @@ public TasksRunner(
7359
context = createContext(sinkFactory, handle, threadPoolSize, arguments);
7460
this.state = state;
7561
this.tasks = tasks;
76-
totalNumberOfTasks = countTasks(tasks);
77-
stopwatch = Stopwatch.createStarted();
78-
numberOfCompletedTasks = new AtomicInteger();
62+
this.taskLoggingUtil = new TaskLoggingUtil(countTasks(tasks));
7963
}
8064

8165
private TaskRunContext createContext(
@@ -107,53 +91,14 @@ public void run() throws MetadataDumperUsageException {
10791
private <T> T handleTask(Task<T> task) throws MetadataDumperUsageException {
10892
T t = runTask(task);
10993
if (!(task instanceof TaskGroup)) {
110-
numberOfCompletedTasks.getAndIncrement();
111-
112-
Duration taskDuration = stopwatch.elapsed();
113-
if (lastTaskDurations.size() == 10) {
114-
lastTaskDurations.removeFirst();
115-
}
116-
lastTaskDurations.addLast(taskDuration);
94+
taskLoggingUtil.recordTaskCompletion();
11795
}
11896
logProgress();
11997
return t;
12098
}
12199

122-
private Duration getAverageTaskDurationFromAllTasks() {
123-
return stopwatch.elapsed().dividedBy(max(1, numberOfCompletedTasks.get()));
124-
}
125-
126-
private Duration getAverageTaskDurationFromLatestTasks() {
127-
if (lastTaskDurations.isEmpty()) {
128-
return Duration.ZERO;
129-
}
130-
Duration total = lastTaskDurations.getLast().minus(lastTaskDurations.getFirst());
131-
132-
return total.dividedBy(lastTaskDurations.size());
133-
}
134-
135-
private Duration getTaskDuration() {
136-
return Collections.max(
137-
Arrays.asList(
138-
getAverageTaskDurationFromAllTasks(), getAverageTaskDurationFromLatestTasks()));
139-
}
140-
141100
private void logProgress() {
142-
int numberOfCompletedTasks = this.numberOfCompletedTasks.get();
143-
144-
Duration averageTimePerTask = getTaskDuration();
145-
146-
int percentFinished = numberOfCompletedTasks * 100 / totalNumberOfTasks;
147-
String progressMessage = percentFinished + "% Completed";
148-
149-
int remainingTasks = totalNumberOfTasks - numberOfCompletedTasks;
150-
Duration remainingTime = averageTimePerTask.multipliedBy(remainingTasks);
151-
152-
if (numberOfCompletedTasks > 10 && remainingTasks > 0) {
153-
progressMessage += ". ETA: " + formatApproximateDuration(remainingTime);
154-
}
155-
156-
PROGRESS_LOG.info(progressMessage);
101+
PROGRESS_LOG.info(taskLoggingUtil.getProgressLog());
157102
}
158103

159104
private static final String ACCESS_CONTROL_EXCEPTION_MSG_SUFFIX =

‎dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/DurationFormatterTest.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,10 @@ public class DurationFormatterTest {
3636
Pair.of(Duration.ZERO, "less than 15 minutes"),
3737
Pair.of(Duration.ofMillis(1), "less than 15 minutes"),
3838
Pair.of(Duration.ofSeconds(1), "less than 15 minutes"),
39-
Pair.of(Duration.ofSeconds(59), "less than 15 minutes"),
40-
Pair.of(Duration.ofMinutes(1), "less than 15 minutes"),
41-
Pair.of(Duration.ofMinutes(1).plusSeconds(1), "less than 15 minutes"),
42-
Pair.of(Duration.ofMinutes(1).plusSeconds(59), "less than 15 minutes"),
4339
Pair.of(Duration.ofMinutes(2), "less than 15 minutes"),
4440
Pair.of(Duration.ofMinutes(2).plusSeconds(1), "less than 15 minutes"),
45-
Pair.of(Duration.ofMinutes(2).plusSeconds(59), "less than 15 minutes"),
46-
Pair.of(Duration.ofMinutes(11), "less than 15 minutes"),
41+
Pair.of(Duration.ofMinutes(14).plusSeconds(59), "less than 15 minutes"),
42+
Pair.of(Duration.ofMinutes(15), "~15 minutes"),
4743
Pair.of(Duration.ofMinutes(21), "~21 minutes"),
4844
Pair.of(Duration.ofMinutes(34), "~34 minutes"),
4945
Pair.of(Duration.ofMinutes(59).plusSeconds(59).plusMillis(999), "~59 minutes"),

‎dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/TaskLoggingUtilTest.java‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* Copyright 2022-2025 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.edwmigration.dumper.application.dumper;
18+
19+
import static org.junit.Assert.assertEquals;
20+
import static org.mockito.Mockito.*;
21+
22+
import com.google.common.base.Stopwatch;
23+
import java.time.Duration;
24+
import org.junit.Before;
25+
import org.junit.Test;
26+
import org.mockito.Mockito;
27+
28+
public class TaskLoggingUtilTest {
29+
30+
private TaskLoggingUtil util;
31+
32+
@Before
33+
public void setUp() {
34+
util = new TaskLoggingUtil(5);
35+
}
36+
37+
@Test
38+
public void testRecordTaskCompletion_IncrementsCompletedTasksAndStoresDurations() {
39+
int initialCompleted = util.numberOfCompletedTasks.get();
40+
util.recordTaskCompletion();
41+
assertEquals(initialCompleted + 1, util.numberOfCompletedTasks.get());
42+
// After first completion, lastTaskDurations should have 1 element
43+
assertEquals(1, util.lastTaskDurations.size());
44+
util.recordTaskCompletion();
45+
assertEquals(2, util.lastTaskDurations.size());
46+
}
47+
48+
@Test
49+
public void testRecordTaskCompletion_QueueDoesNotExceedMaxSize() {
50+
for (int i = 0; i < 15; i++) {
51+
util.recordTaskCompletion();
52+
}
53+
// Should not exceed TASK_QUEUE_SIZE (10)
54+
assertEquals(10, util.lastTaskDurations.size());
55+
}
56+
57+
@Test
58+
public void testGetProgressLog_ETA_NotShownWhenCompletedTasksNotEnough() {
59+
// Complete less than or equal to TASK_QUEUE_SIZE tasks
60+
for (int i = 0; i < 10; i++) {
61+
util.recordTaskCompletion();
62+
}
63+
Stopwatch spyStopwatch = Mockito.spy(Stopwatch.createUnstarted());
64+
spyStopwatch.start();
65+
spyStopwatch.reset();
66+
when(spyStopwatch.elapsed()).thenReturn(Duration.ofMinutes(16));
67+
try {
68+
java.lang.reflect.Field field = TaskLoggingUtil.class.getDeclaredField("stopwatch");
69+
field.setAccessible(true);
70+
field.set(util, spyStopwatch);
71+
} catch (Exception e) {
72+
throw new RuntimeException(e);
73+
}
74+
String log = util.getProgressLog();
75+
// ETA should not be shown since completedTasks <= TASK_QUEUE_SIZE
76+
assertEquals(false, log.contains("ETA"));
77+
}
78+
79+
@Test
80+
public void testGetProgressLog_RemainingDurationFromAllTasksWhenLatestTasksEmpty() {
81+
// Only one task completed, so lastTaskDurations has one element
82+
util.recordTaskCompletion();
83+
Stopwatch spyStopwatch = Mockito.spy(Stopwatch.createUnstarted());
84+
spyStopwatch.start();
85+
spyStopwatch.reset();
86+
when(spyStopwatch.elapsed()).thenReturn(Duration.ofMinutes(15));
87+
try {
88+
java.lang.reflect.Field field = TaskLoggingUtil.class.getDeclaredField("stopwatch");
89+
field.setAccessible(true);
90+
field.set(util, spyStopwatch);
91+
} catch (Exception e) {
92+
throw new RuntimeException(e);
93+
}
94+
String log = util.getProgressLog();
95+
// Should not show ETA, and should use all-tasks average
96+
assertEquals(false, log.contains("ETA"));
97+
assertEquals(true, log.contains("% Completed"));
98+
}
99+
100+
@Test
101+
public void testRecordTaskCompletion_HandlesMultipleWraps() {
102+
// Fill up the queue, then add more to ensure oldest are removed
103+
for (int i = 0; i < 25; i++) {
104+
util.recordTaskCompletion();
105+
}
106+
assertEquals(10, util.lastTaskDurations.size());
107+
}
108+
109+
@Test
110+
public void testGetProgressLog_ZeroTasks() {
111+
// No tasks completed
112+
String log = util.getProgressLog();
113+
assertEquals(true, log.contains("0% Completed"));
114+
assertEquals(false, log.contains("ETA"));
115+
}
116+
117+
@Test
118+
public void testGetProgressLog_UsesAllTasksAverage() {
119+
// Simulate only 1 completion, so latestTasks is empty or not enough
120+
util.recordTaskCompletion();
121+
String log = util.getProgressLog();
122+
// Should not contain ETA since completedTasks <= TASK_QUEUE_SIZE
123+
assertEquals(false, log.contains("ETA"));
124+
// Should show percent completed
125+
assertEquals(true, log.contains("% Completed"));
126+
}
127+
128+
@Test
129+
public void testGetProgressLog_LessThanTaskQueueSize() {
130+
// Complete less than TASK_QUEUE_SIZE tasks
131+
for (int i = 0; i < 5; i++) {
132+
util.recordTaskCompletion();
133+
}
134+
String log = util.getProgressLog();
135+
// Should not contain ETA
136+
assertEquals(false, log.contains("ETA"));
137+
// Should show percent completed
138+
assertEquals(true, log.contains("% Completed"));
139+
}
140+
}

0 commit comments

Comments
 (0)