Skip to content
This repository was archived by the owner on Jun 20, 2023. It is now read-only.

Commit f60e78b

Browse files
author
Julio Faerman
committed
Put Logs task with re-sorting
1 parent f3665a0 commit f60e78b

4 files changed

Lines changed: 150 additions & 56 deletions

File tree

‎src/main/java/cj/StringUtils.java‎

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,32 +22,7 @@ protected String join(String separator, String... context) {
2222
}
2323

2424

25-
static final PrettyTimeParser parser = new PrettyTimeParser();
2625

27-
static final Pattern NOSEP_REGEX = Pattern.compile( "\\d{14}");
28-
static final DateTimeFormatter NOSEP_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
29-
30-
public static Optional<LocalDateTime> parseLocalDateTime(String line) {
31-
var result = tryParse(line, NOSEP_REGEX, NOSEP_FORMAT);
32-
return result;
33-
}
34-
35-
private static Optional<LocalDateTime> tryParse(String line, Pattern regex, DateTimeFormatter format) {
36-
var matcher = regex.matcher(line);
37-
if (matcher.find()) {
38-
var match = matcher.group(0);
39-
var ldt = LocalDateTime.parse(match, format);
40-
return Optional.of(ldt);
41-
}
42-
return Optional.empty();
43-
}
44-
45-
46-
public static void main(String[] args) {
47-
System.out.println(parseLocalDateTime("log-bundle-20230208181716-error-tls-timeout"));
48-
49-
System.out.println(parseLocalDateTime("log-bundle-d-0-8-1-810-1asdf0-error-tls-timeout"));
50-
}
5126

5227

5328
}

‎src/main/java/cj/TimeUtils.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package cj;
2+
3+
import org.slf4j.Logger;
4+
import org.slf4j.LoggerFactory;
5+
6+
import java.time.LocalDate;
7+
import java.time.LocalDateTime;
8+
import java.time.LocalTime;
9+
import java.time.ZoneOffset;
10+
import java.time.format.DateTimeFormatter;
11+
import java.time.format.DateTimeParseException;
12+
import java.util.Optional;
13+
import java.util.regex.Pattern;
14+
15+
public class TimeUtils {
16+
static final Logger log = LoggerFactory.getLogger(TimeUtils.class);
17+
18+
static final Pattern NOSEP_REGEX = Pattern.compile( "\\d{14}");
19+
static final Pattern TIME0_REGEX = Pattern.compile( "\\d{2}:\\d{2}:\\d{2}\\.\\d{6}");
20+
21+
static final DateTimeFormatter NOSEP_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
22+
static final DateTimeFormatter TIME0_FORMAT = DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS");
23+
24+
public static Optional<LocalDateTime> parseLocalDateTime(String line) {
25+
return parseLocalDateTime(line, LocalDate.now());
26+
}
27+
public static Optional<LocalDateTime> parseLocalDateTime(String line, LocalDate baseDay) {
28+
var result = tryParseDateTime(line, NOSEP_REGEX, NOSEP_FORMAT);
29+
if (result.isEmpty()){
30+
result = tryParseTime(line, baseDay, TIME0_REGEX, TIME0_FORMAT);
31+
}
32+
return result;
33+
}
34+
35+
private static Optional<LocalDateTime> tryParseTime(String line, LocalDate baseDay, Pattern regex, DateTimeFormatter format) {
36+
var matcher = regex.matcher(line);
37+
if (matcher.find()) {
38+
var match = matcher.group(0);
39+
var lt = LocalTime.parse(match, format);
40+
var ldt = LocalDateTime.of(baseDay, lt);
41+
return Optional.of(ldt);
42+
}
43+
return Optional.empty();
44+
}
45+
46+
private static Optional<LocalDateTime> tryParseDateTime(String line, Pattern regex, DateTimeFormatter format) {
47+
var matcher = regex.matcher(line);
48+
if (matcher.find()) {
49+
var match = matcher.group(0);
50+
try{
51+
var ldt = LocalDateTime.parse(match, format);
52+
return Optional.of(ldt);
53+
}catch(DateTimeParseException e){
54+
log.trace("Failed to parse [{}] as LocalDateTime", match);
55+
return Optional.empty();
56+
}
57+
}
58+
return Optional.empty();
59+
}
60+
61+
62+
public static void main(String[] args) {
63+
System.out.println(parseLocalDateTime("log-bundle-20230208181716-error-tls-timeout"));
64+
65+
System.out.println(parseLocalDateTime("log-bundle-d-0-8-1-810-1asdf0-error-tls-timeout"));
66+
}
67+
68+
public static Long toTimestamp(LocalDateTime ldt) {
69+
return ldt.toInstant(ZoneOffset.UTC).toEpochMilli();
70+
}
71+
72+
public static LocalDateTime atStartOfDay(LocalDate baseDay) {
73+
return baseDay.atStartOfDay();
74+
}
75+
}

‎src/main/java/cj/logs/LogsPutTask.java‎

Lines changed: 73 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
package cj.logs;
22

33
import cj.OS;
4+
import cj.TimeUtils;
45
import cj.aws.AWSTask;
56
import cj.fs.TaskFiles;
6-
import cj.hello.HelloConfiguration;
7-
import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;
87
import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient;
98
import software.amazon.awssdk.services.cloudwatchlogs.model.*;
109

1110
import javax.enterprise.context.Dependent;
1211
import javax.inject.Inject;
1312
import javax.inject.Named;
1413
import java.nio.file.Path;
14+
import java.time.LocalDate;
15+
import java.time.LocalDateTime;
16+
import java.util.Comparator;
17+
import java.util.Deque;
18+
import java.util.LinkedList;
19+
import java.util.List;
1520

1621
@Dependent
1722
@Named("logs-put")
@@ -25,10 +30,11 @@ public void apply() {
2530
info("Putting logs");
2631
var username = OS.username();
2732
var dataDir = files.dataDir();
28-
debug("Collecting logs from [{}]", dataDir);
2933
var dataDirName = dataDir.getFileName().toString();
3034
var logGroup = "/%s/%s".formatted(username, dataDirName);
3135
var logs = files.findLogFiles();
36+
debug("Collected [{}] log files from [{}]", logs.size(), dataDirName);
37+
3238
try (var cw = aws().cloudwatchlogs()) {
3339
checkLogGroup(cw, logGroup);
3440
logs.forEach(log -> putLog(cw, logGroup, dataDir, log));
@@ -62,10 +68,13 @@ private boolean logGroupExists(CloudWatchLogsClient cw, String logGroup) {
6268

6369
private void putLog(CloudWatchLogsClient cw, String logGroup, Path dataDir, Path logFile) {
6470
try {
71+
var baseDay = TimeUtils.parseLocalDateTime(logFile.toAbsolutePath().toString())
72+
.orElse(LocalDateTime.now())
73+
.toLocalDate();
6574
var logStream = dataDir.relativize(logFile).toString();
6675
checkLogStream(cw, logGroup, logStream);
67-
debug("Putting log [{}] to group [{}]", logStream, logGroup);
68-
cloudwatchPutLog(cw, logGroup, logStream, logFile);
76+
debug("Putting log [{}] to group [{}] with baseDay [{}]", logStream, logGroup, baseDay);
77+
cloudwatchPutLog(cw, logGroup, logStream, logFile, baseDay);
6978
}catch (Exception e){
7079
error("Failed to put log [{}] to group [{}]", logFile, logGroup);
7180
error(e.getMessage());
@@ -102,8 +111,9 @@ private boolean logStreamExists(CloudWatchLogsClient cw, String logGroup, String
102111
return result;
103112
}
104113

105-
private void cloudwatchPutLog(CloudWatchLogsClient cw, String logGroup, String logStream, Path logFile) {
106-
var events = eventsFromFile(logFile);
114+
private void cloudwatchPutLog(CloudWatchLogsClient cw, String logGroup, String logStream, Path logFile, LocalDate baseDay) {
115+
var lines = linesOf(logFile);
116+
var events = eventsFromFile(logFile, baseDay);
107117
if (events.length > 0){
108118
var request = PutLogEventsRequest.builder()
109119
.logGroupName(logGroup)
@@ -118,35 +128,69 @@ private void cloudwatchPutLog(CloudWatchLogsClient cw, String logGroup, String l
118128

119129
}
120130

121-
//TODO: set correct timestamp for untimed log messages instead of filtering out
122-
private InputLogEvent[] eventsFromFile(Path logFile) {
131+
private List<String> linesOf(Path logFile) {
123132
var lines = files.readLines(logFile);
124-
var events = lines.stream()
125-
.map(line -> eventOf(line, logFile))
126-
.filter(event -> event != null)
127-
.toArray(InputLogEvent[]::new);
128-
return events;
133+
debug("Read [{}] lines from [{}]", lines.size(), logFile);
134+
return lines;
129135
}
130136

131-
private InputLogEvent eventOf(String line, Path logFile) {
132-
var time = timestampOf(line, logFile.getFileName());
133-
if (line.length() > CWLOGS_MAX_LINE_LENGTH){
134-
warn("Log line truncated: [{}]", line);
135-
line = line.substring(0, CWLOGS_MAX_LINE_LENGTH);
137+
//TODO: set correct timestamp for untimed log messages instead of filtering out
138+
private InputLogEvent[] eventsFromFile(Path logFile, LocalDate baseDay) {
139+
var lines = files.readLines(logFile);
140+
var events = new LinkedList<InputLogEvent>();
141+
var time = TimeUtils.toTimestamp(TimeUtils.atStartOfDay(baseDay));
142+
for (String line : lines) {
143+
if (line == null || line.isBlank()) continue;
144+
if (line.length() >= CWLOGS_MAX_LINE_LENGTH)
145+
line = line.substring(0, CWLOGS_MAX_LINE_LENGTH - 1);
146+
var timestamp = timestampOf(line, logFile, baseDay);
147+
if (timestamp != null){
148+
time = timestamp;
149+
} else {
150+
timestamp = time;
151+
}
152+
var event = createEvent(timestamp, line);
153+
events.add(event);
136154
}
137-
if (time != null) {
138-
return InputLogEvent.builder()
139-
//TODO: Set correct timestamp
140-
.timestamp(time)
141-
.message(line)
142-
.build();
155+
var ascendingOrder = checkAscendingTimeOrder(events);
156+
if (! ascendingOrder){
157+
warn("Events are not in ascending order. Resorting...");
158+
warn("File: [{}]", logFile);
159+
warn("Events: [{}]", events.size());
160+
events.sort(Comparator.comparing(InputLogEvent::timestamp));
161+
ascendingOrder = checkAscendingTimeOrder(events);
162+
if (! ascendingOrder){
163+
error("Events are still not in ascending order. Skipping...");
164+
return new InputLogEvent[0];
165+
}
143166
}
144-
return null;
167+
var result = events.toArray(InputLogEvent[]::new);
168+
return result;
169+
}
170+
171+
private boolean checkAscendingTimeOrder(LinkedList<InputLogEvent> events) {
172+
var last = Long.MIN_VALUE;
173+
for (InputLogEvent event : events) {
174+
if (event.timestamp() < last){
175+
return false;
176+
}
177+
last = event.timestamp();
178+
}
179+
return true;
180+
}
181+
182+
183+
private InputLogEvent createEvent(Long time, String line) {
184+
return InputLogEvent.builder()
185+
.timestamp(time)
186+
.message(line)
187+
.build();
145188
}
146189

147-
private Long timestampOf(String line, Path fileName) {
148-
//TODO: Map file names to timestamps
149-
return System.currentTimeMillis();
190+
private Long timestampOf(String line, Path fileName, LocalDate baseDay) {
191+
var ldt = TimeUtils.parseLocalDateTime(line, baseDay);
192+
var timestamp = ldt.map(TimeUtils::toTimestamp).orElse(null);
193+
return timestamp;
150194
}
151195

152196
}

src/test/java/cj/StringUtilsTest.java renamed to src/test/java/cj/TimeUtilsTest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@
88
import static org.junit.jupiter.api.Assertions.*;
99

1010
@QuarkusTest
11-
public class StringUtilsTest {
11+
public class TimeUtilsTest {
1212
@Test
1313
public void parseValidDateTimeNoSeparator(){
1414
var line = "log-bundle-20230208181716-error-tls-timeout";
1515
var expected = LocalDateTime.of(
1616
2023, 2, 8,
1717
18, 17, 16);
18-
var actual = StringUtils.parseLocalDateTime(line);
18+
var actual = TimeUtils.parseLocalDateTime(line);
1919
assertTrue(actual.isPresent());
2020
assertEquals(expected, actual.get());
2121
}

0 commit comments

Comments
 (0)