Skip to content

Commit 25e06f2

Browse files
authored
[core] Add more commit info logs (apache#5657)
1 parent 94bcf65 commit 25e06f2

File tree

2 files changed

+48
-12
lines changed

2 files changed

+48
-12
lines changed

paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,10 @@ public void commit(
269269
ManifestCommittable committable,
270270
Map<String, String> properties,
271271
boolean checkAppendFiles) {
272+
LOG.info(
273+
"Ready to commit to table {}, number of commit messages: {}",
274+
tableName,
275+
committable.fileCommittables().size());
272276
if (LOG.isDebugEnabled()) {
273277
LOG.debug("Ready to commit\n{}", committable.toString());
274278
}
@@ -368,6 +372,7 @@ public void commit(
368372
}
369373
} finally {
370374
long commitDuration = (System.nanoTime() - started) / 1_000_000;
375+
LOG.info("Finished commit to table {}, duration {} ms", tableName, commitDuration);
371376
if (this.commitMetrics != null) {
372377
reportCommit(
373378
appendTableFiles,
@@ -406,6 +411,10 @@ public void overwrite(
406411
Map<String, String> partition,
407412
ManifestCommittable committable,
408413
Map<String, String> properties) {
414+
LOG.info(
415+
"Ready to overwrite to table {}, number of commit messages: {}",
416+
tableName,
417+
committable.fileCommittables().size());
409418
if (LOG.isDebugEnabled()) {
410419
LOG.debug(
411420
"Ready to overwrite partition {}\nManifestCommittable: {}\nProperties: {}",
@@ -513,6 +522,7 @@ public void overwrite(
513522
}
514523
} finally {
515524
long commitDuration = (System.nanoTime() - started) / 1_000_000;
525+
LOG.info("Finished overwrite to table {}, duration {} ms", tableName, commitDuration);
516526
if (this.commitMetrics != null) {
517527
reportCommit(
518528
appendTableFiles,
@@ -701,6 +711,28 @@ private void collectChanges(
701711
}
702712
});
703713
}
714+
if (!commitMessages.isEmpty()) {
715+
List<String> msg = new ArrayList<>();
716+
if (appendTableFiles.size() > 0) {
717+
msg.add(appendTableFiles.size() + " append table files");
718+
}
719+
if (appendChangelog.size() > 0) {
720+
msg.add(appendChangelog.size() + " append Changelogs");
721+
}
722+
if (compactTableFiles.size() > 0) {
723+
msg.add(compactTableFiles.size() + " compact table files");
724+
}
725+
if (compactChangelog.size() > 0) {
726+
msg.add(compactChangelog.size() + " compact Changelogs");
727+
}
728+
if (appendHashIndexFiles.size() > 0) {
729+
msg.add(appendHashIndexFiles.size() + " append hash index files");
730+
}
731+
if (compactDvIndexFiles.size() > 0) {
732+
msg.add(compactDvIndexFiles.size() + " compact dv index files");
733+
}
734+
LOG.info("Finished collecting changes, including: {}", String.join(", ", msg));
735+
}
704736
}
705737

706738
private ManifestEntry makeEntry(FileKind kind, CommitMessage commitMessage, DataFileMeta file) {
@@ -1013,13 +1045,14 @@ CommitResult tryCommitOnce(
10131045
}
10141046

10151047
if (commitSnapshotImpl(newSnapshot, deltaStatistics)) {
1016-
if (LOG.isDebugEnabled()) {
1017-
LOG.debug(
1018-
String.format(
1019-
"Successfully commit snapshot #%d by user %s "
1020-
+ "with identifier %s and kind %s.",
1021-
newSnapshotId, commitUser, identifier, commitKind.name()));
1022-
}
1048+
LOG.info(
1049+
"Successfully commit snapshot {} to table {} by user {} "
1050+
+ "with identifier {} and kind {}.",
1051+
newSnapshotId,
1052+
tableName,
1053+
commitUser,
1054+
identifier,
1055+
commitKind.name());
10231056
commitCallbacks.forEach(callback -> callback.call(deltaFiles, newSnapshot));
10241057
return new SuccessResult();
10251058
}

paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@ public int expire() {
135135

136136
@VisibleForTesting
137137
public int expireUntil(long earliestId, long endExclusiveId) {
138+
long startTime = System.currentTimeMillis();
139+
138140
if (endExclusiveId <= earliestId) {
139141
// No expire happens:
140142
// write the hint file in order to see the earliest snapshot directly next time
@@ -158,11 +160,6 @@ public int expireUntil(long earliestId, long endExclusiveId) {
158160
}
159161
}
160162

161-
if (LOG.isDebugEnabled()) {
162-
LOG.debug(
163-
"Snapshot expire range is [" + beginInclusiveId + ", " + endExclusiveId + ")");
164-
}
165-
166163
List<Snapshot> taggedSnapshots = tagManager.taggedSnapshots();
167164

168165
// delete merge tree files
@@ -269,6 +266,12 @@ public int expireUntil(long earliestId, long endExclusiveId) {
269266
}
270267

271268
writeEarliestHint(endExclusiveId);
269+
long duration = System.currentTimeMillis() - startTime;
270+
LOG.info(
271+
"Finished expire snapshots, duration {} ms, range is [{}, {})",
272+
duration,
273+
beginInclusiveId,
274+
endExclusiveId);
272275
return (int) (endExclusiveId - beginInclusiveId);
273276
}
274277

0 commit comments

Comments
 (0)