Skip to content

Commit abee3ea

Browse files
predator4annpredator4ann
authored
Merge branch 'master' into feature/add_lineage_info_to_explain_typeio
2 parents 94dcd33 + 4c2dda8 commit abee3ea

22 files changed

+171
-119
lines changed

Diff for: ‎.mvn/extensions.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@
33
<extension>
44
<groupId>io.takari.maven</groupId>
55
<artifactId>takari-smart-builder</artifactId>
6-
<version>0.6.6</version>
6+
<version>1.0.2</version>
77
</extension>
88
</extensions>

Diff for: ‎core/trino-main/src/main/java/io/trino/execution/DistributionSnapshot.java

+1-59
Original file line numberDiff line numberDiff line change
@@ -16,81 +16,23 @@
1616
import com.fasterxml.jackson.annotation.JsonTypeInfo;
1717
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
1818
import com.google.common.base.MoreObjects.ToStringHelper;
19-
import io.trino.operator.OperatorStats;
2019
import io.trino.spi.metrics.Distribution;
2120
import io.trino.spi.metrics.Metric;
2221
import io.trino.spi.metrics.Metrics;
2322

24-
import java.util.List;
2523
import java.util.Locale;
2624
import java.util.Map;
2725

2826
import static com.google.common.base.MoreObjects.toStringHelper;
29-
import static com.google.common.collect.ImmutableList.toImmutableList;
3027
import static com.google.common.collect.ImmutableMap.toImmutableMap;
3128
import static java.lang.String.format;
32-
import static java.util.Objects.requireNonNull;
3329

3430
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE) // Do not add @class property
3531
@JsonSerialize
3632
public record DistributionSnapshot(long total, double min, double max, double p01, double p05, double p10, double p25, double p50, double p75, double p90, double p95, double p99)
3733
implements Metric<DistributionSnapshot>
3834
{
39-
public static List<OperatorStats> pruneOperatorStats(List<OperatorStats> operatorStats)
40-
{
41-
requireNonNull(operatorStats, "operatorStats is null");
42-
return operatorStats.stream()
43-
.map(DistributionSnapshot::pruneOperatorStats)
44-
.collect(toImmutableList());
45-
}
46-
47-
public static OperatorStats pruneOperatorStats(OperatorStats operatorStats)
48-
{
49-
requireNonNull(operatorStats, "operatorStats is null");
50-
return new OperatorStats(
51-
operatorStats.getStageId(),
52-
operatorStats.getPipelineId(),
53-
operatorStats.getOperatorId(),
54-
operatorStats.getPlanNodeId(),
55-
operatorStats.getOperatorType(),
56-
operatorStats.getTotalDrivers(),
57-
operatorStats.getAddInputCalls(),
58-
operatorStats.getAddInputWall(),
59-
operatorStats.getAddInputCpu(),
60-
operatorStats.getPhysicalInputDataSize(),
61-
operatorStats.getPhysicalInputPositions(),
62-
operatorStats.getPhysicalInputReadTime(),
63-
operatorStats.getInternalNetworkInputDataSize(),
64-
operatorStats.getInternalNetworkInputPositions(),
65-
operatorStats.getRawInputDataSize(),
66-
operatorStats.getInputDataSize(),
67-
operatorStats.getInputPositions(),
68-
operatorStats.getSumSquaredInputPositions(),
69-
operatorStats.getGetOutputCalls(),
70-
operatorStats.getGetOutputWall(),
71-
operatorStats.getGetOutputCpu(),
72-
operatorStats.getOutputDataSize(),
73-
operatorStats.getOutputPositions(),
74-
operatorStats.getDynamicFilterSplitsProcessed(),
75-
pruneMetrics(operatorStats.getMetrics()),
76-
pruneMetrics(operatorStats.getConnectorMetrics()),
77-
pruneMetrics(operatorStats.getPipelineMetrics()),
78-
operatorStats.getPhysicalWrittenDataSize(),
79-
operatorStats.getBlockedWall(),
80-
operatorStats.getFinishCalls(),
81-
operatorStats.getFinishWall(),
82-
operatorStats.getFinishCpu(),
83-
operatorStats.getUserMemoryReservation(),
84-
operatorStats.getRevocableMemoryReservation(),
85-
operatorStats.getPeakUserMemoryReservation(),
86-
operatorStats.getPeakRevocableMemoryReservation(),
87-
operatorStats.getPeakTotalMemoryReservation(),
88-
operatorStats.getSpilledDataSize(),
89-
operatorStats.getBlockedReason(),
90-
operatorStats.getInfo());
91-
}
92-
93-
private static Metrics pruneMetrics(Metrics metrics)
35+
public static Metrics pruneMetrics(Metrics metrics)
9436
{
9537
return new Metrics(metrics.getMetrics().entrySet().stream()
9638
.collect(toImmutableMap(

Diff for: ‎core/trino-main/src/main/java/io/trino/execution/QueryStats.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232
import java.util.Set;
3333

3434
import static com.google.common.base.Preconditions.checkArgument;
35+
import static com.google.common.collect.ImmutableList.toImmutableList;
3536
import static io.airlift.units.DataSize.succinctBytes;
36-
import static io.trino.execution.DistributionSnapshot.pruneOperatorStats;
3737
import static io.trino.server.DynamicFilterService.DynamicFiltersStats;
3838
import static java.util.Objects.requireNonNull;
3939

@@ -336,7 +336,9 @@ public QueryStats(
336336

337337
this.dynamicFiltersStats = requireNonNull(dynamicFiltersStats, "dynamicFiltersStats is null");
338338

339-
this.operatorSummaries = pruneOperatorStats(requireNonNull(operatorSummaries, "operatorSummaries is null"));
339+
requireNonNull(operatorSummaries, "operatorSummaries is null");
340+
this.operatorSummaries = operatorSummaries.stream().map(OperatorStats::pruneDigests).collect(toImmutableList());
341+
340342
this.optimizerRulesSummaries = ImmutableList.copyOf(requireNonNull(optimizerRulesSummaries, "optimizerRulesSummaries is null"));
341343
}
342344

Diff for: ‎core/trino-main/src/main/java/io/trino/execution/StageInfo.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ public StageInfo pruneDigests()
176176
plan,
177177
coordinatorOnly,
178178
types,
179-
stageStats,
179+
stageStats.pruneDigests(),
180180
tasks.stream().map(TaskInfo::pruneDigests).collect(toImmutableList()),
181181
subStages.stream().map(StageInfo::pruneDigests).collect(toImmutableList()),
182182
tables,

Diff for: ‎core/trino-main/src/main/java/io/trino/execution/StageStats.java

+69-2
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,10 @@
3434
import java.util.Set;
3535

3636
import static com.google.common.base.Preconditions.checkArgument;
37+
import static com.google.common.collect.ImmutableList.toImmutableList;
3738
import static io.airlift.units.DataSize.Unit.BYTE;
3839
import static io.airlift.units.DataSize.succinctBytes;
39-
import static io.trino.execution.DistributionSnapshot.pruneOperatorStats;
40+
import static io.trino.execution.DistributionSnapshot.pruneMetrics;
4041
import static io.trino.execution.StageState.RUNNING;
4142
import static java.lang.Math.min;
4243
import static java.util.Objects.requireNonNull;
@@ -285,7 +286,8 @@ public StageStats(
285286

286287
this.gcInfo = requireNonNull(gcInfo, "gcInfo is null");
287288

288-
this.operatorSummaries = pruneOperatorStats(requireNonNull(operatorSummaries, "operatorSummaries is null"));
289+
requireNonNull(operatorSummaries, "operatorSummaries is null");
290+
this.operatorSummaries = operatorSummaries.stream().map(OperatorStats::pruneDigests).collect(toImmutableList());
289291
}
290292

291293
@JsonProperty
@@ -680,6 +682,71 @@ public BasicStageStats toBasicStageStats(StageState stageState)
680682
runningPercentage);
681683
}
682684

685+
public StageStats pruneDigests()
686+
{
687+
return new StageStats(
688+
schedulingComplete,
689+
getSplitDistribution,
690+
totalTasks,
691+
runningTasks,
692+
completedTasks,
693+
failedTasks,
694+
totalDrivers,
695+
queuedDrivers,
696+
runningDrivers,
697+
blockedDrivers,
698+
completedDrivers,
699+
cumulativeUserMemory,
700+
failedCumulativeUserMemory,
701+
userMemoryReservation,
702+
revocableMemoryReservation,
703+
totalMemoryReservation,
704+
peakUserMemoryReservation,
705+
peakRevocableMemoryReservation,
706+
totalScheduledTime,
707+
failedScheduledTime,
708+
totalCpuTime,
709+
failedCpuTime,
710+
totalBlockedTime,
711+
fullyBlocked,
712+
blockedReasons,
713+
physicalInputDataSize,
714+
failedPhysicalInputDataSize,
715+
physicalInputPositions,
716+
failedPhysicalInputPositions,
717+
physicalInputReadTime,
718+
failedPhysicalInputReadTime,
719+
internalNetworkInputDataSize,
720+
failedInternalNetworkInputDataSize,
721+
internalNetworkInputPositions,
722+
failedInternalNetworkInputPositions,
723+
rawInputDataSize,
724+
failedRawInputDataSize,
725+
rawInputPositions,
726+
failedRawInputPositions,
727+
processedInputDataSize,
728+
failedProcessedInputDataSize,
729+
processedInputPositions,
730+
failedProcessedInputPositions,
731+
inputBlockedTime,
732+
failedInputBlockedTime,
733+
bufferedDataSize,
734+
outputBufferUtilization,
735+
outputDataSize,
736+
failedOutputDataSize,
737+
outputPositions,
738+
failedOutputPositions,
739+
pruneMetrics(outputBufferMetrics),
740+
outputBlockedTime,
741+
failedOutputBlockedTime,
742+
physicalWrittenDataSize,
743+
failedPhysicalWrittenDataSize,
744+
gcInfo,
745+
operatorSummaries.stream()
746+
.map(OperatorStats::pruneDigests)
747+
.collect(toImmutableList()));
748+
}
749+
683750
public static StageStats createInitial()
684751
{
685752
DataSize zeroBytes = DataSize.of(0, BYTE);

Diff for: ‎core/trino-main/src/main/java/io/trino/execution/TaskInfo.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public TaskInfo pruneSpoolingOutputStats()
6666

6767
public TaskInfo pruneDigests()
6868
{
69-
return new TaskInfo(taskStatus, lastHeartbeat, outputBuffers, noMoreSplits, stats.pruneDigests(), estimatedMemory, needsPlan);
69+
return new TaskInfo(taskStatus, lastHeartbeat, outputBuffers.pruneDigests(), noMoreSplits, stats.pruneDigests(), estimatedMemory, needsPlan);
7070
}
7171

7272
@Override

Diff for: ‎core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferInfo.java

+18
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.fasterxml.jackson.annotation.JsonCreator;
1717
import com.fasterxml.jackson.annotation.JsonProperty;
1818
import com.google.common.collect.ImmutableList;
19+
import io.trino.execution.DistributionSnapshot;
1920
import io.trino.plugin.base.metrics.TDigestHistogram;
2021
import io.trino.spi.metrics.Metrics;
2122

@@ -192,6 +193,23 @@ public OutputBufferInfo pruneSpoolingOutputStats()
192193
metrics);
193194
}
194195

196+
public OutputBufferInfo pruneDigests()
197+
{
198+
return new OutputBufferInfo(
199+
type,
200+
state,
201+
canAddBuffers,
202+
canAddPages,
203+
totalBufferedBytes,
204+
totalBufferedPages,
205+
totalRowsSent,
206+
totalPagesSent,
207+
pipelinedBufferStates,
208+
Optional.empty(),
209+
spoolingOutputStats,
210+
metrics.map(DistributionSnapshot::pruneMetrics));
211+
}
212+
195213
@Override
196214
public String toString()
197215
{

Diff for: ‎core/trino-main/src/main/java/io/trino/operator/OperatorStats.java

+46
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929

3030
import static com.google.common.base.Preconditions.checkArgument;
3131
import static com.google.common.base.Verify.verify;
32+
import static io.trino.execution.DistributionSnapshot.pruneMetrics;
3233
import static java.lang.Math.max;
3334
import static java.util.Objects.requireNonNull;
3435
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -642,6 +643,51 @@ private static <T> Mergeable<T> mergeInfos(Mergeable<T> base, List<T> others)
642643
return (Mergeable<T>) base.mergeWith(others);
643644
}
644645

646+
public OperatorStats pruneDigests()
647+
{
648+
return new OperatorStats(
649+
stageId,
650+
pipelineId,
651+
operatorId,
652+
planNodeId,
653+
operatorType,
654+
totalDrivers,
655+
addInputCalls,
656+
addInputWall,
657+
addInputCpu,
658+
physicalInputDataSize,
659+
physicalInputPositions,
660+
physicalInputReadTime,
661+
internalNetworkInputDataSize,
662+
internalNetworkInputPositions,
663+
rawInputDataSize,
664+
inputDataSize,
665+
inputPositions,
666+
sumSquaredInputPositions,
667+
getOutputCalls,
668+
getOutputWall,
669+
getOutputCpu,
670+
outputDataSize,
671+
outputPositions,
672+
dynamicFilterSplitsProcessed,
673+
pruneMetrics(metrics),
674+
pruneMetrics(connectorMetrics),
675+
pruneMetrics(pipelineMetrics),
676+
physicalWrittenDataSize,
677+
blockedWall,
678+
finishCalls,
679+
finishWall,
680+
finishCpu,
681+
userMemoryReservation,
682+
revocableMemoryReservation,
683+
peakUserMemoryReservation,
684+
peakRevocableMemoryReservation,
685+
peakTotalMemoryReservation,
686+
spilledDataSize,
687+
blockedReason,
688+
info);
689+
}
690+
645691
public OperatorStats summarize()
646692
{
647693
if (info == null || info.isFinal()) {

Diff for: ‎core/trino-main/src/main/java/io/trino/operator/PipelineStats.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,7 @@ public PipelineStats pruneDigests()
549549
outputBlockedTime,
550550
physicalWrittenDataSize,
551551
operatorSummaries.stream()
552-
.map(io.trino.execution.DistributionSnapshot::pruneOperatorStats)
552+
.map(OperatorStats::pruneDigests)
553553
.collect(toImmutableList()),
554554
drivers);
555555
}

Diff for: ‎lib/trino-cache/pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,12 @@
3737
<artifactId>modernizer-maven-annotations</artifactId>
3838
</dependency>
3939

40+
<dependency>
41+
<groupId>org.jetbrains</groupId>
42+
<artifactId>annotations</artifactId>
43+
<optional>true</optional>
44+
</dependency>
45+
4046
<dependency>
4147
<groupId>org.weakref</groupId>
4248
<artifactId>jmxutils</artifactId>

Diff for: ‎lib/trino-cache/src/main/java/io/trino/cache/ElementTypesAreNonnullByDefault.java

-31
This file was deleted.

Diff for: ‎lib/trino-cache/src/main/java/io/trino/cache/EmptyCache.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.common.collect.ImmutableSet;
2222
import com.google.common.util.concurrent.UncheckedExecutionException;
2323
import jakarta.annotation.Nullable;
24+
import org.jetbrains.annotations.NotNullByDefault;
2425

2526
import java.util.Collection;
2627
import java.util.Map;
@@ -31,7 +32,7 @@
3132

3233
import static java.util.Objects.requireNonNull;
3334

34-
@ElementTypesAreNonnullByDefault
35+
@NotNullByDefault
3536
class EmptyCache<K, V>
3637
extends AbstractLoadingCache<K, V>
3738
{

0 commit comments

Comments
 (0)