Skip to content

Commit 657cbd4

Browse files
snazysnazy
committed
NoSQL persistence
1 parent c8806df commit 657cbd4

31 files changed

+1364
-6
lines changed

‎LICENSE‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,9 @@ This product includes code from Project Nessie.
338338
* helm/polaris/templates/servicemonitor.yaml
339339
* helm/polaris/templates/storage.yaml
340340

341+
Code underneath the components/persistence directory, especially pluggable object types, index related, cache,
342+
atomic commit logic and fundamental persistence implementations.
343+
341344
Copyright: Copyright 2015-2025 Dremio Corporation
342345
Home page: https://projectnessie.org/
343346
License: https://www.apache.org/licenses/LICENSE-2.0

‎runtime/admin/build.gradle.kts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,28 @@ dependencies {
3232

3333
compileOnly("com.fasterxml.jackson.core:jackson-annotations")
3434

35+
implementation(project(":polaris-persistence-nosql-api"))
36+
implementation(project(":polaris-persistence-nosql-maintenance-api"))
37+
runtimeOnly(project(":polaris-persistence-nosql-metastore"))
38+
runtimeOnly(project(":polaris-persistence-nosql-cdi-quarkus"))
39+
runtimeOnly(project(":polaris-persistence-nosql-maintenance-impl"))
40+
runtimeOnly(project(":polaris-persistence-nosql-metastore-maintenance"))
41+
runtimeOnly(project(":polaris-persistence-nosql-authz-store-nosql"))
42+
3543
runtimeOnly(project(":polaris-relational-jdbc"))
3644
runtimeOnly("org.postgresql:postgresql")
3745

3846
implementation("io.quarkus:quarkus-jdbc-postgresql")
47+
3948
implementation(enforcedPlatform(libs.quarkus.bom))
4049
implementation("io.quarkus:quarkus-picocli")
4150
implementation("io.quarkus:quarkus-container-image-docker")
4251

4352
implementation(project(":polaris-runtime-common"))
4453

54+
compileOnly("com.fasterxml.jackson.core:jackson-annotations")
55+
compileOnly("com.fasterxml.jackson.core:jackson-databind")
56+
4557
testImplementation(project(":polaris-runtime-test-common"))
4658
testFixturesApi(project(":polaris-core"))
4759

@@ -51,7 +63,8 @@ dependencies {
5163
testFixturesApi(project(":polaris-container-spec-helper"))
5264
testFixturesApi(platform(libs.testcontainers.bom))
5365
testFixturesApi("org.testcontainers:testcontainers")
54-
testFixturesApi("org.testcontainers:postgresql")
66+
testFixturesApi("org.testcontainers:testcontainers-postgresql")
67+
testFixturesImplementation(testFixtures(project(":polaris-persistence-nosql-mongodb")))
5568

5669
testRuntimeOnly("org.postgresql:postgresql")
5770
}

‎runtime/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import io.quarkus.picocli.runtime.annotations.TopCommand;
1919
import java.io.PrintWriter;
20+
import org.apache.polaris.admintool.maintenance.NoSqlMaintenanceCommand;
2021
import org.apache.polaris.version.PolarisVersionProvider;
2122
import picocli.CommandLine.Command;
2223
import picocli.CommandLine.HelpCommand;
@@ -31,8 +32,9 @@
3132
HelpCommand.class,
3233
BootstrapCommand.class,
3334
PurgeCommand.class,
35+
NoSqlMaintenanceCommand.class,
3436
})
35-
public class PolarisAdminTool extends BaseMetaStoreCommand {
37+
public class PolarisAdminTool extends BaseCommand {
3638

3739
@Override
3840
public Integer call() {

‎runtime/admin/src/main/java/org/apache/polaris/admintool/maintenance/BaseMaintenanceCommand.java‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.admintool.maintenance;
20+
21+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_COUNT_FROM_LAST_RUN_MULTIPLIER;
22+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_CREATED_AT_GRACE_TIME;
23+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_DELETE_BATCH_SIZE;
24+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_EXPECTED_OBJ_COUNT;
25+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_EXPECTED_REFERENCE_COUNT;
26+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_INITIALIZED_FPP;
27+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_MAX_ACCEPTABLE_FPP;
28+
import static org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig.DEFAULT_RETAINED_RUNS;
29+
30+
import jakarta.inject.Inject;
31+
import java.io.PrintWriter;
32+
import java.time.Instant;
33+
import org.apache.polaris.admintool.BaseCommand;
34+
import org.apache.polaris.persistence.nosql.api.backend.Backend;
35+
import org.apache.polaris.persistence.nosql.api.obj.ObjTypes;
36+
import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceConfig;
37+
import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceRunInformation;
38+
import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceRunInformation.MaintenanceStats;
39+
import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceRunSpec;
40+
import org.apache.polaris.persistence.nosql.maintenance.api.MaintenanceService;
41+
42+
@SuppressWarnings("CdiInjectionPointsInspection")
43+
public abstract class BaseMaintenanceCommand extends BaseCommand {
44+
@Inject protected MaintenanceService maintenanceService;
45+
@Inject protected Backend backend;
46+
@Inject protected MaintenanceConfig maintenanceConfig;
47+
48+
protected void checkInMemory() {
49+
if ("InMemory".equals(backend.type())) {
50+
var err = spec.commandLine().getErr();
51+
52+
err.println();
53+
err.println("Running persistence-maintenance against InMemory is useless...");
54+
err.println();
55+
}
56+
}
57+
58+
protected MaintenanceRunSpec printRealmStates() {
59+
var out = spec.commandLine().getOut();
60+
61+
var runSpec = maintenanceService.buildMaintenanceRunSpec();
62+
out.println();
63+
out.printf("Process system realm: %s%n", runSpec.includeSystemRealm());
64+
out.println("Realms to process:");
65+
var realms = runSpec.realmsToProcess();
66+
if (realms.isEmpty()) {
67+
out.println("(none)");
68+
}
69+
for (var realm : realms) {
70+
out.printf(" %s%n", realm);
71+
}
72+
73+
out.println("Realms to purge:");
74+
realms = runSpec.realmsToPurge();
75+
if (realms.isEmpty()) {
76+
out.println("(none)");
77+
}
78+
for (var realm : realms) {
79+
out.printf(" %s%n", realm);
80+
}
81+
82+
return runSpec;
83+
}
84+
85+
protected void printMaintenanceConfig() {
86+
var out = spec.commandLine().getOut();
87+
88+
out.println();
89+
out.println("Maintenance configuration:");
90+
out.printf(
91+
" created-at grace time: %s%n",
92+
maintenanceConfig.createdAtGraceTime().orElse(DEFAULT_CREATED_AT_GRACE_TIME));
93+
out.printf(
94+
" delete batch size: %s%n",
95+
maintenanceConfig.deleteBatchSize().orElse(DEFAULT_DELETE_BATCH_SIZE));
96+
out.printf(
97+
" retained runs: %s%n",
98+
maintenanceConfig.retainedRuns().orElse(DEFAULT_RETAINED_RUNS));
99+
100+
out.printf(
101+
" expected object count: %d%n",
102+
maintenanceConfig.expectedObjCount().orElse(DEFAULT_EXPECTED_OBJ_COUNT));
103+
out.printf(
104+
" expected reference count: %d%n",
105+
maintenanceConfig.expectedReferenceCount().orElse(DEFAULT_EXPECTED_REFERENCE_COUNT));
106+
out.printf(
107+
" last-run multiplier: %f%n",
108+
maintenanceConfig
109+
.countFromLastRunMultiplier()
110+
.orElse(DEFAULT_COUNT_FROM_LAST_RUN_MULTIPLIER));
111+
out.printf(
112+
" initialized FPP: %f%n",
113+
maintenanceConfig.filterInitializedFpp().orElse(DEFAULT_INITIALIZED_FPP));
114+
out.printf(
115+
" expected FPP: %f%n",
116+
maintenanceConfig.maxAcceptableFilterFpp().orElse(DEFAULT_MAX_ACCEPTABLE_FPP));
117+
118+
out.printf(
119+
" reference scan rate limit / sec: %s%n",
120+
maintenanceConfig.referenceScanRateLimitPerSecond().stream()
121+
.mapToObj(Integer::toString)
122+
.findFirst()
123+
.orElse("(unlimited)"));
124+
out.printf(
125+
" object scan rate limit / sec: %s%n",
126+
maintenanceConfig.objectScanRateLimitPerSecond().stream()
127+
.mapToObj(Integer::toString)
128+
.findFirst()
129+
.orElse("(unlimited)"));
130+
}
131+
132+
protected void printRunInformation(MaintenanceRunInformation info, boolean expert) {
133+
var out = spec.commandLine().getOut();
134+
out.println();
135+
out.println(
136+
"==================================================================================");
137+
out.println();
138+
out.printf("Run started: %s%n", info.started());
139+
out.printf(
140+
" status: %s%n",
141+
info.statusMessage().orElse("(no exceptional information, all good so far)"));
142+
out.printf(" finished: %s%n", info.finished().map(Instant::toString).orElse("(running)"));
143+
out.printf(" details: %s%n", info.detailedInformation().orElse("-"));
144+
145+
out.println();
146+
out.println("Realms:");
147+
out.printf(" purged: %d%n", info.purgedRealms().orElse(0));
148+
149+
out.println();
150+
out.println("References:");
151+
if (expert) {
152+
// This is the number of calls to RetainedCollector.retainReference(), which is usually higher
153+
// than the actual number of distinct reference names.
154+
out.printf(" identified calls: %d%n", info.identifiedReferences().orElse(0));
155+
}
156+
info.referenceStats().ifPresent(stats -> printStats(out, " ", stats));
157+
info.perRealmReferenceStats()
158+
.forEach(
159+
(realm, stats) -> {
160+
out.printf(" Realm: %s%n", realm);
161+
printStats(out, " ", stats);
162+
});
163+
164+
out.println();
165+
out.println("Objects:");
166+
if (expert) {
167+
// This is the number of calls to RetainedCollector.retainObj(), which is usually much higher
168+
// than the actual number of distinct object references.
169+
out.printf(" identified calls: %d%n", info.identifiedObjs().orElse(0));
170+
}
171+
info.objStats().ifPresent(stats -> printStats(out, " ", stats));
172+
info.perRealmPerObjTypeStats()
173+
.forEach(
174+
(realm, perTypeStats) -> {
175+
out.printf(" Realm: %s%n", realm);
176+
perTypeStats.forEach(
177+
(type, stats) -> {
178+
out.printf(" Type: %s (%s)%n", type, ObjTypes.objTypeById(type).name());
179+
printStats(out, " ", stats);
180+
});
181+
});
182+
}
183+
184+
private void printStats(PrintWriter out, String indent, MaintenanceStats stats) {
185+
out.printf("%s scanned: %d%n", indent, stats.scanned().orElse(0L));
186+
out.printf("%s retained: %d%n", indent, stats.retained().orElse(0L));
187+
out.printf("%s too new: %d%n", indent, stats.newer().orElse(0L));
188+
out.printf("%s purged: %d%n", indent, stats.purged().orElse(0L));
189+
}
190+
}

‎runtime/admin/src/main/java/org/apache/polaris/admintool/maintenance/NoSqlMaintenanceCommand.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.admintool.maintenance;
20+
21+
import picocli.CommandLine;
22+
23+
@CommandLine.Command(
24+
name = "maintenance",
25+
subcommands = {
26+
NoSqlMaintenanceLogCommand.class,
27+
NoSqlMaintenanceRunCommand.class,
28+
},
29+
mixinStandardHelpOptions = true,
30+
description = "Polaris persistence maintenance.")
31+
public class NoSqlMaintenanceCommand extends BaseMaintenanceCommand {
32+
33+
@Override
34+
public Integer call() {
35+
var out = spec.commandLine().getOut();
36+
37+
out.println("Polaris NoSql persistence maintenance has multiple subcommands,");
38+
out.println("use the 'help maintenance' command.");
39+
out.println();
40+
41+
checkInMemory();
42+
43+
out.println();
44+
out.println("Information: selected NoSql persistence backend: " + backend.type());
45+
46+
printMaintenanceConfig();
47+
48+
printRealmStates();
49+
50+
return 0;
51+
}
52+
}

‎runtime/admin/src/main/java/org/apache/polaris/admintool/maintenance/NoSqlMaintenanceLogCommand.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.admintool.maintenance;
20+
21+
import picocli.CommandLine;
22+
23+
@CommandLine.Command(
24+
name = "log",
25+
mixinStandardHelpOptions = true,
26+
description = "Show Polaris persistence maintenance log.")
27+
public class NoSqlMaintenanceLogCommand extends BaseMaintenanceCommand {
28+
29+
@CommandLine.Option(
30+
names = {"--show-expert"},
31+
description = "Show expert values, which only reflect internal operations.")
32+
boolean showExpert;
33+
34+
@Override
35+
public Integer call() {
36+
checkInMemory();
37+
38+
var infos = maintenanceService.maintenanceRunLog();
39+
var out = spec.commandLine().getOut();
40+
41+
out.println("Recorded Polaris NoSql persistence maintenance runs:");
42+
if (!infos.isEmpty()) {
43+
infos.forEach(i -> printRunInformation(i, showExpert));
44+
} else {
45+
out.println("(none)");
46+
}
47+
48+
return 0;
49+
}
50+
}

0 commit comments

Comments
 (0)