Skip to content

Commit 1123520

Browse files
JingsongLiJingsongLi
committed
[core] Introduce paimon-api module for light rest api, part 1
1 parent 1a8c173 commit 1123520

File tree

152 files changed

+413
-299
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

152 files changed

+413
-299
lines changed

‎docs/content/program-api/catalog-api.md‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -250,30 +250,30 @@ public class AlterTable {
250250

251251
try {
252252
catalog.createTable(
253-
identifier,
254-
new Schema(
255-
Lists.newArrayList(
256-
new DataField(0, "col1", DataTypes.STRING(), "field1"),
257-
new DataField(1, "col2", DataTypes.STRING(), "field2"),
258-
new DataField(2, "col3", DataTypes.STRING(), "field3"),
259-
new DataField(3, "col4", DataTypes.BIGINT(), "field4"),
260-
new DataField(
261-
4,
262-
"col5",
263-
DataTypes.ROW(
264-
new DataField(
265-
5, "f1", DataTypes.STRING(), "f1"),
266-
new DataField(
267-
6, "f2", DataTypes.STRING(), "f2"),
268-
new DataField(
269-
7, "f3", DataTypes.STRING(), "f3")),
270-
"field5"),
271-
new DataField(8, "col6", DataTypes.STRING(), "field6")),
272-
Lists.newArrayList("col1"), // partition keys
273-
Lists.newArrayList("col1", "col2"), // primary key
274-
options,
275-
"table comment"),
276-
false);
253+
identifier,
254+
new Schema(
255+
Lists.newArrayList(
256+
new DataField(0, "col1", DataTypes.STRING(), "field1"),
257+
new DataField(1, "col2", DataTypes.STRING(), "field2"),
258+
new DataField(2, "col3", DataTypes.STRING(), "field3"),
259+
new DataField(3, "col4", DataTypes.BIGINT(), "field4"),
260+
new DataField(
261+
4,
262+
"col5",
263+
DataTypes.ROW(
264+
new DataField(
265+
5, "f1", DataTypes.STRING(), "f1"),
266+
new DataField(
267+
6, "f2", DataTypes.STRING(), "f2"),
268+
new DataField(
269+
7, "f3", DataTypes.STRING(), "f3")),
270+
"field5"),
271+
new DataField(8, "col6", DataTypes.STRING(), "field6")),
272+
Lists.newArrayList("col1"), // partition keys
273+
Lists.newArrayList("col1", "col2"), // primary key
274+
options,
275+
"table comment"),
276+
false);
277277
} catch (Catalog.TableAlreadyExistException e) {
278278
// do something
279279
} catch (Catalog.DatabaseNotExistException e) {
@@ -287,44 +287,44 @@ public class AlterTable {
287287
// add a column after col1
288288
SchemaChange.Move after = SchemaChange.Move.after("col1_after", "col1");
289289
SchemaChange addColumnAfterField =
290-
SchemaChange.addColumn("col7", DataTypes.STRING(), "", after);
290+
SchemaChange.addColumn("col7", DataTypes.STRING(), "", after);
291291
// rename column
292292
SchemaChange renameColumn = SchemaChange.renameColumn("col3", "col3_new_name");
293293
// drop column
294294
SchemaChange dropColumn = SchemaChange.dropColumn("col6");
295295
// update column comment
296296
SchemaChange updateColumnComment =
297-
SchemaChange.updateColumnComment(new String[] {"col4"}, "col4 field");
297+
SchemaChange.updateColumnComment(new String[]{"col4"}, "col4 field");
298298
// update nested column comment
299299
SchemaChange updateNestedColumnComment =
300-
SchemaChange.updateColumnComment(new String[] {"col5", "f1"}, "col5 f1 field");
300+
SchemaChange.updateColumnComment(new String[]{"col5", "f1"}, "col5 f1 field");
301301
// update column type
302302
SchemaChange updateColumnType = SchemaChange.updateColumnType("col4", DataTypes.DOUBLE());
303303
// update column position, you need to pass in a parameter of type Move
304304
SchemaChange updateColumnPosition =
305-
SchemaChange.updateColumnPosition(SchemaChange.Move.first("col4"));
305+
SchemaChange.updateColumnPosition(SchemaChange.Move.first("col4"));
306306
// update column nullability
307307
SchemaChange updateColumnNullability =
308-
SchemaChange.updateColumnNullability(new String[] {"col4"}, false);
308+
SchemaChange.updateColumnNullability(new String[]{"col4"}, false);
309309
// update nested column nullability
310310
SchemaChange updateNestedColumnNullability =
311-
SchemaChange.updateColumnNullability(new String[] {"col5", "f2"}, false);
311+
SchemaChange.updateColumnNullability(new String[]{"col5", "f2"}, false);
312312

313313
SchemaChange[] schemaChanges =
314-
new SchemaChange[] {
315-
addOption,
316-
removeOption,
317-
addColumn,
318-
addColumnAfterField,
319-
renameColumn,
320-
dropColumn,
321-
updateColumnComment,
322-
updateNestedColumnComment,
323-
updateColumnType,
324-
updateColumnPosition,
325-
updateColumnNullability,
326-
updateNestedColumnNullability
327-
};
314+
new SchemaChange[]{
315+
addOption,
316+
removeOption,
317+
addColumn,
318+
addColumnAfterField,
319+
renameColumn,
320+
dropColumn,
321+
updateColumnComment,
322+
updateNestedColumnComment,
323+
updateColumnType,
324+
updateColumnPosition,
325+
updateColumnNullability,
326+
updateNestedColumnNullability
327+
};
328328
try {
329329
catalog.alterTable(identifier, Arrays.asList(schemaChanges), false);
330330
} catch (Catalog.TableNotExistException e) {

‎docs/content/program-api/flink-api.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ public class WriteCdcToTable {
217217
Identifier identifier = Identifier.create("my_db", "T");
218218
Options catalogOptions = new Options();
219219
catalogOptions.set("warehouse", "/path/to/warehouse");
220-
CatalogLoader catalogLoader =
220+
CatalogLoader catalogLoader =
221221
() -> FlinkCatalogFactory.createPaimonCatalog(catalogOptions);
222222
Table table = catalogLoader.load().getTable(identifier);
223223

‎docs/content/program-api/java-api.md‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -175,16 +175,16 @@ public class ReadTable {
175175
Table table = GetTable.getTable();
176176

177177
PredicateBuilder builder =
178-
new PredicateBuilder(RowType.of(DataTypes.STRING(), DataTypes.INT()));
178+
new PredicateBuilder(RowType.of(DataTypes.STRING(), DataTypes.INT()));
179179
Predicate notNull = builder.isNotNull(0);
180180
Predicate greaterOrEqual = builder.greaterOrEqual(1, 12);
181181

182-
int[] projection = new int[] {0, 1};
182+
int[] projection = new int[]{0, 1};
183183

184184
ReadBuilder readBuilder =
185-
table.newReadBuilder()
186-
.withProjection(projection)
187-
.withFilter(Lists.newArrayList(notNull, greaterOrEqual));
185+
table.newReadBuilder()
186+
.withProjection(projection)
187+
.withFilter(Lists.newArrayList(notNull, greaterOrEqual));
188188

189189
// 2. Plan splits in 'Coordinator' (or named 'Driver')
190190
List<Split> splits = readBuilder.newScan().plan().splits();
@@ -283,16 +283,16 @@ public class StreamReadTable {
283283
Table table = GetTable.getTable();
284284

285285
PredicateBuilder builder =
286-
new PredicateBuilder(RowType.of(DataTypes.STRING(), DataTypes.INT()));
286+
new PredicateBuilder(RowType.of(DataTypes.STRING(), DataTypes.INT()));
287287
Predicate notNull = builder.isNotNull(0);
288288
Predicate greaterOrEqual = builder.greaterOrEqual(1, 12);
289289

290-
int[] projection = new int[] {0, 1};
290+
int[] projection = new int[]{0, 1};
291291

292292
ReadBuilder readBuilder =
293-
table.newReadBuilder()
294-
.withProjection(projection)
295-
.withFilter(Lists.newArrayList(notNull, greaterOrEqual));
293+
table.newReadBuilder()
294+
.withProjection(projection)
295+
.withFilter(Lists.newArrayList(notNull, greaterOrEqual));
296296

297297
// 2. Plan splits in 'Coordinator' (or named 'Driver')
298298
StreamTableScan scan = readBuilder.newStreamScan();

‎paimon-api/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
## Paimon API
2+
3+
This mode is for light API, aims to simplify dependencies as much as possible and avoid
4+
introducing dependencies such as Hadoop, including:
5+
6+
1. Including types, table, view, function and etc.
7+
2. Including http client REST API.

‎paimon-api/pom.xml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<groupId>org.apache.paimon</groupId>
26+
<artifactId>paimon-parent</artifactId>
27+
<version>1.2-SNAPSHOT</version>
28+
</parent>
29+
30+
<artifactId>paimon-api</artifactId>
31+
<name>Paimon : API</name>
32+
33+
<dependencies>
34+
<dependency>
35+
<groupId>org.apache.paimon</groupId>
36+
<artifactId>paimon-shade-jackson-2</artifactId>
37+
<version>${paimon.shade.jackson.version}-${paimon.shade.version}</version>
38+
</dependency>
39+
</dependencies>
40+
41+
<build>
42+
<plugins>
43+
<plugin>
44+
<groupId>org.apache.maven.plugins</groupId>
45+
<artifactId>maven-shade-plugin</artifactId>
46+
<executions>
47+
<execution>
48+
<id>shade-paimon</id>
49+
<phase>package</phase>
50+
<goals>
51+
<goal>shade</goal>
52+
</goals>
53+
<configuration>
54+
<artifactSet>
55+
<includes combine.children="append">
56+
<include>org.apache.paimon:paimon-shade-jackson-2</include>
57+
</includes>
58+
</artifactSet>
59+
<filters>
60+
<!-- Another copy of the Apache license, which we don't need. -->
61+
<filter>
62+
<artifact>*</artifact>
63+
<excludes>
64+
<exclude>META-INF/LICENSE.txt</exclude>
65+
</excludes>
66+
</filter>
67+
</filters>
68+
</configuration>
69+
</execution>
70+
</executions>
71+
</plugin>
72+
</plugins>
73+
</build>
74+
</project>

paimon-common/src/main/java/org/apache/paimon/CoreOptions.java renamed to paimon-api/src/main/java/org/apache/paimon/CoreOptions.java

Lines changed: 10 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.paimon.annotation.VisibleForTesting;
2525
import org.apache.paimon.compression.CompressOptions;
2626
import org.apache.paimon.fileindex.FileIndexOptions;
27-
import org.apache.paimon.format.FileFormat;
2827
import org.apache.paimon.fs.Path;
2928
import org.apache.paimon.lookup.LookupStrategy;
3029
import org.apache.paimon.options.ConfigOption;
@@ -35,7 +34,6 @@
3534
import org.apache.paimon.options.description.DescribedEnum;
3635
import org.apache.paimon.options.description.Description;
3736
import org.apache.paimon.options.description.InlineElement;
38-
import org.apache.paimon.utils.DateTimeUtils;
3937
import org.apache.paimon.utils.MathUtils;
4038
import org.apache.paimon.utils.Pair;
4139
import org.apache.paimon.utils.StringUtils;
@@ -55,7 +53,6 @@
5553
import java.util.Map;
5654
import java.util.Optional;
5755
import java.util.Set;
58-
import java.util.TimeZone;
5956
import java.util.UUID;
6057
import java.util.stream.Collectors;
6158

@@ -1780,16 +1777,12 @@ public String formatType() {
17801777
return normalizeFileFormat(options.get(FILE_FORMAT));
17811778
}
17821779

1783-
public FileFormat fileFormat() {
1784-
return createFileFormat(options, FILE_FORMAT);
1785-
}
1786-
17871780
public String fileFormatString() {
17881781
return normalizeFileFormat(options.get(FILE_FORMAT));
17891782
}
17901783

1791-
public FileFormat manifestFormat() {
1792-
return createFileFormat(options, MANIFEST_FORMAT);
1784+
public String manifestFormatString() {
1785+
return normalizeFileFormat(options.get(MANIFEST_FORMAT));
17931786
}
17941787

17951788
public String manifestCompression() {
@@ -1824,11 +1817,6 @@ public Integer getLocalSampleMagnification() {
18241817
return options.get(SORT_COMPACTION_SAMPLE_MAGNIFICATION);
18251818
}
18261819

1827-
public static FileFormat createFileFormat(Options options, ConfigOption<String> formatOption) {
1828-
String formatIdentifier = normalizeFileFormat(options.get(formatOption));
1829-
return FileFormat.fromIdentifier(formatIdentifier, options);
1830-
}
1831-
18321820
public String objectLocation() {
18331821
checkArgument(type() == TableType.OBJECT_TABLE, "Only object table has object location!");
18341822
return options.get(OBJECT_LOCATION);
@@ -1859,7 +1847,7 @@ public Map<Integer, String> statsModePerLevel() {
18591847
.collect(Collectors.toMap(e -> Integer.valueOf(e.getKey()), Map.Entry::getValue));
18601848
}
18611849

1862-
private static String normalizeFileFormat(String fileFormat) {
1850+
public static String normalizeFileFormat(String fileFormat) {
18631851
return fileFormat.toLowerCase();
18641852
}
18651853

@@ -2296,13 +2284,7 @@ public static StartupMode startupMode(Options options) {
22962284
}
22972285

22982286
public Long scanTimestampMills() {
2299-
String timestampStr = scanTimestamp();
2300-
Long timestampMillis = options.get(SCAN_TIMESTAMP_MILLIS);
2301-
if (timestampMillis == null && timestampStr != null) {
2302-
return DateTimeUtils.parseTimestampData(timestampStr, 3, TimeZone.getDefault())
2303-
.getMillisecond();
2304-
}
2305-
return timestampMillis;
2287+
return options.get(SCAN_TIMESTAMP_MILLIS);
23062288
}
23072289

23082290
public String scanTimestamp() {
@@ -2345,32 +2327,8 @@ public Pair<String, String> incrementalBetween() {
23452327
return Pair.of(split[0], split[1]);
23462328
}
23472329

2348-
public Pair<Long, Long> incrementalBetweenTimestamp() {
2349-
String str = options.get(INCREMENTAL_BETWEEN_TIMESTAMP);
2350-
String[] split = str.split(",");
2351-
if (split.length != 2) {
2352-
throw new IllegalArgumentException(
2353-
"The incremental-between-timestamp must specific start(exclusive) and end timestamp. But is: "
2354-
+ str);
2355-
}
2356-
2357-
try {
2358-
return Pair.of(Long.parseLong(split[0]), Long.parseLong(split[1]));
2359-
} catch (NumberFormatException nfe) {
2360-
try {
2361-
long startTimestamp =
2362-
DateTimeUtils.parseTimestampData(split[0], 3, TimeZone.getDefault())
2363-
.getMillisecond();
2364-
long endTimestamp =
2365-
DateTimeUtils.parseTimestampData(split[1], 3, TimeZone.getDefault())
2366-
.getMillisecond();
2367-
return Pair.of(startTimestamp, endTimestamp);
2368-
} catch (Exception e) {
2369-
throw new IllegalArgumentException(
2370-
"The incremental-between-timestamp must specific start(exclusive) and end timestamp. But is: "
2371-
+ str);
2372-
}
2373-
}
2330+
public String incrementalBetweenTimestamp() {
2331+
return options.get(INCREMENTAL_BETWEEN_TIMESTAMP);
23742332
}
23752333

23762334
public IncrementalBetweenScanMode incrementalBetweenScanMode() {
@@ -2638,6 +2596,10 @@ public boolean deletionVectorsEnabled() {
26382596
return options.get(DELETION_VECTORS_ENABLED);
26392597
}
26402598

2599+
public boolean forceLookup() {
2600+
return options.get(FORCE_LOOKUP);
2601+
}
2602+
26412603
public boolean batchScanSkipLevel0() {
26422604
return deletionVectorsEnabled() || mergeEngine() == FIRST_ROW;
26432605
}

paimon-common/src/main/java/org/apache/paimon/TableType.java renamed to paimon-api/src/main/java/org/apache/paimon/TableType.java

File renamed without changes.

paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroup.java renamed to paimon-api/src/main/java/org/apache/paimon/annotation/ConfigGroup.java

File renamed without changes.

paimon-common/src/main/java/org/apache/paimon/annotation/ConfigGroups.java renamed to paimon-api/src/main/java/org/apache/paimon/annotation/ConfigGroups.java

File renamed without changes.

paimon-common/src/main/java/org/apache/paimon/annotation/Documentation.java renamed to paimon-api/src/main/java/org/apache/paimon/annotation/Documentation.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818

1919
package org.apache.paimon.annotation;
2020

21-
import org.apache.paimon.options.ConfigOption;
22-
2321
import java.lang.annotation.ElementType;
2422
import java.lang.annotation.Retention;
2523
import java.lang.annotation.RetentionPolicy;
@@ -35,7 +33,7 @@ public final class Documentation {
3533
String value();
3634
}
3735

38-
/** Annotation used on {@link ConfigOption} fields to exclude it from schema change. */
36+
/** Annotation used on {@code ConfigOption} fields to exclude it from schema change. */
3937
@Target(ElementType.FIELD)
4038
@Retention(RetentionPolicy.RUNTIME)
4139
public @interface Immutable {}

0 commit comments

Comments
 (0)