Skip to content

Commit 28febe9

Browse files
predator4annpredator4ann
committed
feat: support read time/datetime/timestamp columns with precision
Signed-off-by: zacsun <[email protected]>
1 parent 121888c commit 28febe9

File tree

40 files changed

+1510
-216
lines changed

40 files changed

+1510
-216
lines changed

‎api/src/main/java/org/apache/gravitino/rel/types/Type.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,23 @@ abstract class PrimitiveType implements Type {}
106106
abstract class NumericType extends PrimitiveType {}
107107

108108
/** The base type of all date/time types. */
109-
abstract class DateTimeType extends PrimitiveType {}
109+
abstract class DateTimeType extends PrimitiveType {
110+
/**
111+
* Indicates that precision for the date/time type was not explicitly set by the user. The value
112+
* should be converted to the catalog's default precision.
113+
*/
114+
protected static final int DATE_TIME_PRECISION_NOT_SET = -1;
115+
/**
116+
* Represents the minimum precision range for timestamp, time and other date/time types. The
117+
* minimum precision is 0, which means second-level precision.
118+
*/
119+
protected static final int MIN_ALLOWED_PRECISION = 0;
120+
/**
121+
* Represents the maximum precision allowed for timestamp, time and other date/time types. The
122+
* maximum precision is 12, which means picosecond-level precision.
123+
*/
124+
protected static final int MAX_ALLOWED_PRECISION = 12;
125+
}
110126

111127
/** The base type of all interval types. */
112128
abstract class IntervalType extends PrimitiveType {}

‎api/src/main/java/org/apache/gravitino/rel/types/Types.java

Lines changed: 112 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,39 @@ public static TimeType get() {
337337
return INSTANCE;
338338
}
339339

340-
private TimeType() {}
340+
/**
341+
* @param precision The precision of the time type.
342+
* @return A {@link TimeType} with the given precision.
343+
*/
344+
public static TimeType of(int precision) {
345+
Preconditions.checkArgument(
346+
precision >= MIN_ALLOWED_PRECISION && precision <= MAX_ALLOWED_PRECISION,
347+
"precision must be in range [%s, %s]: precision: %s",
348+
MIN_ALLOWED_PRECISION,
349+
MAX_ALLOWED_PRECISION,
350+
precision);
351+
return new TimeType(precision);
352+
}
353+
354+
private final int precision;
355+
356+
private TimeType() {
357+
this(DATE_TIME_PRECISION_NOT_SET);
358+
}
359+
360+
private TimeType(int precision) {
361+
this.precision = precision;
362+
}
363+
364+
/** @return The precision of the time type. */
365+
public int precision() {
366+
return precision;
367+
}
368+
369+
/** @return True if the time type has precision set, false otherwise. */
370+
public boolean hasPrecisionSet() {
371+
return precision != DATE_TIME_PRECISION_NOT_SET;
372+
}
341373

342374
@Override
343375
public Name name() {
@@ -346,7 +378,20 @@ public Name name() {
346378

347379
@Override
348380
public String simpleString() {
349-
return "time";
381+
return hasPrecisionSet() ? String.format("time(%d)", precision) : "time";
382+
}
383+
384+
@Override
385+
public boolean equals(Object o) {
386+
if (this == o) return true;
387+
if (!(o instanceof TimeType)) return false;
388+
TimeType that = (TimeType) o;
389+
return precision == that.precision;
390+
}
391+
392+
@Override
393+
public int hashCode() {
394+
return precision;
350395
}
351396
}
352397

@@ -355,27 +400,71 @@ public static class TimestampType extends Type.DateTimeType {
355400
private static final TimestampType INSTANCE_WITHOUT_TIME_ZONE = new TimestampType(false);
356401
private static final TimestampType INSTANCE_WITH_TIME_ZONE = new TimestampType(true);
357402

403+
/** @return A {@link TimestampType} without time zone. */
404+
public static TimestampType withoutTimeZone() {
405+
return INSTANCE_WITHOUT_TIME_ZONE;
406+
}
407+
358408
/** @return A {@link TimestampType} with time zone. */
359409
public static TimestampType withTimeZone() {
360410
return INSTANCE_WITH_TIME_ZONE;
361411
}
362412

363-
/** @return A {@link TimestampType} without time zone. */
364-
public static TimestampType withoutTimeZone() {
365-
return INSTANCE_WITHOUT_TIME_ZONE;
413+
/**
414+
* @param precision The precision of the timestamp type.
415+
* @return A {@link TimestampType} with the given precision and without time zone.
416+
*/
417+
public static TimestampType withoutTimeZone(int precision) {
418+
Preconditions.checkArgument(
419+
precision >= MIN_ALLOWED_PRECISION && precision <= MAX_ALLOWED_PRECISION,
420+
"precision must be in range [%s, %s]: precision: %s",
421+
MIN_ALLOWED_PRECISION,
422+
MAX_ALLOWED_PRECISION,
423+
precision);
424+
return new TimestampType(false, precision);
425+
}
426+
427+
/**
428+
* @param precision The precision of the timestamp type.
429+
* @return A {@link TimestampType} with the given precision and time zone.
430+
*/
431+
public static TimestampType withTimeZone(int precision) {
432+
Preconditions.checkArgument(
433+
precision >= MIN_ALLOWED_PRECISION && precision <= MAX_ALLOWED_PRECISION,
434+
"precision must be in range [%s, %s]: precision: %s",
435+
MIN_ALLOWED_PRECISION,
436+
MAX_ALLOWED_PRECISION,
437+
precision);
438+
return new TimestampType(true, precision);
366439
}
367440

368441
private final boolean withTimeZone;
442+
private final int precision;
369443

370444
private TimestampType(boolean withTimeZone) {
445+
this(withTimeZone, DATE_TIME_PRECISION_NOT_SET);
446+
}
447+
448+
private TimestampType(boolean withTimeZone, int precision) {
371449
this.withTimeZone = withTimeZone;
450+
this.precision = precision;
372451
}
373452

374453
/** @return True if the timestamp type has time zone, false otherwise. */
375454
public boolean hasTimeZone() {
376455
return withTimeZone;
377456
}
378457

458+
/** @return The precision of the timestamp type. */
459+
public int precision() {
460+
return precision;
461+
}
462+
463+
/** @return True if the timestamp type has precision set, false otherwise. */
464+
public boolean hasPrecisionSet() {
465+
return precision != DATE_TIME_PRECISION_NOT_SET;
466+
}
467+
379468
@Override
380469
public Name name() {
381470
return Name.TIMESTAMP;
@@ -384,7 +473,24 @@ public Name name() {
384473
/** @return The simple string representation of the timestamp type. */
385474
@Override
386475
public String simpleString() {
387-
return withTimeZone ? "timestamp_tz" : "timestamp";
476+
return hasPrecisionSet()
477+
? withTimeZone
478+
? String.format("timestamp_tz(%d)", precision)
479+
: String.format("timestamp(%d)", precision)
480+
: withTimeZone ? "timestamp_tz" : "timestamp";
481+
}
482+
483+
@Override
484+
public boolean equals(Object o) {
485+
if (this == o) return true;
486+
if (!(o instanceof TimestampType)) return false;
487+
TimestampType that = (TimestampType) o;
488+
return withTimeZone == that.withTimeZone && precision == that.precision;
489+
}
490+
491+
@Override
492+
public int hashCode() {
493+
return Objects.hash(withTimeZone, precision);
388494
}
389495
}
390496

‎catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/converter/JdbcTypeConverter.java

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public abstract class JdbcTypeConverter
2929
public static final String TIMESTAMP = "timestamp";
3030
public static final String VARCHAR = "varchar";
3131
public static final String TEXT = "text";
32+
public static final int PRECISION_SECOND = 0;
3233

3334
public static class JdbcTypeBean {
3435
/** Data type name. */
@@ -40,6 +41,8 @@ public static class JdbcTypeBean {
4041
/** Scale. For example: 2 in decimal (10,2). */
4142
private Integer scale;
4243

44+
private Integer datetimePrecision;
45+
4346
public JdbcTypeBean(String typeName) {
4447
this.typeName = typeName;
4548
}
@@ -68,19 +71,28 @@ public void setScale(Integer scale) {
6871
this.scale = scale;
6972
}
7073

74+
public Integer getDatetimePrecision() {
75+
return datetimePrecision;
76+
}
77+
78+
public void setDatetimePrecision(Integer datetimePrecision) {
79+
this.datetimePrecision = datetimePrecision;
80+
}
81+
7182
@Override
7283
public boolean equals(Object o) {
7384
if (this == o) return true;
7485
if (!(o instanceof JdbcTypeBean)) return false;
7586
JdbcTypeBean typeBean = (JdbcTypeBean) o;
7687
return Objects.equals(typeName, typeBean.typeName)
7788
&& Objects.equals(columnSize, typeBean.columnSize)
78-
&& Objects.equals(scale, typeBean.scale);
89+
&& Objects.equals(scale, typeBean.scale)
90+
&& Objects.equals(datetimePrecision, typeBean.datetimePrecision);
7991
}
8092

8193
@Override
8294
public int hashCode() {
83-
return Objects.hash(typeName, columnSize, scale);
95+
return Objects.hash(typeName, columnSize, scale, datetimePrecision);
8496
}
8597

8698
@Override
@@ -95,6 +107,9 @@ public String toString() {
95107
+ ", scale='"
96108
+ scale
97109
+ '\''
110+
+ ", datetimePrecision='"
111+
+ datetimePrecision
112+
+ '\''
98113
+ '}';
99114
}
100115
}

‎catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/operation/JdbcTableOperations.java

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ public abstract class JdbcTableOperations implements TableOperation {
6666

6767
public static final String COMMENT = "COMMENT";
6868
public static final String SPACE = " ";
69+
public static final String TIME_FORMAT_WITH_DOT = "HH:MM:SS.";
70+
public static final String DATETIME_FORMAT_WITH_DOT = "YYYY-MM-DD HH:MM:SS.";
6971

7072
public static final String MODIFY_COLUMN = "MODIFY COLUMN ";
7173
public static final String AFTER = "AFTER ";
@@ -607,10 +609,15 @@ protected JdbcTable.Builder getBasicJdbcTableInfo(ResultSet table) throws SQLExc
607609
}
608610

609611
protected JdbcColumn.Builder getBasicJdbcColumnInfo(ResultSet column) throws SQLException {
610-
JdbcTypeConverter.JdbcTypeBean typeBean =
611-
new JdbcTypeConverter.JdbcTypeBean(column.getString("TYPE_NAME"));
612-
typeBean.setColumnSize(column.getInt("COLUMN_SIZE"));
613-
typeBean.setScale(column.getInt("DECIMAL_DIGITS"));
612+
String typeName = column.getString("TYPE_NAME");
613+
int columnSize = column.getInt("COLUMN_SIZE");
614+
int scale = column.getInt("DECIMAL_DIGITS");
615+
JdbcTypeConverter.JdbcTypeBean typeBean = new JdbcTypeConverter.JdbcTypeBean(typeName);
616+
typeBean.setColumnSize(columnSize);
617+
typeBean.setScale(scale);
618+
Integer datetimePrecision = calculateDatetimePrecision(typeName, columnSize, scale);
619+
typeBean.setDatetimePrecision(datetimePrecision);
620+
614621
String comment = column.getString("REMARKS");
615622
boolean nullable = column.getBoolean("NULLABLE");
616623

@@ -626,4 +633,16 @@ protected JdbcColumn.Builder getBasicJdbcColumnInfo(ResultSet column) throws SQL
626633
.withNullable(nullable)
627634
.withDefaultValue(defaultValue);
628635
}
636+
637+
/**
638+
* Calculate the precision for time/datetime/timestamp types.
639+
*
640+
* @param typeName the type name from database
641+
* @param columnSize the column size from database
642+
* @param scale
643+
* @return the precision of the time/datetime/timestamp type
644+
*/
645+
public Integer calculateDatetimePrecision(String typeName, int columnSize, int scale) {
646+
return null;
647+
}
629648
}

‎catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/converter/DorisTypeConverter.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.apache.gravitino.catalog.doris.converter;
2020

21+
import java.util.Optional;
2122
import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter;
2223
import org.apache.gravitino.rel.types.Type;
2324
import org.apache.gravitino.rel.types.Types;
@@ -58,7 +59,9 @@ public Type toGravitino(JdbcTypeBean typeBean) {
5859
case DATE:
5960
return Types.DateType.get();
6061
case DATETIME:
61-
return Types.TimestampType.withoutTimeZone();
62+
return Optional.ofNullable(typeBean.getDatetimePrecision())
63+
.map(Types.TimestampType::withoutTimeZone)
64+
.orElseGet(Types.TimestampType::withoutTimeZone);
6265
case CHAR:
6366
return Types.FixedCharType.of(typeBean.getColumnSize());
6467
case VARCHAR:
@@ -97,7 +100,10 @@ public String fromGravitino(Type type) {
97100
} else if (type instanceof Types.DateType) {
98101
return DATE;
99102
} else if (type instanceof Types.TimestampType) {
100-
return DATETIME;
103+
Types.TimestampType timestampType = (Types.TimestampType) type;
104+
return timestampType.hasPrecisionSet()
105+
? String.format("%s(%d)", DATETIME, timestampType.precision())
106+
: DATETIME;
101107
} else if (type instanceof Types.VarCharType) {
102108
int length = ((Types.VarCharType) type).length();
103109
if (length < 1 || length > 65533) {

‎catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -795,4 +795,16 @@ protected Distribution getDistributionInfo(
795795
return DorisUtils.extractDistributionInfoFromSql(createTableSyntax);
796796
}
797797
}
798+
799+
@Override
800+
public Integer calculateDatetimePrecision(String typeName, int columnSize, int scale) {
801+
String upperTypeName = typeName.toUpperCase();
802+
if (upperTypeName.equals("DATETIME")) {
803+
// DATETIME format: 'YYYY-MM-DD HH:MM:SS' (19 chars) + decimal point + precision
804+
return columnSize >= DATETIME_FORMAT_WITH_DOT.length()
805+
? columnSize - DATETIME_FORMAT_WITH_DOT.length()
806+
: 0;
807+
}
808+
return null;
809+
}
798810
}

0 commit comments

Comments
 (0)