Skip to content

Commit 7b8298a

Browse files
[Fix][Connector-v2] Add DateMilliConvertor to Convert DateMilliVector into Default Timezone (#8736)
1 parent 87e9355 commit 7b8298a

File tree

4 files changed

+51
-15
lines changed

4 files changed

+51
-15
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter;
19+
20+
import org.apache.seatunnel.shade.org.apache.arrow.vector.DateMilliVector;
21+
import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types;
22+
23+
import java.time.LocalDateTime;
24+
import java.time.ZoneId;
25+
import java.time.ZoneOffset;
26+
27+
public class DateMilliConvertor implements Converter<DateMilliVector> {
28+
@Override
29+
public Object convert(int rowIndex, DateMilliVector fieldVector) {
30+
if (fieldVector == null || fieldVector.isNull(rowIndex)) {
31+
return null;
32+
}
33+
LocalDateTime localDateTime = fieldVector.getObject(rowIndex);
34+
return localDateTime
35+
.atZone(ZoneOffset.UTC)
36+
.withZoneSameInstant(ZoneId.systemDefault())
37+
.toLocalDateTime();
38+
}
39+
40+
@Override
41+
public boolean support(Types.MinorType type) {
42+
return Types.MinorType.DATEMILLI == type;
43+
}
44+
}

Diff for: seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ private void convertSeatunnelRow() {
135135
Integer fieldIndex = fieldIndexMap.get(name);
136136
Types.MinorType minorType = fieldVector.getMinorType();
137137
for (int i = 0; i < seatunnelRowBatch.size(); i++) {
138-
// arrow field not in the Seatunnel Sechma field, skip it
138+
// arrow field not in the Seatunnel Schema field, skip it
139139
if (fieldIndex != null) {
140140
SeaTunnelDataType<?> seaTunnelDataType = seaTunnelDataTypes[fieldIndex];
141141
Object fieldValue =

Diff for: seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter

+1
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.StructCo
2222
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMicroConverter
2323
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMilliConverter
2424
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampNanoConverter
25+
org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DateMilliConvertor

Diff for: seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java

+5-14
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
import java.nio.charset.StandardCharsets;
7070
import java.time.LocalDateTime;
7171
import java.time.ZoneId;
72-
import java.time.temporal.ChronoUnit;
72+
import java.time.format.DateTimeFormatter;
7373
import java.util.ArrayList;
7474
import java.util.Arrays;
7575
import java.util.Collections;
@@ -84,13 +84,9 @@ public class ArrowToSeatunnelRowReaderTest {
8484
private static RootAllocator rootAllocator;
8585
private static final List<SeaTunnelDataTypeHolder> seaTunnelDataTypeHolder = new ArrayList<>();
8686

87-
/**
88-
* LocalDateTime.now() is timestamped with a precision of nanoseconds on linux and milliseconds
89-
* on windows The test case uses TimeStampMicroVector to test the timestamp, thus truncating the
90-
* timestamp accuracy to ChronoUnit.MILLIS
91-
*/
9287
private static final LocalDateTime localDateTime =
93-
LocalDateTime.now().truncatedTo(ChronoUnit.MILLIS);
88+
LocalDateTime.parse(
89+
"2025-02-15 02:21:23", DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
9490

9591
private static final List<String> stringData = new ArrayList<>();
9692
private static final List<Byte> byteData = new ArrayList<>();
@@ -172,13 +168,8 @@ private static VectorSchemaRoot buildVectorSchemaRoot(
172168
}
173169
// allocate storage
174170
vectors.forEach(FieldVector::allocateNew);
175-
// setVectorVaule
176-
long epochMilli =
177-
localDateTime
178-
.truncatedTo(ChronoUnit.MILLIS)
179-
.atZone(zoneId)
180-
.toInstant()
181-
.toEpochMilli();
171+
long epochMilli = localDateTime.atZone(zoneId).toInstant().toEpochMilli();
172+
182173
byte byteStart = 'a';
183174

184175
// setVectorValue

0 commit comments

Comments
 (0)