Skip to content

Commit 66d4480

Browse files
MuLeiSY2021MuLeiSY2021
committed
feat(loader): Restore functions rolled back due to force push and ensure date compatibility
1 parent efdf5c1 commit 66d4480

File tree

2 files changed

+92
-64
lines changed

2 files changed

+92
-64
lines changed

‎hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/DataTypeUtil.java‎

Lines changed: 53 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,22 @@
1717

1818
package org.apache.hugegraph.loader.util;
1919

20-
import com.google.common.base.Splitter;
21-
import com.google.common.collect.ImmutableSet;
20+
import java.time.LocalDateTime;
21+
import java.time.ZoneId;
22+
import java.util.ArrayList;
23+
import java.util.Collection;
24+
import java.util.Date;
25+
import java.util.HashSet;
26+
import java.util.List;
27+
import java.util.Set;
28+
import java.util.UUID;
29+
2230
import org.apache.hugegraph.loader.constant.Constants;
2331
import org.apache.hugegraph.loader.source.AbstractSource;
2432
import org.apache.hugegraph.loader.source.InputSource;
2533
import org.apache.hugegraph.loader.source.file.FileSource;
2634
import org.apache.hugegraph.loader.source.file.ListFormat;
35+
import org.apache.hugegraph.loader.source.jdbc.JDBCSource;
2736
import org.apache.hugegraph.loader.source.kafka.KafkaSource;
2837
import org.apache.hugegraph.structure.constant.Cardinality;
2938
import org.apache.hugegraph.structure.constant.DataType;
@@ -32,7 +41,8 @@
3241
import org.apache.hugegraph.util.InsertionOrderUtil;
3342
import org.apache.hugegraph.util.ReflectionUtil;
3443

35-
import java.util.*;
44+
import com.google.common.base.Splitter;
45+
import com.google.common.collect.ImmutableSet;
3646

3747
public final class DataTypeUtil {
3848

@@ -130,36 +140,48 @@ private static Object parseSingleValue(String key, Object rawValue,
130140
} else if (dataType.isBoolean()) {
131141
return parseBoolean(key, value);
132142
} else if (dataType.isDate()) {
133-
E.checkState(source instanceof FileSource,
134-
"Only accept FileSource when convert String value " +
135-
"to Date, but got '%s'", source.getClass().getName());
136-
String dateFormat = ((FileSource) source).dateFormat();
137-
String timeZone = ((FileSource) source).timeZone();
138-
143+
List<String> extraDateFormats = null;
144+
String dateFormat = null;
145+
String timeZone = null;
139146
if (source instanceof KafkaSource) {
140-
List<String> extraDateFormats =
141-
((KafkaSource) source).getExtraDateFormats();
142-
dateFormat = ((KafkaSource) source).getDateFormat();
143-
timeZone = ((KafkaSource) source).getTimeZone();
144-
if (extraDateFormats == null || extraDateFormats.isEmpty()) {
145-
return parseDate(key, value, dateFormat, timeZone);
146-
} else {
147-
HashSet<String> allDateFormats = new HashSet<>();
148-
allDateFormats.add(dateFormat);
149-
allDateFormats.addAll(extraDateFormats);
150-
int size = allDateFormats.size();
151-
for (String df : allDateFormats) {
152-
try {
153-
return parseDate(key, value, df, timeZone);
154-
} catch (Exception e) {
155-
if (--size <= 0) {
156-
throw e;
157-
}
147+
KafkaSource kafkaSource = (KafkaSource) source;
148+
extraDateFormats = kafkaSource.getExtraDateFormats();
149+
dateFormat = kafkaSource.getDateFormat();
150+
timeZone = kafkaSource.getTimeZone();
151+
} else if (source instanceof JDBCSource) {
152+
/*Warn:it uses system default timezone,
153+
should we thought a better way to compatible differ timezone people?*/
154+
long timestamp = 0L;
155+
if (value instanceof Date) {
156+
timestamp = ((Date) value).getTime();
157+
} else if (value instanceof LocalDateTime) {
158+
timestamp = ((LocalDateTime) value).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
159+
}
160+
value = new Date(timestamp);
161+
162+
} else if (source instanceof FileSource) {
163+
FileSource fileSource = (FileSource) source;
164+
dateFormat = fileSource.dateFormat();
165+
timeZone = fileSource.timeZone();
166+
}
167+
168+
if (extraDateFormats == null || extraDateFormats.isEmpty()) {
169+
return parseDate(key, value, dateFormat, timeZone);
170+
} else {
171+
HashSet<String> allDateFormats = new HashSet<>();
172+
allDateFormats.add(dateFormat);
173+
allDateFormats.addAll(extraDateFormats);
174+
int size = allDateFormats.size();
175+
for (String df : allDateFormats) {
176+
try {
177+
return parseDate(key, value, df, timeZone);
178+
} catch (Exception e) {
179+
if (--size <= 0) {
180+
throw e;
158181
}
159182
}
160183
}
161184
}
162-
163185
return parseDate(key, value, dateFormat, timeZone);
164186
} else if (dataType.isUUID()) {
165187
return parseUUID(key, value);
@@ -220,9 +242,9 @@ private static Boolean parseBoolean(String key, Object rawValue) {
220242
return false;
221243
} else {
222244
throw new IllegalArgumentException(String.format(
223-
"Failed to convert '%s'(key='%s') to Boolean, " +
224-
"the acceptable boolean strings are %s or %s",
225-
key, rawValue, ACCEPTABLE_TRUE, ACCEPTABLE_FALSE));
245+
"Failed to convert '%s'(key='%s') to Boolean, " +
246+
"the acceptable boolean strings are %s or %s",
247+
key, rawValue, ACCEPTABLE_TRUE, ACCEPTABLE_FALSE));
226248
}
227249
}
228250
throw new IllegalArgumentException(String.format("Failed to convert value(key='%s') " +

‎hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/JDBCLoadTest.java‎

Lines changed: 39 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,23 @@
1717

1818
package org.apache.hugegraph.loader.test.functional;
1919

20-
import org.apache.hugegraph.loader.HugeGraphLoader;
21-
import org.apache.hugegraph.structure.graph.Edge;
22-
import org.apache.hugegraph.structure.graph.Vertex;
23-
import org.apache.hugegraph.testutil.Assert;
24-
import org.junit.*;
25-
2620
import java.time.LocalDate;
2721
import java.time.LocalDateTime;
2822
import java.time.LocalTime;
2923
import java.time.Year;
3024
import java.time.format.DateTimeFormatter;
3125
import java.util.List;
3226

27+
import org.apache.hugegraph.loader.HugeGraphLoader;
28+
import org.apache.hugegraph.structure.graph.Edge;
29+
import org.apache.hugegraph.structure.graph.Vertex;
30+
import org.apache.hugegraph.testutil.Assert;
31+
import org.junit.After;
32+
import org.junit.AfterClass;
33+
import org.junit.Before;
34+
import org.junit.BeforeClass;
35+
import org.junit.Test;
36+
3337
/**
3438
* TODO: add more test cases
3539
*/
@@ -73,14 +77,14 @@ public static void setUp() {
7377
") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
7478
// vertex date
7579
dbUtil.execute("CREATE TABLE IF NOT EXISTS `date_test` (" +
76-
"`id` int(10) unsigned NOT NULL," +
77-
"`calendar_date` DATE NOT NULL," +
78-
"`calendar_datetime` DATETIME NOT NULL," +
79-
"`calendar_timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP," +
80-
"`calendar_time` TIME NOT NULL," +
81-
"`calendar_year` YEAR NOT NULL," +
82-
"PRIMARY KEY (`id`)" +
83-
") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
80+
"`id` int(10) unsigned NOT NULL," +
81+
"`calendar_date` DATE NOT NULL," +
82+
"`calendar_datetime` DATETIME NOT NULL," +
83+
"`calendar_timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP," +
84+
"`calendar_time` TIME NOT NULL," +
85+
"`calendar_year` YEAR NOT NULL," +
86+
"PRIMARY KEY (`id`)" +
87+
") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
8488

8589
// edge knows
8690
dbUtil.execute("CREATE TABLE IF NOT EXISTS `knows` (" +
@@ -101,7 +105,6 @@ public static void setUp() {
101105
"PRIMARY KEY (`id`)" +
102106
") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
103107

104-
105108
}
106109

107110
@AfterClass
@@ -229,16 +232,16 @@ public void testValueMappingInJDBCSource() {
229232
@Test
230233
public void testNumberToStringInJDBCSource() {
231234
dbUtil.insert("INSERT INTO `person` VALUES " +
232-
"(1,'marko',29,'Beijing')," +
233-
"(2,'vadas',27,'HongKong')," +
234-
"(3,'josh',32,'Beijing')," +
235-
"(4,'peter',35,'Shanghai')," +
236-
"(5,'li,nary',26,'Wu,han')," +
237-
"(6,'tom',NULL,NULL);");
235+
"(1,'marko',29,'Beijing')," +
236+
"(2,'vadas',27,'HongKong')," +
237+
"(3,'josh',32,'Beijing')," +
238+
"(4,'peter',35,'Shanghai')," +
239+
"(5,'li,nary',26,'Wu,han')," +
240+
"(6,'tom',NULL,NULL);");
238241

239242
dbUtil.insert("INSERT INTO `software` VALUES " +
240-
"(100,'lop','java',328.08)," +
241-
"(200,'ripple','java',199.67);");
243+
"(100,'lop','java',328.08)," +
244+
"(200,'ripple','java',199.67);");
242245

243246
String[] args = new String[]{
244247
"-f", configPath("jdbc_number_to_string/struct.json"),
@@ -261,9 +264,12 @@ public void testNumberToStringInJDBCSource() {
261264
@Test
262265
public void testJdbcSqlDateConvert() {
263266
dbUtil.execute("INSERT INTO `date_test` VALUES " +
264-
"(1, '2017-12-10', '2017-12-10 15:30:45', '2017-12-10 15:30:45', '15:30:45', '2017')," +
265-
"(2, '2009-11-11', '2009-11-11 08:15:30', '2009-11-11 08:15:30', '08:15:30', '2009')," +
266-
"(3, '2017-03-24', '2017-03-24 12:00:00', '2017-03-24 12:00:00', '12:00:00', '2017');");
267+
"(1, '2017-12-10', '2017-12-10 15:30:45', '2017-12-10 15:30:45', " +
268+
"'15:30:45', '2017')," +
269+
"(2, '2009-11-11', '2009-11-11 08:15:30', '2009-11-11 08:15:30', " +
270+
"'08:15:30', '2009')," +
271+
"(3, '2017-03-24', '2017-03-24 12:00:00', '2017-03-24 12:00:00', " +
272+
"'12:00:00', '2017');");
267273

268274
String[] args = new String[]{
269275
"-f", configPath("jdbc_sql_date_convert/struct.json"),
@@ -280,8 +286,8 @@ public void testJdbcSqlDateConvert() {
280286

281287
Assert.assertEquals(3, vertices.size());
282288
// Define formatters
283-
DateTimeFormatter serverDateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
284-
289+
DateTimeFormatter serverDateFormatter =
290+
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
285291

286292
// DATE check
287293
DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
@@ -302,10 +308,10 @@ public void testJdbcSqlDateConvert() {
302308
LocalDateTime yearStart = year.atDay(1).atStartOfDay(); // 补充日期为该年的第一天
303309

304310
assertContains(vertices, "date_test",
305-
"calendar_date", date.format(serverDateFormatter),
306-
"calendar_datetime", datetime.format(serverDateFormatter),
307-
"calendar_timestamp", timestamp.format(serverDateFormatter),
308-
"calendar_time", timeWithDate.format(serverDateFormatter),
309-
"calendar_year", yearStart.format(serverDateFormatter));
311+
"calendar_date", date.format(serverDateFormatter),
312+
"calendar_datetime", datetime.format(serverDateFormatter),
313+
"calendar_timestamp", timestamp.format(serverDateFormatter),
314+
"calendar_time", timeWithDate.format(serverDateFormatter),
315+
"calendar_year", yearStart.format(serverDateFormatter));
310316
}
311317
}

0 commit comments

Comments
 (0)