Skip to content

[BitSail][Feature] Support generate constant value in FakeSource #341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[BitSail][Feature] Support generate constant value in FakeSource #341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f17102c
close #244 [BitSail][Feature] Support generate constant value in Fake…
kyle-hawk Jan 29, 2023
8801d4b
doc for #244
kyle-hawk Jan 29, 2023
3813515
changed fellowed the guidance from https://github.com/bytedance/bitsa…
kyle-hawk Feb 21, 2023
7fec17a
fix checkstyle problem
kyle-hawk Feb 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 110 additions & 12 deletions ...ctor-fake/src/main/java/com/bytedance/bitsail/connector/fake/source/FakeRowGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.bytedance.bitsail.connector.fake.source;

import com.bytedance.bitsail.common.configuration.BitSailConfiguration;
import com.bytedance.bitsail.common.model.ColumnInfo;
import com.bytedance.bitsail.common.row.Row;
import com.bytedance.bitsail.common.typeinfo.BasicArrayTypeInfo;
import com.bytedance.bitsail.common.typeinfo.BasicTypeInfo;
Expand All @@ -38,7 +39,13 @@
import java.math.BigInteger;
import java.sql.Time;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import java.util.Objects;

import static com.bytedance.bitsail.common.typeinfo.TypeProperty.NULLABLE;

Expand All @@ -52,6 +59,7 @@ public class FakeRowGenerator {
private final long lower;
private final transient Timestamp fromTimestamp;
private final transient Timestamp toTimestamp;
private final List<ColumnInfo> columnInfos;

public FakeRowGenerator(BitSailConfiguration jobConf, int taskId) {
this.faker = new Faker();
Expand All @@ -62,6 +70,7 @@ public FakeRowGenerator(BitSailConfiguration jobConf, int taskId) {
this.lower = jobConf.get(FakeReaderOptions.LOWER_LIMIT);
this.fromTimestamp = Timestamp.valueOf(jobConf.get(FakeReaderOptions.FROM_TIMESTAMP));
this.toTimestamp = Timestamp.valueOf(jobConf.get(FakeReaderOptions.TO_TIMESTAMP));
this.columnInfos = jobConf.get(FakeReaderOptions.COLUMNS);
}

public Row fakeOneRecord(TypeInfo<?>[] typeInfos) {
Expand All @@ -71,7 +80,8 @@ public Row fakeOneRecord(TypeInfo<?>[] typeInfos) {
if (isNullable(typeInfo) && isNull()) {
row.setField(index, null);
} else {
row.setField(index, fakeRawValue(typeInfo));
Object constantValue = this.columnInfos.get(index).getDefaultValue();
row.setField(index, fakeRawValue(typeInfo, constantValue == null ? null : constantValue.toString()));
}
}
return row;
Expand All @@ -89,74 +99,162 @@ private boolean isNull() {
}

@SuppressWarnings("checkstyle:MagicNumber")
private Object fakeRawValue(TypeInfo<?> typeInfo) {

private Object fakeRawValue(TypeInfo<?> typeInfo, String constantValue) {
if (TypeInfos.LONG_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (CollectionUtils.isNotEmpty(typeInfo.getTypeProperties()) && typeInfo.getTypeProperties().contains(TypeProperty.UNIQUE)) {
if (!Objects.isNull(constantValue)) {
throw new RuntimeException("unique and defaultValue can't be specified at the same time");
}
return snowflakeIdGenerator.nextId();
} else {
if (!Objects.isNull(constantValue)) {
return Long.valueOf(constantValue).longValue();
}
return faker.number().randomNumber();
}
} else if (TypeInfos.INT_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return Long.valueOf(constantValue).intValue();
}
return Long.valueOf(faker.number().randomNumber()).intValue();

} else if (TypeInfos.SHORT_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return Long.valueOf(constantValue).shortValue();
}
return Long.valueOf(faker.number().randomNumber()).shortValue();

} else if (TypeInfos.STRING_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return constantValue;
}
return faker.name().fullName();

} else if (TypeInfos.BOOLEAN_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return Boolean.valueOf(constantValue).booleanValue();
}
return faker.bool().bool();

} else if (TypeInfos.DOUBLE_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return Double.valueOf(constantValue).doubleValue();
}
return faker.number().randomDouble(5, lower, upper);

} else if (TypeInfos.FLOAT_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return Double.valueOf(constantValue).floatValue();
}
return Double.valueOf(faker.number().randomDouble(5, lower, upper)).floatValue();

} else if (TypeInfos.BIG_DECIMAL_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return BigDecimal.valueOf(Double.valueOf(constantValue));
}
return new BigDecimal(faker.number().randomDouble(5, lower, upper));

} else if (TypeInfos.BIG_INTEGER_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return BigInteger.valueOf(Long.valueOf(constantValue));
}
return new BigInteger(String.valueOf(faker.number().randomNumber()));

} else if (BasicArrayTypeInfo.BINARY_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
if (!Objects.isNull(constantValue)) {
return constantValue.getBytes();
}
return faker.name().fullName().getBytes();

} else if (TypeInfos.SQL_DATE_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
return new java.sql.Date(faker.date().between(fromTimestamp, toTimestamp).getTime());
if (Objects.isNull(constantValue)) {
return new java.sql.Date(faker.date().between(fromTimestamp, toTimestamp).getTime());
}
if (constantValue.equalsIgnoreCase("now")) {
return new java.sql.Date(System.currentTimeMillis());
} else {
return java.sql.Date.valueOf(constantValue);
}

} else if (TypeInfos.SQL_TIME_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
return new Time(faker.date().between(fromTimestamp, toTimestamp).getTime());
if (Objects.isNull(constantValue)) {
return new Time(faker.date().between(fromTimestamp, toTimestamp).getTime());
}
if (constantValue.equalsIgnoreCase("now")) {
return new Time(System.currentTimeMillis());
} else {
return Time.valueOf(constantValue);
}

} else if (TypeInfos.SQL_TIMESTAMP_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
return new Timestamp(faker.date().between(fromTimestamp, toTimestamp).getTime());
if (Objects.isNull(constantValue)) {
return new Timestamp(faker.date().between(fromTimestamp, toTimestamp).getTime());
}
if (constantValue.equalsIgnoreCase("now")) {
return new Timestamp(System.currentTimeMillis());
} else {
return new Timestamp(Long.valueOf(constantValue));
}

} else if (TypeInfos.LOCAL_DATE_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
return faker.date().between(fromTimestamp, toTimestamp).toLocalDateTime().toLocalDate();
if (Objects.isNull(constantValue)) {
return faker.date().between(fromTimestamp, toTimestamp).toLocalDateTime().toLocalDate();
}
if (constantValue.equalsIgnoreCase("now")) {
return LocalDate.now();
} else {
return LocalDate.parse(constantValue);
}

} else if (TypeInfos.LOCAL_TIME_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
return faker.date().between(fromTimestamp, toTimestamp).toLocalDateTime().toLocalTime();
if (Objects.isNull(constantValue)) {
return faker.date().between(fromTimestamp, toTimestamp).toLocalDateTime().toLocalTime();
}
if (constantValue.equalsIgnoreCase("now")) {
return LocalTime.now();
} else {
return LocalTime.parse(constantValue);
}

} else if (TypeInfos.LOCAL_DATE_TIME_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
return faker.date().between(fromTimestamp, toTimestamp).toLocalDateTime();
if (Objects.isNull(constantValue)) {
return faker.date().between(fromTimestamp, toTimestamp).toLocalDateTime();
}
if (constantValue.equalsIgnoreCase("now")) {
return LocalDateTime.now();
} else if (constantValue.contains(":")) {
DateTimeFormatter fm = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
return LocalDateTime.parse(constantValue, fm);
} else {
throw new IllegalArgumentException(String.format("unSupport timestamp value [%s]", constantValue));
}

} else if (TypeInfos.VOID_TYPE_INFO.getTypeClass() == typeInfo.getTypeClass()) {
return null;
}

if (typeInfo instanceof ListTypeInfo) {
ListTypeInfo<?> listTypeInfo = (ListTypeInfo<?>) typeInfo;
return Lists.newArrayList(fakeRawValue(listTypeInfo.getElementTypeInfo()));
return Lists.newArrayList(fakeRawValue(listTypeInfo.getElementTypeInfo(), constantValue));
}

if (typeInfo instanceof MapTypeInfo) {
MapTypeInfo<?, ?> mapTypeInfo = (MapTypeInfo<?, ?>) typeInfo;
Map<Object, Object> mapRawValue = Maps.newHashMap();
mapRawValue.put(fakeRawValue(mapTypeInfo.getKeyTypeInfo()), fakeRawValue(mapTypeInfo.getValueTypeInfo()));
return mapRawValue;
if (Objects.isNull(constantValue)) {
mapRawValue.put(fakeRawValue(mapTypeInfo.getKeyTypeInfo(), null), fakeRawValue(mapTypeInfo.getValueTypeInfo(), null));
return mapRawValue;
} else {
String[] kv = constantValue.split("_:_");
if (kv.length < 2) {
throw new IllegalArgumentException("defualt value of MapType requires key and value with _:_ as splitor");
}
mapRawValue.put(fakeRawValue(mapTypeInfo.getKeyTypeInfo(), kv[0]), fakeRawValue(mapTypeInfo.getValueTypeInfo(), kv[1]));
return mapRawValue;
}

}
throw new RuntimeException("Unsupported type " + typeInfo);
}

}
114 changes: 114 additions & 0 deletions bitsail-connectors/connector-fake/src/test/resources/fake_with_constant_value.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
{
"job": {
"common": {
"cid": 0,
"domain": "test",
"job_id": -23,
"job_name": "bitsail_hive_mysql_tqs_regression",
"instance_id": -203,
"user_name": "root",
"bitsail_location": "./"
},
"reader": {
"class": "com.bytedance.bitsail.connector.fake.source.FakeSource",
"total_count": 300,
"rate": 10,
"columns": [
{
"name": "id",
"type": "long",
"properties": "unique"
},
{
"name": "list_value",
"type": "list<string>",
"defaultValue": "test_element"
},
{
"name": "map_value",
"type": "map<string,string>",
"defaultValue": "test_key_:_test_value"
},
{
"name": "long_value",
"type": "long",
"defaultValue": "9223372036854775807"
},
{
"name": "int_value",
"type": "int",
"defaultValue": "2147483647"
},
{
"name": "short_value",
"type": "short",
"defaultValue": "32767"
},
{
"name": "string_value",
"type": "string",
"defaultValue": "this is test constant string"
},
{
"name": "boolean_value",
"type": "boolean",
"defaultValue": "false"
},
{
"name": "double_value",
"type": "double",
"defaultValue": "13.5"
},
{
"name": "float_value",
"type": "float",
"defaultValue": "17.8"
},
{
"name": "bigdecimal_value",
"type": "bigdecimal",
"defaultValue": "17.8"
},
{
"name": "biginteger_value",
"type": "biginteger",
"defaultValue": "17"
},
{
"name": "sql_date_value",
"type": "date.date",
"defaultValue": "2023-01-22"
},
{
"name": "sql_time_value",
"type": "date.time",
"defaultValue": "23:13:22"
},
{
"name": "sql_timestamp_value",
"type": "date.datetime",
"defaultValue": "1674897553000"
},
{
"name": "localdatetime_value",
"type": "timestamp",
"defaultValue": "2023-01-06 23:14:22.111"
},
{
"name": "localdate_value",
"type": "date",
"defaultValue": "2023-01-22"
},
{
"name": "localdate_value_now",
"type": "date",
"defaultValue": "now"
}
]
},
"writer": {
"class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink",
"writer_parallelism_num": 2
}
}
}
Loading