Skip to content

Commit b1dcd4a

Browse files
authored
[BugFix][Spark-translation] map type cast error (#6552)
1 parent 216efb7 commit b1dcd4a

File tree

2 files changed

+19
-10
lines changed
  • seatunnel-e2e/seatunnel-connector-v2-e2e/connector-fake-e2e/src/test/resources
  • seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization

2 files changed

+19
-10
lines changed

Diff for: seatunnel-e2e/seatunnel-connector-v2-e2e/connector-fake-e2e/src/test/resources/fake_to_assert.conf

+9
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,17 @@ source {
6767
}
6868
}
6969

70+
transform {
71+
Sql {
72+
source_table_name = "fake"
73+
result_table_name = "tmp1"
74+
query = """select * from fake"""
75+
}
76+
}
77+
7078
sink {
7179
Assert {
80+
source_table_name = "tmp1"
7281
rules {
7382
row_rules = [
7483
{

Diff for: seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/serialization/SeaTunnelRowConverter.java

+10-10
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import org.apache.spark.unsafe.types.UTF8String;
3232

3333
import scala.Tuple2;
34-
import scala.collection.immutable.HashMap.HashTrieMap;
34+
import scala.collection.immutable.AbstractMap;
3535
import scala.collection.mutable.WrappedArray;
3636

3737
import java.io.IOException;
@@ -179,7 +179,7 @@ private Object reconvert(Object field, SeaTunnelDataType<?> dataType) {
179179
case STRING:
180180
return field.toString();
181181
case MAP:
182-
return reconvertMap((HashTrieMap<?, ?>) field, (MapType<?, ?>) dataType);
182+
return reconvertMap((AbstractMap<?, ?>) field, (MapType<?, ?>) dataType);
183183
case ARRAY:
184184
return reconvertArray((WrappedArray.ofRef<?>) field, (ArrayType<?, ?>) dataType);
185185
default:
@@ -206,23 +206,23 @@ private SeaTunnelRow reconvert(SeaTunnelRow engineRow, SeaTunnelRowType rowType)
206206
}
207207

208208
/**
209-
* Convert HashTrieMap to LinkedHashMap
209+
* Convert AbstractMap to LinkedHashMap
210210
*
211-
* @param hashTrieMap HashTrieMap data
211+
* @param abstractMap AbstractMap data
212212
* @param mapType fields type map
213213
* @return java.util.LinkedHashMap
214-
* @see HashTrieMap
214+
* @see AbstractMap
215215
*/
216-
private Map<Object, Object> reconvertMap(HashTrieMap<?, ?> hashTrieMap, MapType<?, ?> mapType) {
217-
if (hashTrieMap == null || hashTrieMap.size() == 0) {
216+
private Map<Object, Object> reconvertMap(AbstractMap<?, ?> abstractMap, MapType<?, ?> mapType) {
217+
if (abstractMap == null || abstractMap.size() == 0) {
218218
return Collections.emptyMap();
219219
}
220-
int num = hashTrieMap.size();
220+
int num = abstractMap.size();
221221
Map<Object, Object> newMap = new LinkedHashMap<>(num);
222222
SeaTunnelDataType<?> keyType = mapType.getKeyType();
223223
SeaTunnelDataType<?> valueType = mapType.getValueType();
224-
scala.collection.immutable.List<?> keyList = hashTrieMap.keySet().toList();
225-
scala.collection.immutable.List<?> valueList = hashTrieMap.values().toList();
224+
scala.collection.immutable.List<?> keyList = abstractMap.keySet().toList();
225+
scala.collection.immutable.List<?> valueList = abstractMap.values().toList();
226226
for (int i = 0; i < num; i++) {
227227
Object key = keyList.apply(i);
228228
Object value = valueList.apply(i);

0 commit comments

Comments
 (0)