Skip to content

[Feature][transform-v2] SQL transform support schema change #8438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 25 commits into
base: dev
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
b2639d6
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 3, 2025
0a609a2
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 3, 2025
36757c3
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 6, 2025
c93e625
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 6, 2025
7c9f250
Merge remote-tracking branch 'refs/remotes/upstream/dev' into transfo…
CosmosNi Jan 6, 2025
dd21c33
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 6, 2025
2962de8
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 6, 2025
cc95077
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 7, 2025
e8ebfce
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 7, 2025
5d1a14d
Merge remote-tracking branch 'refs/remotes/upstream/dev' into transfo…
CosmosNi Jan 7, 2025
d061b14
[Feature][connector-elasticsearch] elasticsearch support nested type
CosmosNi Jan 8, 2025
497e8eb
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 8, 2025
537aa64
Merge remote-tracking branch 'origin/transform_sql_schema_change' int…
CosmosNi Jan 8, 2025
4dbe153
Merge remote-tracking branch 'refs/remotes/upstream/dev' into transfo…
CosmosNi Jan 8, 2025
c903b3a
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 8, 2025
f83eff0
Merge remote-tracking branch 'upstream/dev' into transform_sql_schema…
CosmosNi Jan 11, 2025
9bbea15
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 11, 2025
b880a18
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 11, 2025
269ee68
[Feature][transform-v2] SQL transform support schema change
Jan 14, 2025
9c5c5c6
[Feature][transform-v2] SQL transform support schema change
Jan 14, 2025
2d17b66
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 14, 2025
482f43b
[Feature][transform-v2] SQL transform support schema change
Jan 15, 2025
f34f675
Merge remote-tracking branch 'refs/remotes/upstream/dev' into transfo…
CosmosNi Jan 24, 2025
f20e995
[Feature][transform-v2] SQL transform support schema change
CosmosNi Jan 26, 2025
ec10c9e
Merge remote-tracking branch 'upstream/dev' into transform_sql_schema…
CosmosNi Feb 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[Feature][transform-v2] SQL transform support schema change
nijiahui authored and CosmosNi committed Jan 24, 2025
commit 482f43b38de1c1afac278cc6d0dd32125ed54a5a
Original file line number Diff line number Diff line change
@@ -48,13 +48,16 @@

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.stream.Collectors;

public class ZetaSQLEngine implements SQLEngine {
private static final Logger log = LoggerFactory.getLogger(ZetaSQLEngine.class);
public static final String ESCAPE_IDENTIFIER = "`";
public static Map<String, String> columnNameMapping = new HashMap<>();

private String inputTableName;
@Nullable private String catalogTableName;
@@ -185,23 +188,36 @@ public SeaTunnelRowType typeMapping(List<String> inputColumnsMapping) {
for (SelectItem selectItem : selectItems) {
if (selectItem.getExpression() instanceof AllColumns) {
for (int i = 0; i < inputRowType.getFieldNames().length; i++) {
fieldNames[idx] = cleanEscape(inputRowType.getFieldName(i));
String fieldName = cleanEscape(inputRowType.getFieldName(i));
fieldNames[idx] = fieldName;
seaTunnelDataTypes[idx] = inputRowType.getFieldType(i);
if (inputColumnsMapping != null) {
inputColumnsMapping.set(idx, inputRowType.getFieldName(i));
}
columnNameMapping.put(fieldName, fieldName);
idx++;
}
} else {
Expression expression = selectItem.getExpression();
if (selectItem.getAlias() != null) {
String aliasName = selectItem.getAlias().getName();
fieldNames[idx] = cleanEscape(aliasName);
String aliasName = cleanEscape(selectItem.getAlias().getName());
String fieldName;
fieldNames[idx] = aliasName;
if (expression instanceof Column) {
fieldName = cleanEscape(((Column) expression).getColumnName());
} else {
fieldName = cleanEscape(expression.toString());
}
columnNameMapping.put(fieldName, aliasName);
} else {
if (expression instanceof Column) {
fieldNames[idx] = cleanEscape(((Column) expression).getColumnName());
String fieldName = cleanEscape(((Column) expression).getColumnName());
fieldNames[idx] = fieldName;
columnNameMapping.put(fieldName, fieldName);
} else {
fieldNames[idx] = cleanEscape(expression.toString());
String fieldName = cleanEscape(expression.toString());
fieldNames[idx] = fieldName;
columnNameMapping.put(fieldName, fieldName);
}
}

@@ -228,21 +244,9 @@ public SeaTunnelRowType typeMapping(List<String> inputColumnsMapping) {

@Override
public String getChangeColumnName(String columnName) {
List<SelectItem<?>> selectItems = selectBody.getSelectItems();
for (SelectItem selectItem : selectItems) {
if (selectItem.getExpression() instanceof AllColumns) {
return columnName;
}
Expression expression = selectItem.getExpression();
if (columnName.equals(cleanEscape(((Column) expression).getColumnName()))
|| columnName.equals(cleanEscape(expression.toString()))) {
if (selectItem.getAlias() != null) {
String aliasName = selectItem.getAlias().getName();
return cleanEscape(aliasName);
}
}
if (columnNameMapping.containsKey(columnName)) {
return columnNameMapping.get(columnName);
}

return columnName;
}