Skip to content

Commit c063436

Browse files
JingsongLiJingsongLi
committed
fix
1 parent f98470b commit c063436

File tree

9 files changed

+141
-10
lines changed

9 files changed

+141
-10
lines changed

‎docs/content/spark/default-value.md‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,26 @@ CREATE TABLE my_table (
4646
For SQL commands that execute table writes, such as the `INSERT`, `UPDATE`, and `MERGE` commands, the `DEFAULT` keyword
4747
or `NULL` value is parsed into the default value specified for the corresponding column.
4848

49-
## Limitation
49+
## Alter Default Value
50+
51+
Paimon supports alter column default value.
52+
53+
For example:
54+
55+
```sql
56+
CREATE TABLE T (a INT, b INT DEFAULT 2);
57+
58+
INSERT INTO T (a) VALUES (1);
59+
-- result: [[1, 2]]
5060

51-
Currently, only specifying default values when creating tables is supported, and the following usage is not supported:
61+
ALTER TABLE T ALTER COLUMN b SET DEFAULT 3;
62+
63+
INSERT INTO T (a) VALUES (2);
64+
-- result: [[1, 2], [2, 3]]
65+
```
66+
67+
The default value of `'b'` column has been changed to 3 from 2.
68+
69+
## Limitation
5270

53-
1. Not support alter table add column with default value, for example: `ALTER TABLE T ADD COLUMN d INT DEFAULT 5;`.
54-
2. Not support alter table alter column set default value, for example: `ALTER TABLE T ALTER COLUMN d SET DEFAULT 5;`.
71+
Not support alter table add column with default value, for example: `ALTER TABLE T ADD COLUMN d INT DEFAULT 5;`.

‎docs/static/rest-catalog-open-api.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2233,6 +2233,7 @@ components:
22332233
- $ref: '#/components/schemas/RenameColumn'
22342234
- $ref: '#/components/schemas/DropColumn'
22352235
- $ref: '#/components/schemas/UpdateColumnComment'
2236+
- $ref: '#/components/schemas/UpdateColumnDefaultValue'
22362237
- $ref: '#/components/schemas/UpdateColumnType'
22372238
- $ref: '#/components/schemas/UpdateColumnPosition'
22382239
- $ref: '#/components/schemas/UpdateColumnNullability'
@@ -2247,6 +2248,7 @@ components:
22472248
renameColumn: '#/components/schemas/RenameColumn'
22482249
dropColumn: '#/components/schemas/DropColumn'
22492250
updateColumnComment: '#/components/schemas/UpdateColumnComment'
2251+
updateColumnDefaultValue: '#/components/schemas/UpdateColumnDefaultValue'
22502252
updateColumnType: '#/components/schemas/UpdateColumnType'
22512253
updateColumnPosition: '#/components/schemas/UpdateColumnPosition'
22522254
updateColumnNullability: '#/components/schemas/UpdateColumnNullability'
@@ -2339,6 +2341,19 @@ components:
23392341
type: string
23402342
newComment:
23412343
type: string
2344+
UpdateColumnDefaultValue:
2345+
allOf:
2346+
- $ref: '#/components/schemas/BaseSchemaChange'
2347+
properties:
2348+
action:
2349+
type: string
2350+
const: "updateColumnDefaultValue"
2351+
fieldNames:
2352+
type: array
2353+
items:
2354+
type: string
2355+
newDefaultValue:
2356+
type: string
23422357
UpdateColumnType:
23432358
allOf:
23442359
- $ref: '#/components/schemas/BaseSchemaChange'

‎paimon-api/src/main/java/org/apache/paimon/schema/SchemaChange.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@
7272
@JsonSubTypes.Type(
7373
value = SchemaChange.UpdateColumnComment.class,
7474
name = SchemaChange.Actions.UPDATE_COLUMN_COMMENT_ACTION),
75+
@JsonSubTypes.Type(
76+
value = SchemaChange.UpdateColumnDefaultValue.class,
77+
name = SchemaChange.Actions.UPDATE_COLUMN_DEFAULT_VALUE_ACTION),
7578
@JsonSubTypes.Type(
7679
value = SchemaChange.UpdateColumnPosition.class,
7780
name = SchemaChange.Actions.UPDATE_COLUMN_POSITION_ACTION),
@@ -153,6 +156,10 @@ static SchemaChange updateColumnComment(String[] fieldNames, String comment) {
153156
return new UpdateColumnComment(fieldNames, comment);
154157
}
155158

159+
static SchemaChange updateColumnDefaultValue(String[] fieldNames, String defaultValue) {
160+
return new UpdateColumnDefaultValue(fieldNames, defaultValue);
161+
}
162+
156163
static SchemaChange updateColumnPosition(Move move) {
157164
return new UpdateColumnPosition(move);
158165
}
@@ -751,6 +758,58 @@ public int hashCode() {
751758
}
752759
}
753760

761+
/** A SchemaChange to update the default value. */
762+
final class UpdateColumnDefaultValue implements SchemaChange {
763+
764+
private static final long serialVersionUID = 1L;
765+
private static final String FIELD_FILED_NAMES = "fieldNames";
766+
private static final String FIELD_NEW_DEFAULT_VALUE = "newDefaultValue";
767+
768+
@JsonProperty(FIELD_FILED_NAMES)
769+
private final String[] fieldNames;
770+
771+
@JsonProperty(FIELD_NEW_DEFAULT_VALUE)
772+
private final String newDefaultValue;
773+
774+
@JsonCreator
775+
private UpdateColumnDefaultValue(
776+
@JsonProperty(FIELD_FILED_NAMES) String[] fieldNames,
777+
@JsonProperty(FIELD_NEW_DEFAULT_VALUE) String newDefaultValue) {
778+
this.fieldNames = fieldNames;
779+
this.newDefaultValue = newDefaultValue;
780+
}
781+
782+
@JsonGetter(FIELD_FILED_NAMES)
783+
public String[] fieldNames() {
784+
return fieldNames;
785+
}
786+
787+
@JsonGetter(FIELD_NEW_DEFAULT_VALUE)
788+
public String newDefaultValue() {
789+
return newDefaultValue;
790+
}
791+
792+
@Override
793+
public boolean equals(Object o) {
794+
if (this == o) {
795+
return true;
796+
}
797+
if (o == null || getClass() != o.getClass()) {
798+
return false;
799+
}
800+
UpdateColumnDefaultValue that = (UpdateColumnDefaultValue) o;
801+
return Arrays.equals(fieldNames, that.fieldNames)
802+
&& newDefaultValue.equals(that.newDefaultValue);
803+
}
804+
805+
@Override
806+
public int hashCode() {
807+
int result = Objects.hash(newDefaultValue);
808+
result = 31 * result + Objects.hashCode(fieldNames);
809+
return result;
810+
}
811+
}
812+
754813
/** Actions for schema changes： identify for schema change. */
755814
class Actions {
756815
public static final String FIELD_ACTION = "action";
@@ -763,6 +822,7 @@ class Actions {
763822
public static final String UPDATE_COLUMN_TYPE_ACTION = "updateColumnType";
764823
public static final String UPDATE_COLUMN_NULLABILITY_ACTION = "updateColumnNullability";
765824
public static final String UPDATE_COLUMN_COMMENT_ACTION = "updateColumnComment";
825+
public static final String UPDATE_COLUMN_DEFAULT_VALUE_ACTION = "updateColumnDefaultValue";
766826
public static final String UPDATE_COLUMN_POSITION_ACTION = "updateColumnPosition";
767827

768828
private Actions() {}

‎paimon-api/src/main/java/org/apache/paimon/types/DataField.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.paimon.types;
2020

2121
import org.apache.paimon.annotation.Public;
22+
import org.apache.paimon.utils.StringUtils;
2223

2324
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonGenerator;
2425

@@ -121,7 +122,7 @@ public DataField copy(boolean isNullable) {
121122
public String asSQLString() {
122123
StringBuilder sb = new StringBuilder();
123124
sb.append(escapeIdentifier(name)).append(" ").append(type.asSQLString());
124-
if (description != null) {
125+
if (StringUtils.isNotEmpty(description)) {
125126
sb.append(" COMMENT '").append(escapeSingleQuotes(description)).append("'");
126127
}
127128
if (defaultValue != null) {

‎paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.paimon.schema.SchemaChange.RenameColumn;
3232
import org.apache.paimon.schema.SchemaChange.SetOption;
3333
import org.apache.paimon.schema.SchemaChange.UpdateColumnComment;
34+
import org.apache.paimon.schema.SchemaChange.UpdateColumnDefaultValue;
3435
import org.apache.paimon.schema.SchemaChange.UpdateColumnNullability;
3536
import org.apache.paimon.schema.SchemaChange.UpdateColumnPosition;
3637
import org.apache.paimon.schema.SchemaChange.UpdateColumnType;
@@ -515,6 +516,18 @@ protected void updateLastColumn(
515516
UpdateColumnPosition update = (UpdateColumnPosition) change;
516517
SchemaChange.Move move = update.move();
517518
applyMove(newFields, move);
519+
} else if (change instanceof UpdateColumnDefaultValue) {
520+
UpdateColumnDefaultValue update = (UpdateColumnDefaultValue) change;
521+
updateNestedColumn(
522+
newFields,
523+
update.fieldNames(),
524+
(field, depth) ->
525+
new DataField(
526+
field.id(),
527+
field.name(),
528+
field.type(),
529+
field.description(),
530+
update.newDefaultValue()));
518531
} else {
519532
throw new UnsupportedOperationException("Unsupported change: " + change.getClass());
520533
}

‎paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -750,8 +750,8 @@ public void testIncompatibleMySqlTable() {
750750
anyCauseMatches(
751751
IllegalArgumentException.class,
752752
"Column v1 have different types when merging schemas.\n"
753-
+ "Current table '{paimon_sync_table.incompatible_field_1}' field: `v1` TIMESTAMP(0) ''\n"
754-
+ "To be merged table 'paimon_sync_table.incompatible_field_2' field: `v1` INT ''"));
753+
+ "Current table '{paimon_sync_table.incompatible_field_1}' field: `v1` TIMESTAMP(0)\n"
754+
+ "To be merged table 'paimon_sync_table.incompatible_field_2' field: `v1` INT"));
755755
}
756756

757757
@Test

‎paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,10 @@
9292
import static org.apache.paimon.spark.util.OptionUtils.copyWithSQLConf;
9393
import static org.apache.paimon.spark.utils.CatalogUtils.checkNamespace;
9494
import static org.apache.paimon.spark.utils.CatalogUtils.checkNoDefaultValue;
95+
import static org.apache.paimon.spark.utils.CatalogUtils.isUpdateColumnDefaultValue;
9596
import static org.apache.paimon.spark.utils.CatalogUtils.removeCatalogName;
9697
import static org.apache.paimon.spark.utils.CatalogUtils.toIdentifier;
98+
import static org.apache.paimon.spark.utils.CatalogUtils.toUpdateColumnDefaultValue;
9799
import static org.apache.spark.sql.connector.catalog.TableCatalogCapability.SUPPORT_COLUMN_DEFAULT_VALUE;
98100

99101
/** Spark {@link TableCatalog} for paimon. */
@@ -390,6 +392,8 @@ private SchemaChange toSchemaChange(TableChange change) {
390392
TableChange.UpdateColumnPosition update = (TableChange.UpdateColumnPosition) change;
391393
SchemaChange.Move move = getMove(update.position(), update.fieldNames());
392394
return SchemaChange.updateColumnPosition(move);
395+
} else if (isUpdateColumnDefaultValue(change)) {
396+
return toUpdateColumnDefaultValue(change);
393397
} else {
394398
throw new UnsupportedOperationException(
395399
"Change is not supported: " + change.getClass());

‎paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/utils/CatalogUtils.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.paimon.spark.utils;
2020

21+
import org.apache.paimon.schema.SchemaChange;
2122
import org.apache.paimon.types.ArrayType;
2223
import org.apache.paimon.types.DataType;
2324
import org.apache.paimon.types.DecimalType;
@@ -263,4 +264,18 @@ public static void checkNoDefaultValue(TableChange.AddColumn addColumn) {
263264
} catch (NoClassDefFoundError ignored) {
264265
}
265266
}
267+
268+
public static boolean isUpdateColumnDefaultValue(TableChange tableChange) {
269+
try {
270+
return tableChange instanceof TableChange.UpdateColumnDefaultValue;
271+
} catch (NoClassDefFoundError ignored) {
272+
return false;
273+
}
274+
}
275+
276+
public static SchemaChange toUpdateColumnDefaultValue(TableChange tableChange) {
277+
TableChange.UpdateColumnDefaultValue update =
278+
(TableChange.UpdateColumnDefaultValue) tableChange;
279+
return SchemaChange.updateColumnDefaultValue(update.fieldNames(), update.newDefaultValue());
280+
}
266281
}

‎paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,19 +103,25 @@ public void testWriteWithDefaultValue() {
103103
rows = spark.sql("SELECT * FROM T").collectAsList();
104104
assertThat(rows.toString()).isEqualTo("[[1,2,my_value], [2,2,my_value], [3,2,my_value]]");
105105

106-
// test alter with DEFAULT not support
106+
// test add column with DEFAULT not support
107107
assertThatThrownBy(() -> spark.sql("ALTER TABLE T ADD COLUMN d INT DEFAULT 5"))
108108
.hasMessageContaining(
109109
"Unsupported table change: Cannot add column [d] with default value");
110-
assertThatThrownBy(() -> spark.sql("ALTER TABLE T ALTER COLUMN a SET DEFAULT 3"))
111-
.hasMessageContaining("Change is not supported");
112110

113111
// test alter type to default column
114112
spark.sql("ALTER TABLE T ALTER COLUMN b TYPE STRING").collectAsList();
115113
spark.sql("INSERT INTO T (a) VALUES (4)").collectAsList();
116114
rows = spark.sql("SELECT * FROM T").collectAsList();
117115
assertThat(rows.toString())
118116
.isEqualTo("[[1,2,my_value], [2,2,my_value], [3,2,my_value], [4,2,my_value]]");
117+
118+
// test alter default column
119+
spark.sql("ALTER TABLE T ALTER COLUMN b SET DEFAULT '3'");
120+
spark.sql("INSERT INTO T (a) VALUES (5)").collectAsList();
121+
rows = spark.sql("SELECT * FROM T").collectAsList();
122+
assertThat(rows.toString())
123+
.isEqualTo(
124+
"[[1,2,my_value], [2,2,my_value], [3,2,my_value], [4,2,my_value], [5,3,my_value]]");
119125
}
120126

121127
@Test

0 commit comments

Comments
 (0)