Skip to content

Commit 4740bcd

Browse files
timothyw553timothyw553
authored
[Spark] Expand UC Delta ALTER TABLE integration tests (#6902)
#### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Follow-up to the UC Delta `ALTER TABLE` testing breadth requested in #6840. This PR expands `sparkuctest` coverage for UC Delta `ALTER TABLE` behavior by replacing the broad smoke coverage with focused integration tests. The suite now covers both supported ALTER operations and blocked operations, including checks that rejected ALTERs do not create a new Delta version or partially mutate UC Delta metadata. Added coverage includes: - column changes: ADD COLUMNS with primitive and complex types, MODIFY type, DROP, RENAME, quoted column rename, nested column changes, comments, and column position changes - table comments through `COMMENT ON TABLE` - table properties: custom property SET/UNSET, recognized `delta.autoOptimize.*` properties, protocol-derived no-op commits, unknown `delta.*` properties blocked as no-op failures, and external-table supported ALTER smoke coverage - table features: feature-backed properties such as CDF, direct feature-property enablement blocked, required feature drops blocked, and CCv2/catalog-managed upgrade and downgrade blocked - partition behavior: partition management commands blocked and dropping partition columns blocked without metadata/data mutation - constraints: ADD/DROP constraint behavior, constraint enforcement, and failed ADD CONSTRAINT leaving no constraint metadata behind - metadata ALTERs: SET LOCATION, SET SERDE, SET OWNER TO, and RENAME TABLE blocked without changing the existing table - tags: SET/UNSET TAGS blocked - cluster keys: CLUSTER BY, changing cluster keys, and CLUSTER BY NONE with UC Delta metadata validation ## How was this patch tested? ```bash ./build/sbt sparkUnityCatalog/javafmtAll ./build/sbt "sparkUnityCatalog/testOnly io.sparkuctest.UCDeltaTableAlterTest" git diff --check -- spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableAlterTest.java ``` The focused ALTER TABLE suite passed with 29 tests. ## Does this PR introduce _any_ user-facing changes? No. Test-only change.
1 parent 7d540cb commit 4740bcd

1 file changed

Lines changed: 152 additions & 14 deletions

File tree

‎spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableAlterTest.java‎

Lines changed: 152 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@
1818

1919
import static org.junit.jupiter.api.Assertions.assertEquals;
2020
import static org.junit.jupiter.api.Assertions.assertFalse;
21+
import static org.junit.jupiter.api.Assertions.assertNull;
2122

2223
import io.unitycatalog.client.delta.api.TablesApi;
2324
import io.unitycatalog.client.delta.model.LoadTableResponse;
2425
import io.unitycatalog.client.delta.model.StructField;
2526
import io.unitycatalog.client.delta.model.StructType;
2627
import io.unitycatalog.client.model.TableInfo;
2728
import java.util.List;
29+
import java.util.Map;
2830
import java.util.stream.Collectors;
2931
import org.apache.hadoop.fs.Path;
3032
import org.junit.jupiter.api.Test;
@@ -36,9 +38,9 @@ public class UCDeltaTableAlterTest extends UCDeltaTableIntegrationBaseTest {
3638
private static final String CHAR_VARCHAR_TYPE_METADATA_KEY = "__CHAR_VARCHAR_TYPE_STRING";
3739

3840
@Test
39-
public void testSimpleAlterTableOperationsUpdateUcMetadata() throws Exception {
41+
public void testAlterTableCustomPropertiesUpdateUcDeltaMetadata() throws Exception {
4042
withNewTable(
41-
"alter_simple_operations_test",
43+
"alter_custom_props_test",
4244
"id INT, name STRING",
4345
TableType.MANAGED,
4446
tableName -> {
@@ -51,13 +53,101 @@ public void testSimpleAlterTableOperationsUpdateUcMetadata() throws Exception {
5153

5254
response = loadTableViaDeltaRest(tableName);
5355
assertFalse(response.getMetadata().getProperties().containsKey("custom.key"));
56+
});
57+
}
58+
59+
@Test
60+
public void testAlterTableDeltaPropertiesUpdateUcDeltaMetadata() throws Exception {
61+
withNewTable(
62+
"alter_delta_props_test",
63+
"id INT, name STRING",
64+
TableType.MANAGED,
65+
tableName -> {
66+
sql(
67+
"ALTER TABLE %s SET TBLPROPERTIES ("
68+
+ "'delta.autoOptimize.optimizeWrite' = 'true', "
69+
+ "'delta.autoOptimize.autoCompact' = 'true')",
70+
tableName);
71+
72+
assertEquals("true", tableProperty(tableName, "delta.autoOptimize.optimizeWrite"));
73+
assertEquals("true", tableProperty(tableName, "delta.autoOptimize.autoCompact"));
5474

55-
sql("ALTER TABLE %s ADD COLUMNS (extra STRING)", tableName);
75+
LoadTableResponse response = loadTableViaDeltaRest(tableName);
76+
Map<String, String> properties = response.getMetadata().getProperties();
77+
assertEquals("true", properties.get("delta.autoOptimize.optimizeWrite"));
78+
assertEquals("true", properties.get("delta.autoOptimize.autoCompact"));
79+
80+
sql(
81+
"ALTER TABLE %s UNSET TBLPROPERTIES ("
82+
+ "'delta.autoOptimize.optimizeWrite', "
83+
+ "'delta.autoOptimize.autoCompact')",
84+
tableName);
85+
86+
assertNull(tableProperty(tableName, "delta.autoOptimize.optimizeWrite"));
87+
assertNull(tableProperty(tableName, "delta.autoOptimize.autoCompact"));
5688

5789
response = loadTableViaDeltaRest(tableName);
58-
assertEquals(3, response.getMetadata().getColumns().getFields().size());
59-
assertEquals("extra", response.getMetadata().getColumns().getFields().get(2).getName());
90+
properties = response.getMetadata().getProperties();
91+
assertFalse(properties.containsKey("delta.autoOptimize.optimizeWrite"));
92+
assertFalse(properties.containsKey("delta.autoOptimize.autoCompact"));
93+
});
94+
}
95+
96+
@Test
97+
public void testAlterTableProtocolDerivedPropertiesNoOpButCommit() throws Exception {
98+
withNewTable(
99+
"alter_protocol_props_test",
100+
"id INT, name STRING",
101+
TableType.MANAGED,
102+
tableName -> {
103+
assertEquals("supported", tableProperty(tableName, "delta.feature.catalogManaged"));
104+
long versionBeforeUnset = currentVersion(tableName);
105+
106+
sql("ALTER TABLE %s UNSET TBLPROPERTIES ('delta.feature.catalogManaged')", tableName);
107+
108+
assertEquals(versionBeforeUnset + 1, currentVersion(tableName));
109+
assertLatestHistoryOperation(tableName, versionBeforeUnset + 1, "UNSET TBLPROPERTIES");
110+
assertEquals("supported", tableProperty(tableName, "delta.feature.catalogManaged"));
111+
112+
assertEquals("3", tableProperty(tableName, "delta.minReaderVersion"));
113+
long versionBeforeSet = currentVersion(tableName);
114+
115+
sql("ALTER TABLE %s SET TBLPROPERTIES ('delta.minReaderVersion' = '2')", tableName);
116+
117+
assertEquals(versionBeforeSet + 1, currentVersion(tableName));
118+
assertLatestHistoryOperation(tableName, versionBeforeSet + 1, "SET TBLPROPERTIES");
119+
assertEquals("3", tableProperty(tableName, "delta.minReaderVersion"));
120+
});
121+
}
60122

123+
@Test
124+
public void testAlterTableAddColumnsWithDifferentTypesUpdatesUcDeltaMetadata() throws Exception {
125+
withNewTable(
126+
"alter_add_columns_types_test",
127+
"id INT, name STRING",
128+
TableType.MANAGED,
129+
tableName -> {
130+
sql(
131+
"ALTER TABLE %s ADD COLUMNS ("
132+
+ "price DECIMAL(10, 2), "
133+
+ "active BOOLEAN, "
134+
+ "created_at TIMESTAMP)",
135+
tableName);
136+
137+
LoadTableResponse response = loadTableViaDeltaRest(tableName);
138+
assertEquals(
139+
List.of("id", "name", "price", "active", "created_at"),
140+
fieldNames(response.getMetadata().getColumns()));
141+
});
142+
}
143+
144+
@Test
145+
public void testAlterTableColumnCommentUpdatesUcDeltaMetadata() throws Exception {
146+
withNewTable(
147+
"alter_column_comment_test",
148+
"id INT, name STRING",
149+
TableType.MANAGED,
150+
tableName -> {
61151
sql("ALTER TABLE %s CHANGE COLUMN name COMMENT 'display name'", tableName);
62152

63153
StructField name =
@@ -68,25 +158,58 @@ public void testSimpleAlterTableOperationsUpdateUcMetadata() throws Exception {
68158

69159
name = field(loadTableViaDeltaRest(tableName).getMetadata().getColumns(), "name");
70160
assertEquals("", name.getMetadata().get("comment"));
161+
});
162+
}
163+
164+
@Test
165+
public void testCommentOnTableUpdatesUcMetadata() throws Exception {
166+
withNewTable(
167+
"alter_table_comment_test",
168+
"id INT, name STRING",
169+
TableType.MANAGED,
170+
tableName -> {
171+
sql("COMMENT ON TABLE %s IS 'table comment'", tableName);
71172

173+
TableInfo tableInfo = loadTableInfoViaUc(tableName);
174+
assertEquals("table comment", tableInfo.getComment());
175+
176+
sql("COMMENT ON TABLE %s IS NULL", tableName);
177+
178+
tableInfo = loadTableInfoViaUc(tableName);
179+
assertEquals("", tableInfo.getComment());
180+
});
181+
}
182+
183+
@Test
184+
public void testAlterTableFeatureBackedPropertiesUpdateUcDeltaMetadata() throws Exception {
185+
withNewTable(
186+
"alter_feature_backed_props_test",
187+
"id INT, name STRING",
188+
TableType.MANAGED,
189+
tableName -> {
72190
sql(
73191
"ALTER TABLE %s SET TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')",
74192
tableName);
75193

76-
response = loadTableViaDeltaRest(tableName);
194+
LoadTableResponse response = loadTableViaDeltaRest(tableName);
77195
assertEquals(
78196
"true", response.getMetadata().getProperties().get("delta.enableChangeDataFeed"));
197+
});
198+
}
79199

200+
@Test
201+
public void testAlterTableClusterKeysUpdateUcDeltaMetadata() throws Exception {
202+
withNewTable(
203+
"alter_cluster_keys_test",
204+
"id INT, name STRING",
205+
TableType.MANAGED,
206+
tableName -> {
80207
sql("ALTER TABLE %s CLUSTER BY (id)", tableName);
81208

82-
response = loadTableViaDeltaRest(tableName);
83-
assertEquals(
84-
"supported", response.getMetadata().getProperties().get("delta.feature.clustering"));
85-
86-
sql("COMMENT ON TABLE %s IS 'table comment'", tableName);
87-
88-
TableInfo tableInfo = loadTableInfoViaUc(tableName);
89-
assertEquals("table comment", tableInfo.getComment());
209+
LoadTableResponse response = loadTableViaDeltaRest(tableName);
210+
Map<String, String> properties = response.getMetadata().getProperties();
211+
assertEquals("supported", properties.get("delta.feature.clustering"));
212+
assertEquals("[[\"id\"]]", tableProperty(tableName, "clusteringColumns"));
90213
});
91214
}
92215

@@ -374,6 +497,21 @@ private TableInfo loadTableInfoViaUc(String tableName) throws Exception {
374497
.getTable(tableName, false, false);
375498
}
376499

500+
private String tableProperty(String tableName, String key) {
501+
return sql("SHOW TBLPROPERTIES %s", tableName).stream()
502+
.filter(row -> row.size() >= 2 && key.equals(row.get(0)))
503+
.map(row -> row.get(1))
504+
.findFirst()
505+
.orElse(null);
506+
}
507+
508+
private void assertLatestHistoryOperation(
509+
String tableName, long expectedVersion, String expectedOperation) {
510+
List<List<String>> history = sql("DESCRIBE HISTORY %s LIMIT 1", tableName);
511+
assertEquals(String.valueOf(expectedVersion), history.get(0).get(0));
512+
assertEquals(expectedOperation, history.get(0).get(4));
513+
}
514+
377515
private static List<String> fieldNames(StructType structType) {
378516
return structType.getFields().stream().map(StructField::getName).collect(Collectors.toList());
379517
}

0 commit comments

Comments
 (0)