Skip to content

Commit 5abb1cb

Browse files
mergify[bot]meegoocursoragent
authored
Revert [BugFix] Hide expression partition generated columns in DESC/SHOW CREATE TABLE (backport #69793) (#70037) (#70901)
Co-authored-by: meegoo <meegoo.sr@gmail.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: meegoo <meegoo@users.noreply.github.com>
1 parent 1c593ad commit 5abb1cb

11 files changed

Lines changed: 37 additions & 260 deletions

File tree

‎fe/fe-core/src/main/java/com/starrocks/common/proc/IndexSchemaProcNode.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import com.google.common.collect.Lists;
4040
import com.starrocks.catalog.Column;
4141
import com.starrocks.common.AnalysisException;
42-
import com.starrocks.common.FeConstants;
4342
import com.starrocks.common.SchemaConstants;
4443
import org.apache.commons.lang.StringUtils;
4544

@@ -79,10 +78,6 @@ public ProcResult fetchResult() throws AnalysisException {
7978
result.setNames(TITLE_NAMES);
8079

8180
for (Column column : schema) {
82-
// Filter out expression partition generated columns in DESC
83-
if (column.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) {
84-
continue;
85-
}
8681
// Extra string (aggregation and bloom filter)
8782
List<String> extras = Lists.newArrayList();
8883
if (column.getAggregationType() != null && !isHideAggregateTypeName) {

‎fe/fe-core/src/main/java/com/starrocks/service/FrontendServiceImpl.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@
9191
import com.starrocks.common.ConfigBase;
9292
import com.starrocks.common.DdlException;
9393
import com.starrocks.common.DuplicatedRequestException;
94-
import com.starrocks.common.FeConstants;
9594
import com.starrocks.common.IdGenerator;
9695
import com.starrocks.common.LabelAlreadyUsedException;
9796
import com.starrocks.common.MetaNotFoundException;
@@ -979,11 +978,6 @@ private boolean setColumnDesc(List<TColumnDef> columns, Table table, long limit,
979978
tableKeysType = olapTable.getKeysType().name().substring(0, 3).toUpperCase();
980979
}
981980
for (Column column : table.getBaseSchema()) {
982-
// Filter out expression partition generated columns in DESC and information_schema.columns.
983-
// SHOW CREATE TABLE also filters them in AstToStringBuilder to display user-created DDL.
984-
if (column.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) {
985-
continue;
986-
}
987981
final TColumnDesc desc =
988982
new TColumnDesc(column.getName(), column.getPrimitiveType().toThrift());
989983
final Integer precision = column.getType().getPrecision();

‎fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AstToStringBuilder.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@
7373
import com.starrocks.catalog.Index;
7474
import com.starrocks.catalog.JDBCTable;
7575
import com.starrocks.catalog.KeysType;
76-
import com.starrocks.catalog.ListPartitionInfo;
7776
import com.starrocks.catalog.MaterializedIndexMeta;
7877
import com.starrocks.catalog.MaterializedView;
7978
import com.starrocks.catalog.MysqlTable;
@@ -85,7 +84,6 @@
8584
import com.starrocks.catalog.RangePartitionInfo;
8685
import com.starrocks.catalog.Table;
8786
import com.starrocks.catalog.View;
88-
import com.starrocks.common.FeConstants;
8987
import com.starrocks.common.Pair;
9088
import com.starrocks.common.util.ParseUtil;
9189
import com.starrocks.common.util.PrintableMap;
@@ -1701,10 +1699,6 @@ public static void getDdlStmt(String dbName, Table table, List<String> createTab
17011699
sb.append("`").append(table.getName()).append("` (\n");
17021700
int idx = 0;
17031701
for (Column column : table.getBaseSchema()) {
1704-
// Skip expression partition generated columns to show user-created DDL
1705-
if (column.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) {
1706-
continue;
1707-
}
17081702
if (idx++ != 0) {
17091703
sb.append(",\n");
17101704
}
@@ -1755,14 +1749,7 @@ public static void getDdlStmt(String dbName, Table table, List<String> createTab
17551749
partitionId = Lists.newArrayList();
17561750
}
17571751
if (partitionInfo.isRangePartition() || partitionInfo.getType() == PartitionType.LIST) {
1758-
// Use expression (not generated column name) for user-created DDL display
1759-
if (partitionInfo instanceof ListPartitionInfo) {
1760-
ListPartitionInfo listPartitionInfo = (ListPartitionInfo) partitionInfo;
1761-
sb.append("\n").append(listPartitionInfo.toSql(olapTable,
1762-
listPartitionInfo.isAutomaticPartition(), false));
1763-
} else {
1764-
sb.append("\n").append(partitionInfo.toSql(olapTable, partitionId));
1765-
}
1752+
sb.append("\n").append(partitionInfo.toSql(olapTable, partitionId));
17661753
}
17671754

17681755
// distribution

‎fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
import com.starrocks.common.DdlException;
4141
import com.starrocks.common.ErrorCode;
4242
import com.starrocks.common.ErrorReport;
43-
import com.starrocks.common.FeConstants;
4443
import com.starrocks.common.SchemaConstants;
4544
import com.starrocks.common.proc.ExternalTableProcDir;
4645
import com.starrocks.common.proc.PartitionsProcDir;
@@ -413,9 +412,6 @@ private void descInternalCatalogTable(DescribeStmt node, ConnectContext context)
413412
if (olapTable.getIndexNameById(mvMeta.getIndexId()).equalsIgnoreCase(node.getTableName())) {
414413
List<Column> columns = olapTable.getIndexIdToSchema().get(mvMeta.getIndexId());
415414
for (Column column : columns) {
416-
if (column.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) {
417-
continue;
418-
}
419415
// Extra string (aggregation and bloom filter)
420416
List<String> extras = Lists.newArrayList();
421417
if (column.getAggregationType() != null &&
@@ -490,9 +486,7 @@ private void descInternalCatalogTable(DescribeStmt node, ConnectContext context)
490486
MaterializedIndexMeta indexMeta = olapTable.getIndexMetaByIndexId(indexId);
491487
for (int j = 0; j < columns.size(); ++j) {
492488
Column column = columns.get(j);
493-
if (column.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) {
494-
continue;
495-
}
489+
496490
// Extra string (aggregation and bloom filter)
497491
List<String> extras = Lists.newArrayList();
498492
if (column.getAggregationType() != null &&

‎fe/fe-core/src/main/java/com/starrocks/sql/common/MetaUtils.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import com.starrocks.catalog.Table;
2929
import com.starrocks.common.ErrorCode;
3030
import com.starrocks.common.ErrorReport;
31-
import com.starrocks.common.FeConstants;
3231
import com.starrocks.common.util.DebugUtil;
3332
import com.starrocks.common.util.concurrent.lock.LockType;
3433
import com.starrocks.common.util.concurrent.lock.Locker;
@@ -285,15 +284,8 @@ public static List<ColumnId> getColumnIdsByColumnNames(Table table, List<String>
285284
return columnIds;
286285
}
287286

288-
/**
289-
* Convert partition column to SQL representation for SHOW CREATE TABLE etc.
290-
* Only internal expression partition columns (__generated_partition_column_*) are expanded to
291-
* their expression form; user-defined generated columns retain their column names to preserve
292-
* DDL round-trippability.
293-
*/
294287
public static String getPartitionColumnToSql(Column column) {
295-
if (column.isGeneratedColumn()
296-
&& column.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) {
288+
if (column.isGeneratedColumn()) {
297289
return column.generatedColumnExprToString();
298290
} else {
299291
return "`" + column.getName() + "`";

‎fe/fe-core/src/test/java/com/starrocks/service/FrontendServiceImplTest.java‎

Lines changed: 1 addition & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -256,18 +256,6 @@ public static void beforeClass() throws Exception {
256256
"PROPERTIES (\n" +
257257
"\"replication_num\" = \"1\"\n" +
258258
");")
259-
.withTable("CREATE TABLE site_access_multi_expr (\n" +
260-
" event_day DATETIME,\n" +
261-
" site_id INT DEFAULT '10',\n" +
262-
" city_code VARCHAR(100),\n" +
263-
" pv BIGINT DEFAULT '0'\n" +
264-
")\n" +
265-
"DUPLICATE KEY(event_day, site_id, city_code)\n" +
266-
"PARTITION BY site_id, date_trunc('day', event_day)\n" +
267-
"DISTRIBUTED BY HASH(event_day, site_id)\n" +
268-
"PROPERTIES (\n" +
269-
"\"replication_num\" = \"1\"\n" +
270-
");")
271259
.withTable("CREATE TABLE site_access_day (\n" +
272260
" event_day DATE,\n" +
273261
" site_id INT DEFAULT '10',\n" +
@@ -821,7 +809,7 @@ public void testGetTableNames() throws TException {
821809
params.setCurrent_user_ident(tUserIdentity);
822810

823811
TGetTablesResult result = impl.getTableNames(params);
824-
Assertions.assertEquals(19, result.tables.size());
812+
Assertions.assertEquals(18, result.tables.size());
825813
}
826814

827815
@Test
@@ -1058,82 +1046,6 @@ public void testDefaultValueMeta() throws Exception {
10581046
Assertions.assertEquals("2", testDefaultValue.get(1).getColumnDesc().getColumnDefault());
10591047
}
10601048

1061-
@Test
1062-
public void testDescribeTableAndShowCreateTableExpressionPartition() throws Exception {
1063-
// site_access_hour: PARTITION BY date_trunc - ExpressionRangePartitionInfo, no generated column in schema.
1064-
// Verify DESC never shows __generated_partition_column_, SHOW CREATE TABLE shows partition expr.
1065-
starRocksAssert.useDatabase("test");
1066-
FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
1067-
TDescribeTableParams request = new TDescribeTableParams();
1068-
request.setDb("test");
1069-
request.setTable_name("site_access_hour");
1070-
TUserIdentity userIdentity = new TUserIdentity();
1071-
userIdentity.setUsername("root");
1072-
userIdentity.setHost("%");
1073-
userIdentity.setIs_domain(false);
1074-
request.setCurrent_user_ident(userIdentity);
1075-
1076-
TDescribeTableResult response = impl.describeTable(request);
1077-
List<String> columnNames = response.getColumns().stream()
1078-
.map(c -> c.getColumnDesc().getColumnName())
1079-
.collect(Collectors.toList());
1080-
Assertions.assertFalse(columnNames.stream().anyMatch(
1081-
name -> name.startsWith(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)),
1082-
"DESC should not contain expression partition generated columns: " + columnNames);
1083-
Assertions.assertTrue(columnNames.contains("event_day"));
1084-
1085-
// SHOW CREATE TABLE: show user-created form with partition expression, not generated column
1086-
com.starrocks.sql.ast.ShowCreateTableStmt stmt =
1087-
(com.starrocks.sql.ast.ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(
1088-
"show create table site_access_hour", connectContext);
1089-
String createTableSql = GlobalStateMgr.getCurrentState().getShowExecutor().execute(stmt, connectContext)
1090-
.getResultRows().get(0).get(1);
1091-
Assertions.assertFalse(createTableSql.contains(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX),
1092-
"SHOW CREATE TABLE should not contain internal generated partition columns: " + createTableSql);
1093-
Assertions.assertTrue(createTableSql.contains("date_trunc"),
1094-
"SHOW CREATE TABLE should show partition expression: " + createTableSql);
1095-
Assertions.assertTrue(
1096-
java.util.regex.Pattern.compile("PARTITION BY\\s+date_trunc\\s*\\([^)]*event_day[^)]*\\)")
1097-
.matcher(createTableSql).find(),
1098-
"SHOW CREATE TABLE partition clause should explicitly use expression form: " + createTableSql);
1099-
}
1100-
1101-
@Test
1102-
public void testDescribeTableHidesGeneratedPartitionColumnsMultiExpr() throws Exception {
1103-
starRocksAssert.useDatabase("test");
1104-
FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv);
1105-
TDescribeTableParams request = new TDescribeTableParams();
1106-
request.setDb("test");
1107-
request.setTable_name("site_access_multi_expr");
1108-
TUserIdentity userIdentity = new TUserIdentity();
1109-
userIdentity.setUsername("root");
1110-
userIdentity.setHost("%");
1111-
userIdentity.setIs_domain(false);
1112-
request.setCurrent_user_ident(userIdentity);
1113-
1114-
TDescribeTableResult response = impl.describeTable(request);
1115-
List<String> columnNames = response.getColumns().stream()
1116-
.map(c -> c.getColumnDesc().getColumnName())
1117-
.collect(Collectors.toList());
1118-
Assertions.assertFalse(columnNames.stream().anyMatch(
1119-
name -> name.startsWith(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)),
1120-
"DESC should not show generated partition columns: " + columnNames);
1121-
Assertions.assertTrue(columnNames.contains("event_day"));
1122-
Assertions.assertTrue(columnNames.contains("site_id"));
1123-
1124-
com.starrocks.sql.ast.ShowCreateTableStmt showStmt =
1125-
(com.starrocks.sql.ast.ShowCreateTableStmt) UtFrameUtils.parseStmtWithNewParser(
1126-
"show create table site_access_multi_expr", connectContext);
1127-
String ddl = GlobalStateMgr.getCurrentState().getShowExecutor().execute(showStmt, connectContext)
1128-
.getResultRows().get(0).get(1);
1129-
Assertions.assertFalse(ddl.contains(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX),
1130-
"SHOW CREATE TABLE should not contain generated partition columns: " + ddl);
1131-
Assertions.assertTrue(ddl.contains("date_trunc"),
1132-
"SHOW CREATE TABLE should show partition expression: " + ddl);
1133-
Assertions.assertFalse(ddl.contains("__generated_partition_column"),
1134-
"Column definitions should not include generated partition columns: " + ddl);
1135-
}
1136-
11371049
@Test
11381050
public void testGetSpecialColumn() throws Exception {
11391051
starRocksAssert.withDatabase("test_table").useDatabase("test_table")

‎fe/fe-core/src/test/java/com/starrocks/sql/ast/DescribeStmtTest.java‎

Lines changed: 1 addition & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616

1717
import com.google.common.collect.Maps;
1818
import com.starrocks.catalog.Column;
19-
import com.starrocks.common.FeConstants;
2019
import com.starrocks.qe.ConnectContext;
2120
import com.starrocks.qe.ShowExecutor;
2221
import com.starrocks.qe.ShowResultSet;
@@ -69,18 +68,7 @@ public static void beforeClass() throws Exception {
6968
" AS\n" +
7069
"SELECT store_id, SUM(sale_amt) as sale_amt\n" +
7170
"FROM sales_records\n" +
72-
"GROUP BY store_id;")
73-
.withTable("CREATE TABLE expr_part_tbl (\n" +
74-
" event_day DATETIME,\n" +
75-
" site_id INT,\n" +
76-
" pv BIGINT DEFAULT '0'\n" +
77-
")\n" +
78-
"DUPLICATE KEY(event_day, site_id)\n" +
79-
"PARTITION BY site_id, date_trunc('day', event_day)\n" +
80-
"DISTRIBUTED BY HASH(event_day, site_id)\n" +
81-
"PROPERTIES (\n" +
82-
"\"replication_num\" = \"1\"\n" +
83-
");");
71+
"GROUP BY store_id;");
8472
}
8573

8674
@AfterAll
@@ -250,39 +238,6 @@ public void testDescAsyncMvAll() throws Exception {
250238
Assertions.assertEquals("YES", resultRows.get(1).get(4));
251239
}
252240

253-
@Test
254-
public void testDescExprPartitionTableHidesGeneratedColumns() throws Exception {
255-
String sql = "desc expr_part_tbl";
256-
DescribeStmt describeStmt = (DescribeStmt) UtFrameUtils.parseStmtWithNewParser(sql,
257-
starRocksAssert.getCtx());
258-
ShowResultSet result = ShowExecutor.execute(describeStmt, connectContext);
259-
List<List<String>> rows = result.getResultRows();
260-
for (List<String> row : rows) {
261-
Assertions.assertFalse(row.get(0).startsWith(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX),
262-
"DESC should not show generated partition column: " + row.get(0));
263-
}
264-
Assertions.assertEquals(3, rows.size());
265-
Assertions.assertEquals("event_day", rows.get(0).get(0));
266-
Assertions.assertEquals("site_id", rows.get(1).get(0));
267-
Assertions.assertEquals("pv", rows.get(2).get(0));
268-
}
269-
270-
@Test
271-
public void testDescAllExprPartitionTableHidesGeneratedColumns() throws Exception {
272-
String sql = "desc expr_part_tbl all";
273-
DescribeStmt describeStmt = (DescribeStmt) UtFrameUtils.parseStmtWithNewParser(sql,
274-
starRocksAssert.getCtx());
275-
ShowResultSet result = ShowExecutor.execute(describeStmt, connectContext);
276-
List<List<String>> rows = result.getResultRows();
277-
for (List<String> row : rows) {
278-
String fieldName = row.get(2);
279-
if (fieldName != null && !fieldName.isEmpty()) {
280-
Assertions.assertFalse(fieldName.startsWith(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX),
281-
"DESC ALL should not show generated partition column: " + fieldName);
282-
}
283-
}
284-
}
285-
286241
@Test
287242
public void testDescFilesMask() throws Exception {
288243
Map<String, String> properties = Maps.newHashMap();

‎test/sql/test_automatic_partition/R/test_multi_expr‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ multi_level_expr_par_tbl_1 CREATE TABLE `multi_level_expr_par_tbl_1` (
408408
`__generated_partition_column_0` varchar(1048576) NULL AS from_unixtime(c1) COMMENT ""
409409
) ENGINE=OLAP
410410
PRIMARY KEY(`c1`)
411-
PARTITION BY (from_unixtime(c1))
411+
PARTITION BY (`__generated_partition_column_0`)
412412
DISTRIBUTED BY HASH(`c1`) BUCKETS 3
413413
PROPERTIES (
414414
"compression" = "LZ4",
@@ -459,11 +459,13 @@ multi_level_expr_par_tbl_2 CREATE TABLE `multi_level_expr_par_tbl_2` (
459459
`k10` largeint(40) NULL COMMENT "",
460460
`k11` decimal(38, 9) NULL COMMENT "",
461461
`k12` decimal(38, 9) NULL COMMENT "",
462-
`k13` decimal(27, 9) NULL COMMENT ""
462+
`k13` decimal(27, 9) NULL COMMENT "",
463+
`__generated_partition_column_0` varchar(1048576) NULL AS substring(k4, 1, 5) COMMENT "",
464+
`__generated_partition_column_1` date NULL AS date_trunc('month', k1) COMMENT ""
463465
) ENGINE=OLAP
464466
PRIMARY KEY(`k1`, `k2`, `k3`, `k4`, `k5`)
465467
COMMENT "OLAP"
466-
PARTITION BY (substring(k4, 1, 5),`k3`,date_trunc('month', k1))
468+
PARTITION BY (`__generated_partition_column_0`,`k3`,`__generated_partition_column_1`)
467469
DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3
468470
PROPERTIES (
469471
"compression" = "LZ4",
@@ -597,11 +599,13 @@ base_tbl CREATE TABLE `base_tbl` (
597599
`k10` largeint(40) NULL COMMENT "",
598600
`k11` float NULL COMMENT "",
599601
`k12` double NULL COMMENT "",
600-
`k13` decimal(27, 9) NULL COMMENT ""
602+
`k13` decimal(27, 9) NULL COMMENT "",
603+
`__generated_partition_column_0` varchar(1048576) NULL AS substring(`k4`, 1, 5) COMMENT "",
604+
`__generated_partition_column_1` date NULL AS date_trunc('month', `k1`) COMMENT ""
601605
) ENGINE=OLAP
602606
DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`)
603607
COMMENT "OLAP"
604-
PARTITION BY (substring(k4, 1, 5),`k3`,date_trunc('month', k1))
608+
PARTITION BY (`__generated_partition_column_0`,`k3`,`__generated_partition_column_1`)
605609
DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3
606610
PROPERTIES (
607611
"compression" = "LZ4",
@@ -628,16 +632,18 @@ t1 CREATE TABLE `t1` (
628632
`k10` largeint(40) NULL COMMENT "",
629633
`k11` float NULL COMMENT "",
630634
`k12` double NULL COMMENT "",
631-
`k13` decimal(27, 9) NULL COMMENT ""
635+
`k13` decimal(27, 9) NULL COMMENT "",
636+
`__generated_partition_column_0` varchar(1048576) NULL AS substring(`k4`, 1, 5) COMMENT "",
637+
`__generated_partition_column_1` date NULL AS date_trunc('month', `k1`) COMMENT ""
632638
) ENGINE=OLAP
633639
DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`)
634640
COMMENT "OLAP"
635-
PARTITION BY (substring(k4, 1, 5),`k3`,date_trunc('month', k1))
641+
PARTITION BY (`__generated_partition_column_0`,`k3`,`__generated_partition_column_1`)
636642
DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3
637643
PROPERTIES (
638644
"compression" = "LZ4",
639645
"fast_schema_evolution" = "true",
640646
"replicated_storage" = "true",
641647
"replication_num" = "3"
642648
);
643-
-- !result
649+
-- !result

0 commit comments

Comments
 (0)