Skip to content

feat(query): support create iceberg table with partition and properties #17812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Apr 23, 2025
+520 −14
Merged

feat(query): support create iceberg table with partition and properties #17812

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d7a7a77
feat(query): support create iceberg table with partition and properties
TCeason Apr 18, 2025
51125be
If create table that in default/hive catalog assign partition or PROP…
TCeason Apr 19, 2025
8c41b4d
1. trait Catalog add new function support_partition
TCeason Apr 20, 2025
a617757
iceberg table do not support alter column name
TCeason Apr 20, 2025
aed8d64
refactor: table_partition/properties refactor from TableMeta to Creat…
TCeason Apr 21, 2025
5d49017
fix ci
TCeason Apr 21, 2025
6eccbc5
fix conversation and optimize code
TCeason Apr 21, 2025
367d1bd
refactor support_partition
TCeason Apr 21, 2025
0ed6447
refactor
TCeason Apr 22, 2025
c0fbd43
if partition by keys is empty return Err
TCeason Apr 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions src/meta/api/src/schema_api_test_suite.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,8 @@ impl SchemaApiTestSuite {
name_ident: db_table_name_ident.clone(),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

{
Expand Down Expand Up @@ -1479,6 +1481,8 @@ impl SchemaApiTestSuite {

table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
let table_id = resp.table_id;
Expand Down Expand Up @@ -1553,6 +1557,8 @@ impl SchemaApiTestSuite {
name_ident: name_ident.clone(),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let res = mt.create_table(req.clone()).await?;
table_id = res.table_id;
Expand Down Expand Up @@ -1639,6 +1645,8 @@ impl SchemaApiTestSuite {

table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
// test create table
{
Expand Down Expand Up @@ -1695,6 +1703,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let tb_ident_2 = {
{
Expand Down Expand Up @@ -1789,6 +1799,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let old_db = mt.get_database(Self::req_get_db(&tenant, db_name)).await?;
Expand Down Expand Up @@ -1920,6 +1932,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(old_created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let res = mt.create_table(req.clone()).await?;
Expand Down Expand Up @@ -1949,6 +1963,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let res = mt.create_table(req.clone()).await?;
Expand Down Expand Up @@ -1991,6 +2007,8 @@ impl SchemaApiTestSuite {
},
table_meta: tbl_meta,
as_dropped: true,
table_properties: None,
table_partition: None,
};
let old_db = mt.get_database(Self::req_get_db(&tenant, db_name)).await?;
let create_table_as_dropped_resp = mt.create_table(create_table_req.clone()).await?;
Expand Down Expand Up @@ -2318,6 +2336,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

info!("--- create table for rename");
Expand Down Expand Up @@ -2545,6 +2565,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let _tb_ident_2 = {
Expand Down Expand Up @@ -2994,6 +3016,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let _res = mt.create_table(req.clone()).await?;

Expand All @@ -3006,6 +3030,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let _res = mt.create_table(req.clone()).await?;
}
Expand Down Expand Up @@ -3345,6 +3371,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let _tb_ident_2 = {
Expand Down Expand Up @@ -3679,6 +3707,8 @@ impl SchemaApiTestSuite {
name_ident,
table_meta: create_table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let res = mt.create_table(req).await?;
Expand Down Expand Up @@ -3915,6 +3945,8 @@ impl SchemaApiTestSuite {
name_ident: tbl_name_ident.clone(),
table_meta: create_table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let res = mt.create_table(req).await?;
Expand Down Expand Up @@ -4151,6 +4183,8 @@ impl SchemaApiTestSuite {
name_ident: tbl_name_ident.clone(),
table_meta: create_table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let old_db = mt.get_database(Self::req_get_db(&tenant, db_name)).await?;
Expand Down Expand Up @@ -4240,6 +4274,8 @@ impl SchemaApiTestSuite {
name_ident: TableNameIdent::new(&tenant, "db1", "tb1"),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
let db1_tb1_id = resp.table_id;
Expand Down Expand Up @@ -4292,6 +4328,8 @@ impl SchemaApiTestSuite {
name_ident: table_name.clone(),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
drop_ids_boundary.push(DroppedId::new_table(
Expand Down Expand Up @@ -4326,6 +4364,8 @@ impl SchemaApiTestSuite {
name_ident: TableNameIdent::new(&tenant, "db2", "tb2"),
table_meta: table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
mt.drop_table_by_id(DropTableByIdReq {
Expand Down Expand Up @@ -4360,6 +4400,8 @@ impl SchemaApiTestSuite {
name_ident: TableNameIdent::new(&tenant, "db2", "tb3"),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
db2_tb3_id = resp.table_id;
Expand Down Expand Up @@ -4399,6 +4441,8 @@ impl SchemaApiTestSuite {
name_ident: TableNameIdent::new(&tenant, "db3", "tb1"),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
drop_ids_boundary.push(DroppedId::new_table(*db3_id, resp.table_id, "tb1"));
Expand All @@ -4424,6 +4468,8 @@ impl SchemaApiTestSuite {
name_ident: TableNameIdent::new(&tenant, "db3", "tb2"),
table_meta: table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
drop_ids_no_boundary.push(DroppedId::new_table(*db3_id, resp.table_id, "tb2"));
Expand Down Expand Up @@ -4452,6 +4498,8 @@ impl SchemaApiTestSuite {
name_ident: TableNameIdent::new(&tenant, "db3", "tb3"),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let _resp = mt.create_table(req.clone()).await?;
}
Expand Down Expand Up @@ -4592,6 +4640,8 @@ impl SchemaApiTestSuite {

table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;

Expand Down Expand Up @@ -4818,6 +4868,8 @@ impl SchemaApiTestSuite {
name_ident: tbl_name_ident.clone(),
table_meta: create_table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let old_db = mt.get_database(Self::req_get_db(&tenant, db_name)).await?;
Expand Down Expand Up @@ -4922,6 +4974,8 @@ impl SchemaApiTestSuite {
name_ident: tbl_name_ident.clone(),
table_meta: create_table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
})
.await?;
let cur_db = mt.get_database(Self::req_get_db(&tenant, db_name)).await?;
Expand Down Expand Up @@ -5003,6 +5057,8 @@ impl SchemaApiTestSuite {
name_ident: new_tbl_name_ident.clone(),
table_meta: create_table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let old_db = mt.get_database(Self::req_get_db(&tenant, db_name)).await?;
Expand Down Expand Up @@ -5155,6 +5211,8 @@ impl SchemaApiTestSuite {
},
table_meta: drop_table_meta(created_on),
as_dropped: true,
table_properties: None,
table_partition: None,
};

let create_table_as_dropped_resp = mt.create_table(create_table_req.clone()).await?;
Expand Down Expand Up @@ -5239,6 +5297,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let _ = mt.create_table(create_table_req.clone()).await?;
Expand Down Expand Up @@ -5271,6 +5331,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: true,
table_properties: None,
table_partition: None,
};

let resp = mt.create_table(create_table_req.clone()).await;
Expand All @@ -5288,6 +5350,8 @@ impl SchemaApiTestSuite {
},
table_meta: drop_table_meta(created_on),
as_dropped: true,
table_properties: None,
table_partition: None,
};

let create_table_as_dropped_resp = mt.create_table(create_table_req.clone()).await?;
Expand Down Expand Up @@ -5330,6 +5394,8 @@ impl SchemaApiTestSuite {
},
table_meta: drop_table_meta(created_on),
as_dropped: true,
table_properties: None,
table_partition: None,
};

let create_table_as_dropped_resp = mt.create_table(create_table_req.clone()).await?;
Expand Down Expand Up @@ -5435,6 +5501,8 @@ impl SchemaApiTestSuite {
},
table_meta: drop_table_meta(created_on),
as_dropped: true,
table_properties: None,
table_partition: None,
};

let concurrent_count: usize = 5;
Expand Down Expand Up @@ -5548,6 +5616,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};

let _tb_ident_2 = {
Expand Down Expand Up @@ -5828,6 +5898,8 @@ impl SchemaApiTestSuite {
name_ident: tbl_name_ident.clone(),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
table_id = resp.table_id;
Expand Down Expand Up @@ -6020,6 +6092,8 @@ impl SchemaApiTestSuite {
..Default::default()
},
as_dropped: false,
table_properties: None,
table_partition: None,
};

let tb_ids = {
Expand Down Expand Up @@ -6083,6 +6157,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = util.mt.create_table(req).await?;

Expand Down Expand Up @@ -6150,6 +6226,8 @@ impl SchemaApiTestSuite {
},
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
table_id = resp.table_id;
Expand Down Expand Up @@ -7558,6 +7636,8 @@ impl SchemaApiTestSuite {
..Default::default()
},
as_dropped: false,
table_properties: None,
table_partition: None,
};
let old_db = node_a
.get_database(Self::req_get_db(&tenant, db_name))
Expand Down Expand Up @@ -7631,6 +7711,8 @@ impl SchemaApiTestSuite {
..Default::default()
},
as_dropped: false,
table_properties: None,
table_partition: None,
};

let old_db = node_a
Expand Down Expand Up @@ -7719,6 +7801,8 @@ impl SchemaApiTestSuite {
name_ident: tbl_name_ident.clone(),
table_meta: table_meta(created_on),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = mt.create_table(req.clone()).await?;
table_id = resp.table_id;
Expand Down Expand Up @@ -8222,6 +8306,8 @@ where MT: SchemaApi + kvapi::AsKVApi<Error = MetaError>
},
table_meta: table_meta.clone(),
as_dropped: false,
table_properties: None,
table_partition: None,
};
let resp = self.mt.create_table(req.clone()).await?;
let table_id = resp.table_id;
Expand Down
1 change: 1 addition & 0 deletions src/meta/app/src/schema/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ pub use table::TableIndexType;
pub use table::TableInfo;
pub use table::TableMeta;
pub use table::TableNameIdent;
pub use table::TablePartition;
pub use table::TableStatistics;
pub use table::TruncateTableReply;
pub use table::TruncateTableReq;
Expand Down
Loading
Loading