Skip to content

Commit 581134d

Browse files
zhengkezhou1zhengkezhou1
committed
refactor(core): Optimize Catalog storage queries using JOIN for JDBC backend
1 parent a5169f1 commit 581134d

File tree

5 files changed

+107
-19
lines changed

5 files changed

+107
-19
lines changed

‎core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@
3838
public interface CatalogMetaMapper {
3939
String TABLE_NAME = "catalog_meta";
4040

41+
@SelectProvider(
42+
type = CatalogMetaSQLProviderFactory.class,
43+
method = "listCatalogPOsByMetalakeName")
44+
List<CatalogPO> listCatalogPOsByMetalakeName(@Param("metalakeName") String metalakeName);
45+
4146
@SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "listCatalogPOsByMetalakeId")
4247
List<CatalogPO> listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId);
4348

@@ -50,12 +55,18 @@ public interface CatalogMetaMapper {
5055
Long selectCatalogIdByMetalakeIdAndName(
5156
@Param("metalakeId") Long metalakeId, @Param("catalogName") String name);
5257

58+
@SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "selectCatalogIdByName")
59+
Long selectCatalogIdByName(@Param("catalogName") String name);
60+
5361
@SelectProvider(
5462
type = CatalogMetaSQLProviderFactory.class,
5563
method = "selectCatalogMetaByMetalakeIdAndName")
5664
CatalogPO selectCatalogMetaByMetalakeIdAndName(
5765
@Param("metalakeId") Long metalakeId, @Param("catalogName") String name);
5866

67+
@SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "selectCatalogMetaByName")
68+
CatalogPO selectCatalogMetaByName(@Param("catalogName") String catalogName);
69+
5970
@SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "selectCatalogMetaById")
6071
CatalogPO selectCatalogMetaById(@Param("catalogId") Long catalogId);
6172

‎core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ static class CatalogMetaMySQLProvider extends CatalogMetaBaseSQLProvider {}
5353

5454
static class CatalogMetaH2Provider extends CatalogMetaBaseSQLProvider {}
5555

56+
public static String listCatalogPOsByMetalakeName(@Param("metalakeName") String metalakeName) {
57+
return getProvider().listCatalogPOsByMetalakeName(metalakeName);
58+
}
59+
5660
public static String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) {
5761
return getProvider().listCatalogPOsByMetalakeId(metalakeId);
5862
}
@@ -61,6 +65,10 @@ public static String listCatalogPOsByCatalogIds(@Param("catalogIds") List<Long>
6165
return getProvider().listCatalogPOsByCatalogIds(catalogIds);
6266
}
6367

68+
public static String selectCatalogIdByName(@Param("catalogName") String name) {
69+
return getProvider().selectCatalogIdByName(name);
70+
}
71+
6472
public static String selectCatalogIdByMetalakeIdAndName(
6573
@Param("metalakeId") Long metalakeId, @Param("catalogName") String name) {
6674
return getProvider().selectCatalogIdByMetalakeIdAndName(metalakeId, name);
@@ -71,6 +79,10 @@ public static String selectCatalogMetaByMetalakeIdAndName(
7179
return getProvider().selectCatalogMetaByMetalakeIdAndName(metalakeId, name);
7280
}
7381

82+
public static String selectCatalogMetaByName(@Param("catalogName") String name) {
83+
return getProvider().selectCatalogMetaByName(name);
84+
}
85+
7486
public static String selectCatalogMetaById(@Param("catalogId") Long catalogId) {
7587
return getProvider().selectCatalogMetaById(catalogId);
7688
}

‎core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/CatalogMetaBaseSQLProvider.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,26 @@
2222
import static org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper.TABLE_NAME;
2323

2424
import java.util.List;
25+
import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper;
2526
import org.apache.gravitino.storage.relational.po.CatalogPO;
2627
import org.apache.ibatis.annotations.Param;
2728

2829
public class CatalogMetaBaseSQLProvider {
30+
public String listCatalogPOsByMetalakeName(@Param("metalakeName") String metalakeName) {
31+
return "SELECT cm.catalog_id as catalogId, cm.catalog_name as catalogName,"
32+
+ " cm.metalake_id as metalakeId, cm.type, cm.provider,"
33+
+ " cm.catalog_comment as catalogComment, cm.properties, cm.audit_info as auditInfo,"
34+
+ " cm.current_version as currentVersion, cm.last_version as lastVersion,"
35+
+ " cm.deleted_at as deletedAt"
36+
+ " FROM "
37+
+ TABLE_NAME
38+
+ " cm JOIN "
39+
+ MetalakeMetaMapper.TABLE_NAME
40+
+ " mm ON cm.metalake_id = mm.metalake_id"
41+
+ " WHERE mm.metalake_name = #{metalakeName}"
42+
+ " AND mm.deleted_at = 0 AND cm.deleted_at = 0";
43+
}
44+
2945
public String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) {
3046
return "SELECT catalog_id as catalogId, catalog_name as catalogName,"
3147
+ " metalake_id as metalakeId, type, provider,"
@@ -55,6 +71,15 @@ public String listCatalogPOsByCatalogIds(@Param("catalogIds") List<Long> catalog
5571
+ "</script>";
5672
}
5773

74+
public String selectCatalogIdByName(@Param("catalogName") String name) {
75+
return "SELECT cm.catalog_id as catalogId FROM "
76+
+ TABLE_NAME
77+
+ " cm JOIN "
78+
+ MetalakeMetaMapper.TABLE_NAME
79+
+ " mm ON cm.metalake_id = mm.metalake_id"
80+
+ " WHERE catalog_name = #{catalogName} AND cm.deleted_at = 0";
81+
}
82+
5883
public String selectCatalogIdByMetalakeIdAndName(
5984
@Param("metalakeId") Long metalakeId, @Param("catalogName") String name) {
6085
return "SELECT catalog_id as catalogId FROM "
@@ -74,6 +99,20 @@ public String selectCatalogMetaByMetalakeIdAndName(
7499
+ " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0";
75100
}
76101

102+
public String selectCatalogMetaByName(@Param("catalogName") String name) {
103+
return "SELECT cm.catalog_id as catalogId, cm.catalog_name as catalogName,"
104+
+ " cm.metalake_id as metalakeId, cm.type, cm.provider,"
105+
+ " cm.catalog_comment as catalogComment, cm.properties, cm.audit_info as auditInfo,"
106+
+ " cm.current_version as currentVersion, cm.last_version as lastVersion,"
107+
+ " cm.deleted_at as deletedAt"
108+
+ " FROM "
109+
+ TABLE_NAME
110+
+ " cm JOIN "
111+
+ MetalakeMetaMapper.TABLE_NAME
112+
+ " mm ON cm.metalake_id = mm.metalake_id"
113+
+ " WHERE cm.catalog_name = #{catalogName} AND cm.deleted_at = 0";
114+
}
115+
77116
public String selectCatalogMetaById(@Param("catalogId") Long catalogId) {
78117
return "SELECT catalog_id as catalogId, catalog_name as catalogName,"
79118
+ " metalake_id as metalakeId, type, provider,"

‎core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,10 @@ public static CatalogMetaService getInstance() {
6565

6666
private CatalogMetaService() {}
6767

68-
public CatalogPO getCatalogPOByMetalakeIdAndName(Long metalakeId, String catalogName) {
68+
public CatalogPO getCatalogPOByName(String catalogName) {
6969
CatalogPO catalogPO =
7070
SessionUtils.getWithoutCommit(
71-
CatalogMetaMapper.class,
72-
mapper -> mapper.selectCatalogMetaByMetalakeIdAndName(metalakeId, catalogName));
71+
CatalogMetaMapper.class, mapper -> mapper.selectCatalogMetaByName(catalogName));
7372

7473
if (catalogPO == null) {
7574
throw new NoSuchEntityException(
@@ -105,26 +104,35 @@ public Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName)
105104
return catalogId;
106105
}
107106

107+
public Long getCatalogIdByName(String catalogName) {
108+
Long catalogId =
109+
SessionUtils.doWithCommitAndFetchResult(
110+
CatalogMetaMapper.class, mapper -> mapper.selectCatalogIdByName(catalogName));
111+
112+
if (catalogId == null) {
113+
throw new NoSuchEntityException(
114+
NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE,
115+
Entity.EntityType.CATALOG.name().toLowerCase(),
116+
catalogName);
117+
}
118+
return catalogId;
119+
}
120+
108121
public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) {
109122
NameIdentifierUtil.checkCatalog(identifier);
110123
String catalogName = identifier.name();
111124

112-
Long metalakeId =
113-
CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace());
114-
115-
CatalogPO catalogPO = getCatalogPOByMetalakeIdAndName(metalakeId, catalogName);
125+
CatalogPO catalogPO = getCatalogPOByName(catalogName);
116126

117127
return POConverters.fromCatalogPO(catalogPO, identifier.namespace());
118128
}
119129

120130
public List<CatalogEntity> listCatalogsByNamespace(Namespace namespace) {
121131
NamespaceUtil.checkCatalog(namespace);
122-
123-
Long metalakeId = CommonMetaService.getInstance().getParentEntityIdByNamespace(namespace);
124-
125132
List<CatalogPO> catalogPOS =
126133
SessionUtils.getWithoutCommit(
127-
CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByMetalakeId(metalakeId));
134+
CatalogMetaMapper.class,
135+
mapper -> mapper.listCatalogPOsByMetalakeName(namespace.level(0)));
128136

129137
return POConverters.fromCatalogPOs(catalogPOS, namespace);
130138
}
@@ -158,10 +166,8 @@ public <E extends Entity & HasIdentifier> CatalogEntity updateCatalog(
158166
NameIdentifierUtil.checkCatalog(identifier);
159167

160168
String catalogName = identifier.name();
161-
Long metalakeId =
162-
CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace());
163169

164-
CatalogPO oldCatalogPO = getCatalogPOByMetalakeIdAndName(metalakeId, catalogName);
170+
CatalogPO oldCatalogPO = getCatalogPOByName(catalogName);
165171

166172
CatalogEntity oldCatalogEntity =
167173
POConverters.fromCatalogPO(oldCatalogPO, identifier.namespace());
@@ -179,7 +185,8 @@ public <E extends Entity & HasIdentifier> CatalogEntity updateCatalog(
179185
CatalogMetaMapper.class,
180186
mapper ->
181187
mapper.updateCatalogMeta(
182-
POConverters.updateCatalogPOWithVersion(oldCatalogPO, newEntity, metalakeId),
188+
POConverters.updateCatalogPOWithVersion(
189+
oldCatalogPO, newEntity, oldCatalogPO.getMetalakeId()),
183190
oldCatalogPO));
184191
} catch (RuntimeException re) {
185192
ExceptionUtils.checkSQLException(
@@ -198,10 +205,7 @@ public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) {
198205
NameIdentifierUtil.checkCatalog(identifier);
199206

200207
String catalogName = identifier.name();
201-
Long metalakeId =
202-
CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace());
203-
204-
Long catalogId = getCatalogIdByMetalakeIdAndName(metalakeId, catalogName);
208+
long catalogId = getCatalogIdByName(catalogName);
205209

206210
if (cascade) {
207211
SessionUtils.doMultipleWithCommit(

‎core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import static org.apache.gravitino.Configs.ENTITY_STORE;
2828
import static org.apache.gravitino.Configs.RELATIONAL_ENTITY_STORE;
2929
import static org.apache.gravitino.SupportsRelationOperations.Type.OWNER_REL;
30+
import static org.junit.Assert.assertNotNull;
3031
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
3132
import static org.junit.jupiter.api.Assertions.assertEquals;
3233
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -79,8 +80,10 @@
7980
import org.apache.gravitino.meta.TopicEntity;
8081
import org.apache.gravitino.meta.UserEntity;
8182
import org.apache.gravitino.storage.RandomIdGenerator;
83+
import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper;
8284
import org.apache.gravitino.storage.relational.mapper.GroupMetaMapper;
8385
import org.apache.gravitino.storage.relational.mapper.UserMetaMapper;
86+
import org.apache.gravitino.storage.relational.service.CatalogMetaService;
8487
import org.apache.gravitino.storage.relational.service.MetalakeMetaService;
8588
import org.apache.gravitino.storage.relational.service.RoleMetaService;
8689
import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper;
@@ -759,6 +762,13 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException {
759762
backend.list(catalog.namespace(), Entity.EntityType.CATALOG, true);
760763
assertTrue(catalogs.contains(catalog));
761764

765+
assertEquals(
766+
1,
767+
SessionUtils.doWithCommitAndFetchResult(
768+
CatalogMetaMapper.class,
769+
mapper -> mapper.listCatalogPOsByMetalakeName(metalake.name()))
770+
.size());
771+
762772
List<SchemaEntity> schemas = backend.list(schema.namespace(), Entity.EntityType.SCHEMA, true);
763773
assertTrue(schemas.contains(schema));
764774

@@ -782,6 +792,11 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException {
782792
assertEquals(1, RoleMetaService.getInstance().listRolesByUserId(user.id()).size());
783793
assertEquals(1, RoleMetaService.getInstance().listRolesByGroupId(group.id()).size());
784794

795+
CatalogEntity catalogEntity = backend.get(catalog.nameIdentifier(), Entity.EntityType.CATALOG);
796+
assertEquals(catalog, catalogEntity);
797+
assertNotNull(CatalogMetaService.getInstance().getCatalogPOByName(catalog.name()));
798+
assertEquals(catalog.id(), CatalogMetaService.getInstance().getCatalogIdByName(catalog.name()));
799+
785800
UserEntity userEntity = backend.get(user.nameIdentifier(), Entity.EntityType.USER);
786801
assertEquals(user, userEntity);
787802
assertEquals(
@@ -856,6 +871,13 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException {
856871
// meta data soft delete
857872
backend.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, true);
858873

874+
assertEquals(
875+
0,
876+
SessionUtils.doWithCommitAndFetchResult(
877+
CatalogMetaMapper.class,
878+
mapper -> mapper.listCatalogPOsByMetalakeName(metalake.name()))
879+
.size());
880+
859881
// check existence after soft delete
860882
assertFalse(backend.exists(metalake.nameIdentifier(), Entity.EntityType.METALAKE));
861883
assertTrue(backend.exists(anotherMetaLake.nameIdentifier(), Entity.EntityType.METALAKE));

0 commit comments

Comments
 (0)