From aed035a84c1505cafc0aa3a0f9e96931a29f30ca Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Tue, 4 Mar 2025 10:09:14 +0800 Subject: [PATCH 01/16] feat: move getFilesetObjectFullNames from RoleMetaService to MetadataObjectService --- .../service/MetadataObjectService.java | 71 ++++++++++++++++++ .../relational/service/RoleMetaService.java | 74 +------------------ 2 files changed, 73 insertions(+), 72 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index e6790a602c1..fd0a413f8bc 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -20,9 +20,15 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; +import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; +import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; import org.apache.gravitino.storage.relational.po.CatalogPO; import org.apache.gravitino.storage.relational.po.ColumnPO; import org.apache.gravitino.storage.relational.po.FilesetPO; @@ -31,6 +37,9 @@ import org.apache.gravitino.storage.relational.po.SchemaPO; import org.apache.gravitino.storage.relational.po.TablePO; import org.apache.gravitino.storage.relational.po.TopicPO; +import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * MetadataObjectService is used for converting full name to entity id and converting entity id to @@ -42,6 +51,8 @@ public class MetadataObjectService { private static final Joiner DOT_JOINER = Joiner.on(DOT); private static final Splitter DOT_SPLITTER = Splitter.on(DOT); + private static final Logger LOG = LoggerFactory.getLogger(MetadataObjectService.class); + private MetadataObjectService() {} public static long getMetadataObjectId( @@ -214,4 +225,64 @@ public static String getMetadataObjectFullName(String type, long metadataObjectI return fullName; } + + public static Map getFilesetObjectFullNames(List ids) { + List filesetPOs = + SessionUtils.getWithoutCommit( + FilesetMetaMapper.class, mapper -> mapper.listFilesetPOsByFilesetIds(ids)); + + if (filesetPOs == null || filesetPOs.isEmpty()) { + return new HashMap<>(); + } + + List catalogIds = + filesetPOs.stream().map(FilesetPO::getCatalogId).collect(Collectors.toList()); + List schemaIds = + filesetPOs.stream().map(FilesetPO::getSchemaId).collect(Collectors.toList()); + + Map catalogIdAndNameMap = getCatalogIdAndNameMap(catalogIds); + Map schemaIdAndNameMap = getSchemaIdAndNameMap(schemaIds); + + HashMap filesetIdAndNameMap = new HashMap<>(); + + filesetPOs.forEach( + filesetPO -> { + // since the catalog or schema can be deleted, we need to check the null value, + // and when catalog or schema is deleted, we will set catalogName or schemaName to null + String catalogName = catalogIdAndNameMap.getOrDefault(filesetPO.getCatalogId(), null); + if (catalogName == null) { + LOG.warn("The catalog of fileset {} may be deleted", filesetPO.getFilesetId()); + filesetIdAndNameMap.put(filesetPO.getFilesetId(), null); + return; + } + + String schemaName = schemaIdAndNameMap.getOrDefault(filesetPO.getSchemaId(), null); + if (schemaName == null) { + LOG.warn("The schema of fileset {} may be deleted", filesetPO.getFilesetId()); + filesetIdAndNameMap.put(filesetPO.getFilesetId(), null); + return; + } + + String fullName = DOT_JOINER.join(catalogName, schemaName, filesetPO.getFilesetName()); + filesetIdAndNameMap.put(filesetPO.getFilesetId(), fullName); + }); + + return filesetIdAndNameMap; + } + + public static Map getSchemaIdAndNameMap(List schemaIds) { + List schemaPOS = + SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> mapper.listSchemaPOsBySchemaIds(schemaIds)); + return schemaPOS.stream() + .collect(Collectors.toMap(SchemaPO::getSchemaId, SchemaPO::getSchemaName)); + } + + public static Map getCatalogIdAndNameMap(List catalogIds) { + List catalogPOs = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByCatalogIds(catalogIds)); + return catalogPOs.stream() + .collect(Collectors.toMap(CatalogPO::getCatalogId, CatalogPO::getCatalogName)); + } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 41c4c409ca3..b86cb860c2f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -18,14 +18,12 @@ */ package org.apache.gravitino.storage.relational.service; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.IOException; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -42,18 +40,12 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.meta.RoleEntity; -import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; -import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; import org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper; import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.mapper.RoleMetaMapper; -import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; import org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper; import org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper; -import org.apache.gravitino.storage.relational.po.CatalogPO; -import org.apache.gravitino.storage.relational.po.FilesetPO; import org.apache.gravitino.storage.relational.po.RolePO; -import org.apache.gravitino.storage.relational.po.SchemaPO; import org.apache.gravitino.storage.relational.po.SecurableObjectPO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; @@ -64,9 +56,6 @@ /** The service class for role metadata. It provides the basic database operations for role. */ public class RoleMetaService { - private static final String DOT = "."; - private static final Joiner DOT_JOINER = Joiner.on(DOT); - private static final Logger LOG = LoggerFactory.getLogger(RoleMetaService.class); private static final RoleMetaService INSTANCE = new RoleMetaService(); @@ -377,7 +366,8 @@ private static List listSecurableObjects(RolePO po) { .map(SecurableObjectPO::getMetadataObjectId) .collect(Collectors.toList()); - Map filesetIdAndNameMap = getFilesetObjectFullNames(filesetIds); + Map filesetIdAndNameMap = + MetadataObjectService.getFilesetObjectFullNames(filesetIds); for (SecurableObjectPO securableObjectPO : objects) { String fullName = @@ -443,64 +433,4 @@ private static MetadataObject.Type getType(String type) { private static String getEntityType(SecurableObject securableObject) { return securableObject.type().name(); } - - public static Map getFilesetObjectFullNames(List ids) { - List filesetPOs = - SessionUtils.getWithoutCommit( - FilesetMetaMapper.class, mapper -> mapper.listFilesetPOsByFilesetIds(ids)); - - if (filesetPOs == null || filesetPOs.isEmpty()) { - return new HashMap<>(); - } - - List catalogIds = - filesetPOs.stream().map(FilesetPO::getCatalogId).collect(Collectors.toList()); - List schemaIds = - filesetPOs.stream().map(FilesetPO::getSchemaId).collect(Collectors.toList()); - - Map catalogIdAndNameMap = getCatalogIdAndNameMap(catalogIds); - Map schemaIdAndNameMap = getSchemaIdAndNameMap(schemaIds); - - HashMap filesetIdAndNameMap = new HashMap<>(); - - filesetPOs.forEach( - filesetPO -> { - // since the catalog or schema can be deleted, we need to check the null value, - // and when catalog or schema is deleted, we will set catalogName or schemaName to null - String catalogName = catalogIdAndNameMap.getOrDefault(filesetPO.getCatalogId(), null); - if (catalogName == null) { - LOG.warn("The catalog of fileset {} may be deleted", filesetPO.getFilesetId()); - filesetIdAndNameMap.put(filesetPO.getFilesetId(), null); - return; - } - - String schemaName = schemaIdAndNameMap.getOrDefault(filesetPO.getSchemaId(), null); - if (schemaName == null) { - LOG.warn("The schema of fileset {} may be deleted", filesetPO.getFilesetId()); - filesetIdAndNameMap.put(filesetPO.getFilesetId(), null); - return; - } - - String fullName = DOT_JOINER.join(catalogName, schemaName, filesetPO.getFilesetName()); - filesetIdAndNameMap.put(filesetPO.getFilesetId(), fullName); - }); - - return filesetIdAndNameMap; - } - - public static Map getSchemaIdAndNameMap(List schemaIds) { - List schemaPOS = - SessionUtils.getWithoutCommit( - SchemaMetaMapper.class, mapper -> mapper.listSchemaPOsBySchemaIds(schemaIds)); - return schemaPOS.stream() - .collect(Collectors.toMap(SchemaPO::getSchemaId, SchemaPO::getSchemaName)); - } - - public static Map getCatalogIdAndNameMap(List catalogIds) { - List catalogPOs = - SessionUtils.getWithoutCommit( - CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByCatalogIds(catalogIds)); - return catalogPOs.stream() - .collect(Collectors.toMap(CatalogPO::getCatalogId, CatalogPO::getCatalogName)); - } } From f6ff9d00291973d3cbb2ff84cc8c5831596ff010 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Tue, 4 Mar 2025 17:29:59 +0800 Subject: [PATCH 02/16] feat(RoleMetaService.listSecurableObjects): metalake batch retrival --- .../relational/mapper/MetalakeMetaMapper.java | 5 ++++ .../MetalakeMetaSQLProviderFactory.java | 5 ++++ .../base/MetalakeMetaBaseSQLProvider.java | 19 ++++++++++++++ .../service/MetadataObjectService.java | 25 +++++++++++++++++++ .../relational/service/RoleMetaService.java | 24 ++++++++++++++++++ .../service/TestSecurableObjects.java | 23 +++++++++++++++-- 6 files changed, 99 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index d5dc809bfe2..90027ae1515 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -47,6 +47,11 @@ public interface MetalakeMetaMapper { @SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeMetaById") MetalakePO selectMetalakeMetaById(@Param("metalakeId") Long metalakeId); + @SelectProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "listMetalakePOsByMetalakeIds") + List listMetalakePOsByMetalakeIds(@Param("metalakeIds") List metalakeIds); + @SelectProvider( type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeIdMetaByName") diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java index 5b3ab58fbbb..ee2c52a7dc3 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java @@ -20,6 +20,7 @@ package org.apache.gravitino.storage.relational.mapper; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; import org.apache.gravitino.storage.relational.mapper.provider.base.MetalakeMetaBaseSQLProvider; @@ -68,6 +69,10 @@ public static String selectMetalakeIdMetaByName(@Param("metalakeName") String me return getProvider().selectMetalakeIdMetaByName(metalakeName); } + public static String listMetalakePOsByMetalakeIds(@Param("metalakeIds") List metalakeIds) { + return getProvider().listMetalakePOsByMetalakeIds(metalakeIds); + } + public static String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) { return getProvider().insertMetalakeMeta(metalakePO); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/MetalakeMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/MetalakeMetaBaseSQLProvider.java index b03a228a6c6..5a041a36496 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/MetalakeMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/MetalakeMetaBaseSQLProvider.java @@ -20,6 +20,7 @@ import static org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper.TABLE_NAME; +import java.util.List; import org.apache.gravitino.storage.relational.po.MetalakePO; import org.apache.ibatis.annotations.Param; @@ -65,6 +66,24 @@ public String selectMetalakeIdMetaByName(@Param("metalakeName") String metalakeN + " WHERE metalake_name = #{metalakeName} and deleted_at = 0"; } + public String listMetalakePOsByMetalakeIds(@Param("metalakeIds") List metalakeIds) { + return ""; + } + public String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) { return "INSERT INTO " + TABLE_NAME diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index fd0a413f8bc..d3606b2255c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -28,6 +28,7 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; import org.apache.gravitino.storage.relational.po.CatalogPO; import org.apache.gravitino.storage.relational.po.ColumnPO; @@ -226,6 +227,30 @@ public static String getMetadataObjectFullName(String type, long metadataObjectI return fullName; } + public static Map getMetalakeObjectFullNames(List ids) { + + List metalakePOs = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.listMetalakePOsByMetalakeIds(ids)); + + if (metalakePOs == null || metalakePOs.isEmpty()) { + return new HashMap<>(); + } + + HashMap metalakeIdAndNameMap = new HashMap<>(); + + metalakePOs.forEach( + metalakePO -> { + if (metalakePO.getMetalakeId() == null) { + metalakeIdAndNameMap.put(metalakePO.getMetalakeId(), null); + return; + } + metalakeIdAndNameMap.put(metalakePO.getMetalakeId(), metalakePO.getMetalakeName()); + }); + + return metalakeIdAndNameMap; + } + public static Map getFilesetObjectFullNames(List ids) { List filesetPOs = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index b86cb860c2f..8b6d2dc0d91 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -383,6 +383,30 @@ private static List listSecurableObjects(RolePO po) { securableObjectPO.getType()); } } + } else if (type.equals(MetadataObject.Type.METALAKE.name())) { + // } else if (false) { + List metalakeIds = + objects.stream() + .map(SecurableObjectPO::getMetadataObjectId) + .collect(Collectors.toList()); + + Map metalakeIdAndNameMap = + MetadataObjectService.getMetalakeObjectFullNames(metalakeIds); + + for (SecurableObjectPO securableObjectPO : objects) { + String fullName = + metalakeIdAndNameMap.get(securableObjectPO.getMetadataObjectId()); + if (fullName != null) { + securableObjects.add( + POConverters.fromSecurableObjectPO( + fullName, securableObjectPO, getType(securableObjectPO.getType()))); + } else { + LOG.warn( + "The securable object {} {} may be deleted", + securableObjectPO.getMetadataObjectId(), + securableObjectPO.getType()); + } + } } else { // todo：to get other securable object fullNames using batch retrieving for (SecurableObjectPO securableObjectPO : objects) { diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java index 7d18b45b460..9be4692917b 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java @@ -52,9 +52,13 @@ public void testAllTypeSecurableObjects() throws IOException { AuditInfo auditInfo = AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); BaseMetalake metalake = - createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName + "2", auditInfo); backend.insert(metalake, false); + BaseMetalake metalake2 = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake2, false); + CatalogEntity catalog = createCatalog( RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); @@ -90,6 +94,14 @@ public void testAllTypeSecurableObjects() throws IOException { auditInfo); backend.insert(topic, false); + SecurableObject metalakeObject = + SecurableObjects.ofMetalake( + metalake.name(), Lists.newArrayList(Privileges.UseCatalog.allow())); + + SecurableObject metalakeObject2 = + SecurableObjects.ofMetalake( + metalake2.name(), Lists.newArrayList(Privileges.UseCatalog.allow())); + SecurableObject catalogObject = SecurableObjects.ofCatalog( "catalog", @@ -109,7 +121,14 @@ public void testAllTypeSecurableObjects() throws IOException { schemaObject, "topic", Lists.newArrayList(Privileges.ConsumeTopic.deny())); ArrayList securableObjects = - Lists.newArrayList(catalogObject, schemaObject, tableObject, filesetObject, topicObject); + Lists.newArrayList( + metalakeObject, + metalakeObject2, + catalogObject, + schemaObject, + tableObject, + filesetObject, + topicObject); securableObjects.sort(Comparator.comparing(MetadataObject::fullName)); RoleEntity role1 = From 6911389b4dbabbba31e23c4c32731acc6346188e Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Tue, 4 Mar 2025 20:09:50 +0800 Subject: [PATCH 03/16] feat: dynamically calling object fullname getter --- .../service/MetadataObjectService.java | 1 - .../relational/service/RoleMetaService.java | 121 +++++++++--------- 2 files changed, 59 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index d3606b2255c..618cdf53ff5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -228,7 +228,6 @@ public static String getMetadataObjectFullName(String type, long metadataObjectI } public static Map getMetalakeObjectFullNames(List ids) { - List metalakePOs = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.listMetalakePOsByMetalakeIds(ids)); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 8b6d2dc0d91..4303fbcfe67 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -19,6 +19,7 @@ package org.apache.gravitino.storage.relational.service; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.IOException; @@ -28,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -360,70 +362,65 @@ private static List listSecurableObjects(RolePO po) { (type, objects) -> { // If the type is Fileset, use the batch retrieval interface; // otherwise, use the single retrieval interface - if (type.equals(MetadataObject.Type.FILESET.name())) { - List filesetIds = - objects.stream() - .map(SecurableObjectPO::getMetadataObjectId) - .collect(Collectors.toList()); - - Map filesetIdAndNameMap = - MetadataObjectService.getFilesetObjectFullNames(filesetIds); - - for (SecurableObjectPO securableObjectPO : objects) { - String fullName = - filesetIdAndNameMap.get(securableObjectPO.getMetadataObjectId()); - if (fullName != null) { - securableObjects.add( - POConverters.fromSecurableObjectPO( - fullName, securableObjectPO, getType(securableObjectPO.getType()))); - } else { - LOG.warn( - "The securable object {} {} may be deleted", - securableObjectPO.getMetadataObjectId(), - securableObjectPO.getType()); - } - } - } else if (type.equals(MetadataObject.Type.METALAKE.name())) { - // } else if (false) { - List metalakeIds = - objects.stream() - .map(SecurableObjectPO::getMetadataObjectId) - .collect(Collectors.toList()); - - Map metalakeIdAndNameMap = - MetadataObjectService.getMetalakeObjectFullNames(metalakeIds); - - for (SecurableObjectPO securableObjectPO : objects) { - String fullName = - metalakeIdAndNameMap.get(securableObjectPO.getMetadataObjectId()); - if (fullName != null) { - securableObjects.add( - POConverters.fromSecurableObjectPO( - fullName, securableObjectPO, getType(securableObjectPO.getType()))); - } else { - LOG.warn( - "The securable object {} {} may be deleted", - securableObjectPO.getMetadataObjectId(), - securableObjectPO.getType()); + switch (MetadataObject.Type.valueOf(type)) { + case FILESET: + case METALAKE: + List objectIds = + objects.stream() + .map(SecurableObjectPO::getMetadataObjectId) + .collect(Collectors.toList()); + + Map, Map>> + objectFullNameGetterFnMap = + ImmutableMap.of( + MetadataObject.Type.FILESET, + MetadataObjectService::getFilesetObjectFullNames, + MetadataObject.Type.METALAKE, + MetadataObjectService::getMetalakeObjectFullNames); + + // dynamically calling getter function based on type + Map objectIdAndNameMap = + Optional.of(MetadataObject.Type.valueOf(type)) + .map(objectFullNameGetterFnMap::get) + .map(getter -> getter.apply(objectIds)) + .orElseThrow( + () -> + new IllegalArgumentException( + "Unsupported metadata object type: " + type)); + + for (SecurableObjectPO securableObjectPO : objects) { + String fullName = + objectIdAndNameMap.get(securableObjectPO.getMetadataObjectId()); + if (fullName != null) { + securableObjects.add( + POConverters.fromSecurableObjectPO( + fullName, securableObjectPO, getType(securableObjectPO.getType()))); + } else { + LOG.warn( + "The securable object {} {} may be deleted", + securableObjectPO.getMetadataObjectId(), + securableObjectPO.getType()); + } } - } - } else { - // todo：to get other securable object fullNames using batch retrieving - for (SecurableObjectPO securableObjectPO : objects) { - String fullName = - MetadataObjectService.getMetadataObjectFullName( - securableObjectPO.getType(), securableObjectPO.getMetadataObjectId()); - if (fullName != null) { - securableObjects.add( - POConverters.fromSecurableObjectPO( - fullName, securableObjectPO, getType(securableObjectPO.getType()))); - } else { - LOG.warn( - "The securable object {} {} may be deleted", - securableObjectPO.getMetadataObjectId(), - securableObjectPO.getType()); + break; + default: + + // todo：to get other securable object fullNames using batch retrieving + for (SecurableObjectPO securableObjectPO : objects) { + String fullName = + MetadataObjectService.getMetadataObjectFullName( + securableObjectPO.getType(), securableObjectPO.getMetadataObjectId()); + if (fullName != null) { + securableObjects.add( + POConverters.fromSecurableObjectPO( + fullName, securableObjectPO, getType(securableObjectPO.getType()))); + } else { + LOG.warn( + "The securable object {} {} may be deleted", + securableObjectPO.getMetadataObjectId(), + securableObjectPO.getType()); + } } - } } }); From 0bd79a09c13a4ecfb1d71f52f29b48d8c3f809a8 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Wed, 5 Mar 2025 09:35:14 +0800 Subject: [PATCH 04/16] feat(listSecurableObjects): implement batch retrieval for Catalog and Table --- .../relational/mapper/TableMetaMapper.java | 3 + .../mapper/TableMetaSQLProviderFactory.java | 5 ++ .../base/TableMetaBaseSQLProvider.java | 19 +++++ .../service/MetadataObjectService.java | 73 ++++++++++++++++++- .../relational/service/RoleMetaService.java | 10 ++- 5 files changed, 107 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java index a5224593765..41621027bf5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java @@ -41,6 +41,9 @@ public interface TableMetaMapper { @SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsBySchemaId") List listTablePOsBySchemaId(@Param("schemaId") Long schemaId); + @SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsByTableIds") + List listTablePOsByTableIds(@Param("tableIds") List tableIds); + @SelectProvider( type = TableMetaSQLProviderFactory.class, method = "selectTableIdBySchemaIdAndName") diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java index c7152acd663..664b2423f80 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java @@ -19,6 +19,7 @@ package org.apache.gravitino.storage.relational.mapper; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; import org.apache.gravitino.storage.relational.mapper.provider.base.TableMetaBaseSQLProvider; @@ -54,6 +55,10 @@ public static String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) { return getProvider().listTablePOsBySchemaId(schemaId); } + public static String listTablePOsByTableIds(@Param("tableIds") List tableIds) { + return getProvider().listTablePOsByTableIds(tableIds); + } + public static String selectTableIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name) { return getProvider().selectTableIdBySchemaIdAndName(schemaId, name); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java index 6169cc65a53..41f26e23405 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableMetaBaseSQLProvider.java @@ -20,6 +20,7 @@ import static org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TABLE_NAME; +import java.util.List; import org.apache.gravitino.storage.relational.po.TablePO; import org.apache.ibatis.annotations.Param; @@ -36,6 +37,24 @@ public String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) { + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; } + public String listTablePOsByTableIds(List tableIds) { + return ""; + } + public String selectTableIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name) { return "SELECT table_id as tableId FROM " diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index 618cdf53ff5..8f3a61b2a5c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -30,6 +30,7 @@ import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.po.CatalogPO; import org.apache.gravitino.storage.relational.po.ColumnPO; import org.apache.gravitino.storage.relational.po.FilesetPO; @@ -137,6 +138,8 @@ public static String getMetadataObjectFullName(String type, long metadataObjectI case SCHEMA: SchemaPO schemaPO = SchemaMetaService.getInstance().getSchemaPOById(objectId); if (schemaPO != null) { + // if fullName is null: + // fullName = catalogPO.getCatalogName(),schemaPO.getSchemaName() fullName = fullName != null ? DOT_JOINER.join(schemaPO.getSchemaName(), fullName) @@ -150,6 +153,8 @@ public static String getMetadataObjectFullName(String type, long metadataObjectI case TABLE: TablePO tablePO = TableMetaService.getInstance().getTablePOById(objectId); + // if fullName is null: + // fullName = catalogPO.getSchemaName(),schemaPO.getTableName() if (tablePO != null) { fullName = fullName != null @@ -272,7 +277,7 @@ public static Map getFilesetObjectFullNames(List ids) { filesetPOs.forEach( filesetPO -> { // since the catalog or schema can be deleted, we need to check the null value, - // and when catalog or schema is deleted, we will set catalogName or schemaName to null + // and when catalog or schema is deleted, we will set fullName of filesetPO to null String catalogName = catalogIdAndNameMap.getOrDefault(filesetPO.getCatalogId(), null); if (catalogName == null) { LOG.warn("The catalog of fileset {} may be deleted", filesetPO.getFilesetId()); @@ -294,6 +299,72 @@ public static Map getFilesetObjectFullNames(List ids) { return filesetIdAndNameMap; } + public static Map getTableObjectFullNames(List ids) { + List tablePOs = + SessionUtils.getWithoutCommit( + TableMetaMapper.class, mapper -> mapper.listTablePOsByTableIds(ids)); + + if (tablePOs == null || tablePOs.isEmpty()) { + return new HashMap<>(); + } + + List catalogIds = + tablePOs.stream().map(TablePO::getCatalogId).collect(Collectors.toList()); + List schemaIds = tablePOs.stream().map(TablePO::getSchemaId).collect(Collectors.toList()); + + Map catalogIdAndNameMap = getCatalogIdAndNameMap(catalogIds); + Map schemaIdAndNameMap = getSchemaIdAndNameMap(schemaIds); + + HashMap tableIdAndNameMap = new HashMap<>(); + + tablePOs.forEach( + tablePO -> { + // since the catalog or schema can be deleted, we need to check the null value, + // and when catalog or schema is deleted, we will set fullName of tablePO to null + String catalogName = catalogIdAndNameMap.getOrDefault(tablePO.getCatalogId(), null); + if (catalogName == null) { + LOG.warn("The catalog of table {} may be deleted", tablePO.getTableId()); + tableIdAndNameMap.put(tablePO.getTableId(), null); + return; + } + + String schemaName = schemaIdAndNameMap.getOrDefault(tablePO.getSchemaId(), null); + if (schemaName == null) { + LOG.warn("The schema of table {} may be deleted", tablePO.getTableId()); + tableIdAndNameMap.put(tablePO.getTableId(), null); + return; + } + + String fullName = DOT_JOINER.join(catalogName, schemaName, tablePO.getTableName()); + tableIdAndNameMap.put(tablePO.getTableId(), fullName); + }); + + return tableIdAndNameMap; + } + + public static Map getCatalogObjectFullNames(List ids) { + List catalogPOs = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByCatalogIds(ids)); + + if (catalogPOs == null || catalogPOs.isEmpty()) { + return new HashMap<>(); + } + + HashMap catalogIdAndNameMap = new HashMap<>(); + + catalogPOs.forEach( + catalogPO -> { + if (catalogPO.getCatalogId() == null) { + catalogIdAndNameMap.put(catalogPO.getMetalakeId(), null); + return; + } + catalogIdAndNameMap.put(catalogPO.getCatalogId(), catalogPO.getCatalogName()); + }); + + return catalogIdAndNameMap; + } + public static Map getSchemaIdAndNameMap(List schemaIds) { List schemaPOS = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 4303fbcfe67..6de91be1c77 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -363,8 +363,10 @@ private static List listSecurableObjects(RolePO po) { // If the type is Fileset, use the batch retrieval interface; // otherwise, use the single retrieval interface switch (MetadataObject.Type.valueOf(type)) { - case FILESET: case METALAKE: + case CATALOG: + case FILESET: + case TABLE: List objectIds = objects.stream() .map(SecurableObjectPO::getMetadataObjectId) @@ -376,7 +378,11 @@ private static List listSecurableObjects(RolePO po) { MetadataObject.Type.FILESET, MetadataObjectService::getFilesetObjectFullNames, MetadataObject.Type.METALAKE, - MetadataObjectService::getMetalakeObjectFullNames); + MetadataObjectService::getMetalakeObjectFullNames, + MetadataObject.Type.CATALOG, + MetadataObjectService::getCatalogObjectFullNames, + MetadataObject.Type.TABLE, + MetadataObjectService::getTableObjectFullNames); // dynamically calling getter function based on type Map objectIdAndNameMap = From f7b03e60c705b03debd4f1ff1b8d372616ca6947 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Wed, 5 Mar 2025 10:23:51 +0800 Subject: [PATCH 05/16] feat(listSecurableObjects): implement batch retrieval for Model --- .../authorization/SecurableObjects.java | 16 +++++++ .../relational/mapper/ModelMetaMapper.java | 3 ++ .../mapper/ModelMetaSQLProviderFactory.java | 5 +++ .../base/ModelMetaBaseSQLProvider.java | 18 ++++++++ .../service/MetadataObjectService.java | 44 +++++++++++++++++++ .../relational/service/RoleMetaService.java | 13 +++--- .../service/TestSecurableObjects.java | 16 +++++++ 7 files changed, 110 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java b/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java index e6ca3a851c7..06f7ac95b97 100644 --- a/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java +++ b/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java @@ -120,6 +120,22 @@ public static SecurableObject ofFileset( return of(MetadataObject.Type.FILESET, names, privileges); } + /** + * Create the table {@link SecurableObject} with the given securable schema object, fileset name + * and privileges. + * + * @param schema The schema securable object + * @param model The model name + * @param privileges The privileges of the model + * @return The created model {@link SecurableObject} + */ + public static SecurableObject ofModel( + SecurableObject schema, String model, List privileges) { + List names = Lists.newArrayList(DOT_SPLITTER.splitToList(schema.fullName())); + names.add(model); + return of(MetadataObject.Type.MODEL, names, privileges); + } + private static class SecurableObjectImpl extends MetadataObjectImpl implements SecurableObject { private List privileges; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaMapper.java index 53aba8353d0..b16c556c71c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaMapper.java @@ -40,6 +40,9 @@ public interface ModelMetaMapper { @SelectProvider(type = ModelMetaSQLProviderFactory.class, method = "listModelPOsBySchemaId") List listModelPOsBySchemaId(@Param("schemaId") Long schemaId); + @SelectProvider(type = ModelMetaSQLProviderFactory.class, method = "listModelPOsByModelIds") + List listModelPOsByModelIds(@Param("modelIds") List modelIds); + @SelectProvider( type = ModelMetaSQLProviderFactory.class, method = "selectModelMetaBySchemaIdAndModelName") diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaSQLProviderFactory.java index 71c20508312..796009737bf 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/ModelMetaSQLProviderFactory.java @@ -19,6 +19,7 @@ package org.apache.gravitino.storage.relational.mapper; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; import org.apache.gravitino.storage.relational.mapper.provider.base.ModelMetaBaseSQLProvider; @@ -62,6 +63,10 @@ public static String listModelPOsBySchemaId(@Param("schemaId") Long schemaId) { return getProvider().listModelPOsBySchemaId(schemaId); } + public static String listModelPOsByModelIds(@Param("modelIds") List modelIds) { + return getProvider().listModelPOsByModelIds(modelIds); + } + public static String selectModelMetaBySchemaIdAndModelName( @Param("schemaId") Long schemaId, @Param("modelName") String modelName) { return getProvider().selectModelMetaBySchemaIdAndModelName(schemaId, modelName); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/ModelMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/ModelMetaBaseSQLProvider.java index 0a78de9d09d..aa0c42fe57c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/ModelMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/ModelMetaBaseSQLProvider.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.storage.relational.mapper.provider.base; +import java.util.List; import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; import org.apache.gravitino.storage.relational.po.ModelPO; import org.apache.ibatis.annotations.Param; @@ -66,6 +67,23 @@ public String listModelPOsBySchemaId(@Param("schemaId") Long schemaId) { + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; } + public String listModelPOsByModelIds(List modelIds) { + return ""; + } + public String selectModelMetaBySchemaIdAndModelName( @Param("schemaId") Long schemaId, @Param("modelName") String modelName) { return "SELECT model_id AS modelId, model_name AS modelName, metalake_id AS metalakeId," diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index 8f3a61b2a5c..0309589187b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -29,6 +29,7 @@ import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.po.CatalogPO; @@ -299,6 +300,49 @@ public static Map getFilesetObjectFullNames(List ids) { return filesetIdAndNameMap; } + public static Map getModelObjectFullNames(List ids) { + List modelPOs = + SessionUtils.getWithoutCommit( + ModelMetaMapper.class, mapper -> mapper.listModelPOsByModelIds(ids)); + + if (modelPOs == null || modelPOs.isEmpty()) { + return new HashMap<>(); + } + + List catalogIds = + modelPOs.stream().map(ModelPO::getCatalogId).collect(Collectors.toList()); + List schemaIds = modelPOs.stream().map(ModelPO::getSchemaId).collect(Collectors.toList()); + + Map catalogIdAndNameMap = getCatalogIdAndNameMap(catalogIds); + Map schemaIdAndNameMap = getSchemaIdAndNameMap(schemaIds); + + HashMap modelIdAndNameMap = new HashMap<>(); + + modelPOs.forEach( + modelPO -> { + // since the catalog or schema can be deleted, we need to check the null value, + // and when catalog or schema is deleted, we will set fullName of modelPO to null + String catalogName = catalogIdAndNameMap.getOrDefault(modelPO.getCatalogId(), null); + if (catalogName == null) { + LOG.warn("The catalog of model {} may be deleted", modelPO.getModelId()); + modelIdAndNameMap.put(modelPO.getModelId(), null); + return; + } + + String schemaName = schemaIdAndNameMap.getOrDefault(modelPO.getSchemaId(), null); + if (schemaName == null) { + LOG.warn("The schema of model {} may be deleted", modelPO.getModelId()); + modelIdAndNameMap.put(modelPO.getModelId(), null); + return; + } + + String fullName = DOT_JOINER.join(catalogName, schemaName, modelPO.getModelName()); + modelIdAndNameMap.put(modelPO.getModelId(), fullName); + }); + + return modelIdAndNameMap; + } + public static Map getTableObjectFullNames(List ids) { List tablePOs = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 6de91be1c77..3c473c1804a 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -367,6 +367,7 @@ private static List listSecurableObjects(RolePO po) { case CATALOG: case FILESET: case TABLE: + case MODEL: List objectIds = objects.stream() .map(SecurableObjectPO::getMetadataObjectId) @@ -375,14 +376,16 @@ private static List listSecurableObjects(RolePO po) { Map, Map>> objectFullNameGetterFnMap = ImmutableMap.of( - MetadataObject.Type.FILESET, - MetadataObjectService::getFilesetObjectFullNames, MetadataObject.Type.METALAKE, - MetadataObjectService::getMetalakeObjectFullNames, + MetadataObjectService::getMetalakeObjectFullNames, MetadataObject.Type.CATALOG, - MetadataObjectService::getCatalogObjectFullNames, + MetadataObjectService::getCatalogObjectFullNames, + MetadataObject.Type.FILESET, + MetadataObjectService::getFilesetObjectFullNames, MetadataObject.Type.TABLE, - MetadataObjectService::getTableObjectFullNames); + MetadataObjectService::getTableObjectFullNames, + MetadataObject.Type.MODEL, + MetadataObjectService::getModelObjectFullNames); // dynamically calling getter function based on type Map objectIdAndNameMap = diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java index 9be4692917b..2d68fd6bf6d 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java @@ -34,6 +34,7 @@ import org.apache.gravitino.meta.BaseMetalake; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.meta.FilesetEntity; +import org.apache.gravitino.meta.ModelEntity; import org.apache.gravitino.meta.RoleEntity; import org.apache.gravitino.meta.SchemaEntity; import org.apache.gravitino.meta.TableEntity; @@ -94,6 +95,17 @@ public void testAllTypeSecurableObjects() throws IOException { auditInfo); backend.insert(topic, false); + ModelEntity model = + createModelEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake", "catalog", "schema"), + "model", + "model comment", + 1, + ImmutableMap.of("key", "value"), + auditInfo); + backend.insert(model, false); + SecurableObject metalakeObject = SecurableObjects.ofMetalake( metalake.name(), Lists.newArrayList(Privileges.UseCatalog.allow())); @@ -119,6 +131,9 @@ public void testAllTypeSecurableObjects() throws IOException { SecurableObject topicObject = SecurableObjects.ofTopic( schemaObject, "topic", Lists.newArrayList(Privileges.ConsumeTopic.deny())); + SecurableObject modelObject = + SecurableObjects.ofModel( + schemaObject, "model", Lists.newArrayList(Privileges.ConsumeTopic.deny())); ArrayList securableObjects = Lists.newArrayList( @@ -128,6 +143,7 @@ public void testAllTypeSecurableObjects() throws IOException { schemaObject, tableObject, filesetObject, + modelObject, topicObject); securableObjects.sort(Comparator.comparing(MetadataObject::fullName)); From 1b8dd75ff33219ff8aa125e27b41ac2ea8bf396c Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Wed, 5 Mar 2025 10:41:37 +0800 Subject: [PATCH 06/16] docs(SecurableObject.ofFileset): improve doc --- .../authorization/SecurableObjects.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java b/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java index 06f7ac95b97..6e68d3c1e97 100644 --- a/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java +++ b/api/src/main/java/org/apache/gravitino/authorization/SecurableObjects.java @@ -88,6 +88,22 @@ public static SecurableObject ofTable( return of(MetadataObject.Type.TABLE, names, privileges); } + /** + * Create the column {@link SecurableObject} with the given securable schema object, column name + * and privileges. + * + * @param table The table securable object + * @param column The column name + * @param privileges The privileges of the column + * @return The created column {@link SecurableObject} + */ + public static SecurableObject ofColumn( + SecurableObject table, String column, List privileges) { + List names = Lists.newArrayList(DOT_SPLITTER.splitToList(table.fullName())); + names.add(column); + return of(MetadataObject.Type.TABLE, names, privileges); + } + /** * Create the topic {@link SecurableObject} with the given securable schema object ,topic name and * privileges. @@ -105,7 +121,7 @@ public static SecurableObject ofTopic( } /** - * Create the table {@link SecurableObject} with the given securable schema object, fileset name + * Create the fileset {@link SecurableObject} with the given securable schema object, fileset name * and privileges. * * @param schema The schema securable object From 394c93059cdf319269dd2c384b881a10a194c28f Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Thu, 6 Mar 2025 20:48:12 +0800 Subject: [PATCH 07/16] feat(listSecurableObjects): add support for TOPIC batch retrieval --- .../relational/mapper/TopicMetaMapper.java | 3 + .../mapper/TopicMetaSQLProviderFactory.java | 5 + .../base/TopicMetaBaseSQLProvider.java | 19 +++ .../service/MetadataObjectService.java | 116 +++++++++++++++++- .../relational/service/RoleMetaService.java | 5 +- .../service/TestSecurableObjects.java | 6 + 6 files changed, 148 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java index 8c194caff4f..014ef229ee6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java @@ -40,6 +40,9 @@ public interface TopicMetaMapper { @SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsBySchemaId") List listTopicPOsBySchemaId(@Param("schemaId") Long schemaId); + @SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsByTopicIds") + List listTopicPOsByTopicIds(@Param("topicIds") List topicIds); + @SelectProvider( type = TopicMetaSQLProviderFactory.class, method = "selectTopicMetaBySchemaIdAndName") diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java index 39258c58ddf..dbbfc216044 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java @@ -20,6 +20,7 @@ package org.apache.gravitino.storage.relational.mapper; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; import org.apache.gravitino.storage.relational.mapper.provider.base.TopicMetaBaseSQLProvider; @@ -63,6 +64,10 @@ public static String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) { return getProvider().listTopicPOsBySchemaId(schemaId); } + public static String listTopicPOsByTopicIds(@Param("topicIds") List topicIds) { + return getProvider().listTopicPOsByTopicIds(topicIds); + } + public static String selectTopicMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String topicName) { return getProvider().selectTopicMetaBySchemaIdAndName(schemaId, topicName); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TopicMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TopicMetaBaseSQLProvider.java index 34394780e46..f6a9ef0d616 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TopicMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TopicMetaBaseSQLProvider.java @@ -21,6 +21,7 @@ import static org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TABLE_NAME; +import java.util.List; import org.apache.gravitino.storage.relational.po.TopicPO; import org.apache.ibatis.annotations.Param; @@ -90,6 +91,24 @@ public String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) { + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; } + public String listTopicPOsByTopicIds(@Param("topicIds") List topicIds) { + return ""; + } + public String selectTopicMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String topicName) { return "SELECT topic_id as topicId, topic_name as topicName," diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index 0309589187b..7a539f42f89 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -31,7 +31,9 @@ import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TableColumnMapper; import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; import org.apache.gravitino.storage.relational.po.CatalogPO; import org.apache.gravitino.storage.relational.po.ColumnPO; import org.apache.gravitino.storage.relational.po.FilesetPO; @@ -386,6 +388,103 @@ public static Map getTableObjectFullNames(List ids) { return tableIdAndNameMap; } + public static Map getTopicObjectFullNames(List ids) { + List topicPOs = + SessionUtils.getWithoutCommit( + TopicMetaMapper.class, mapper -> mapper.listTopicPOsByTopicIds(ids)); + + if (topicPOs == null || topicPOs.isEmpty()) { + return new HashMap<>(); + } + + List catalogIds = + topicPOs.stream().map(TopicPO::getCatalogId).collect(Collectors.toList()); + List schemaIds = topicPOs.stream().map(TopicPO::getSchemaId).collect(Collectors.toList()); + + Map catalogIdAndNameMap = getCatalogIdAndNameMap(catalogIds); + Map schemaIdAndNameMap = getSchemaIdAndNameMap(schemaIds); + + HashMap topicIdAndNameMap = new HashMap<>(); + + topicPOs.forEach( + tablePO -> { + // since the catalog or schema can be deleted, we need to check the null value, + // and when catalog or schema is deleted, we will set fullName of tablePO to null + String catalogName = catalogIdAndNameMap.getOrDefault(tablePO.getCatalogId(), null); + if (catalogName == null) { + LOG.warn("The catalog of topic {} may be deleted", tablePO.getTopicId()); + topicIdAndNameMap.put(tablePO.getTopicId(), null); + return; + } + + String schemaName = schemaIdAndNameMap.getOrDefault(tablePO.getSchemaId(), null); + if (schemaName == null) { + LOG.warn("The schema of topic {} may be deleted", tablePO.getTopicId()); + topicIdAndNameMap.put(tablePO.getTopicId(), null); + return; + } + + String fullName = DOT_JOINER.join(catalogName, schemaName, tablePO.getTopicName()); + topicIdAndNameMap.put(tablePO.getTopicId(), fullName); + }); + + return topicIdAndNameMap; + } + + public static Map getColumnObjectFullNames(List ids) { + List columnPOs = + SessionUtils.getWithoutCommit( + TableColumnMapper.class, mapper -> mapper.listColumnPOsByColumnIds(ids)); + + if (columnPOs == null || columnPOs.isEmpty()) { + return new HashMap<>(); + } + + List catalogIds = + columnPOs.stream().map(ColumnPO::getCatalogId).collect(Collectors.toList()); + List schemaIds = + columnPOs.stream().map(ColumnPO::getSchemaId).collect(Collectors.toList()); + List tableIds = columnPOs.stream().map(ColumnPO::getTableId).collect(Collectors.toList()); + + Map catalogIdAndNameMap = getCatalogIdAndNameMap(catalogIds); + Map schemaIdAndNameMap = getSchemaIdAndNameMap(schemaIds); + Map tableIdAndNameMap = getTableIdAndNameMap(tableIds); + + HashMap columnIdAndNameMap = new HashMap<>(); + + columnPOs.forEach( + columnPO -> { + // since the catalog or schema can be deleted, we need to check the null value, + // and when catalog or schema is deleted, we will set fullName of filesetPO to null + String catalogName = catalogIdAndNameMap.getOrDefault(columnPO.getCatalogId(), null); + if (catalogName == null) { + LOG.warn("The catalog of column {} may be deleted", columnPO.getColumnId()); + columnIdAndNameMap.put(columnPO.getColumnId(), null); + return; + } + + String schemaName = schemaIdAndNameMap.getOrDefault(columnPO.getSchemaId(), null); + if (schemaName == null) { + LOG.warn("The schema of column {} may be deleted", columnPO.getColumnId()); + columnIdAndNameMap.put(columnPO.getColumnId(), null); + return; + } + + String tableName = tableIdAndNameMap.getOrDefault(columnPO.getTableId(), null); + if (tableName == null) { + LOG.warn("The table of column {} may be deleted", columnPO.getColumnId()); + columnIdAndNameMap.put(columnPO.getColumnId(), null); + return; + } + + String fullName = + DOT_JOINER.join(catalogName, schemaName, tableName, columnPO.getColumnName()); + columnIdAndNameMap.put(columnPO.getColumnId(), fullName); + }); + + return columnIdAndNameMap; + } + public static Map getCatalogObjectFullNames(List ids) { List catalogPOs = SessionUtils.getWithoutCommit( @@ -409,6 +508,14 @@ public static Map getCatalogObjectFullNames(List ids) { return catalogIdAndNameMap; } + public static Map getCatalogIdAndNameMap(List catalogIds) { + List catalogPOs = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByCatalogIds(catalogIds)); + return catalogPOs.stream() + .collect(Collectors.toMap(CatalogPO::getCatalogId, CatalogPO::getCatalogName)); + } + public static Map getSchemaIdAndNameMap(List schemaIds) { List schemaPOS = SessionUtils.getWithoutCommit( @@ -417,11 +524,10 @@ public static Map getSchemaIdAndNameMap(List schemaIds) { .collect(Collectors.toMap(SchemaPO::getSchemaId, SchemaPO::getSchemaName)); } - public static Map getCatalogIdAndNameMap(List catalogIds) { - List catalogPOs = + public static Map getTableIdAndNameMap(List tableIds) { + List tablePOS = SessionUtils.getWithoutCommit( - CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByCatalogIds(catalogIds)); - return catalogPOs.stream() - .collect(Collectors.toMap(CatalogPO::getCatalogId, CatalogPO::getCatalogName)); + TableMetaMapper.class, mapper -> mapper.listTablePOsByTableIds(tableIds)); + return tablePOS.stream().collect(Collectors.toMap(TablePO::getTableId, TablePO::getTableName)); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 3c473c1804a..c25b98e1bb6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -368,6 +368,7 @@ private static List listSecurableObjects(RolePO po) { case FILESET: case TABLE: case MODEL: + case TOPIC: List objectIds = objects.stream() .map(SecurableObjectPO::getMetadataObjectId) @@ -385,7 +386,9 @@ private static List listSecurableObjects(RolePO po) { MetadataObject.Type.TABLE, MetadataObjectService::getTableObjectFullNames, MetadataObject.Type.MODEL, - MetadataObjectService::getModelObjectFullNames); + MetadataObjectService::getModelObjectFullNames, + MetadataObject.Type.TOPIC, + MetadataObjectService::getTopicObjectFullNames); // dynamically calling getter function based on type Map objectIdAndNameMap = diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java index 2d68fd6bf6d..cb1919a03b0 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestSecurableObjects.java @@ -80,6 +80,7 @@ public void testAllTypeSecurableObjects() throws IOException { "fileset", auditInfo); backend.insert(fileset, false); + TableEntity table = createTableEntity( RandomIdGenerator.INSTANCE.nextId(), @@ -87,6 +88,7 @@ public void testAllTypeSecurableObjects() throws IOException { "table", auditInfo); backend.insert(table, false); + TopicEntity topic = createTopicEntity( RandomIdGenerator.INSTANCE.nextId(), @@ -122,15 +124,19 @@ public void testAllTypeSecurableObjects() throws IOException { SecurableObject schemaObject = SecurableObjects.ofSchema( catalogObject, "schema", Lists.newArrayList(Privileges.UseSchema.allow())); + SecurableObject tableObject = SecurableObjects.ofTable( schemaObject, "table", Lists.newArrayList(Privileges.SelectTable.allow())); + SecurableObject filesetObject = SecurableObjects.ofFileset( schemaObject, "fileset", Lists.newArrayList(Privileges.ReadFileset.allow())); + SecurableObject topicObject = SecurableObjects.ofTopic( schemaObject, "topic", Lists.newArrayList(Privileges.ConsumeTopic.deny())); + SecurableObject modelObject = SecurableObjects.ofModel( schemaObject, "model", Lists.newArrayList(Privileges.ConsumeTopic.deny())); From 76a7051e024619ca57acf3a01a3c7eb4c4a4e8b6 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Thu, 6 Mar 2025 20:52:42 +0800 Subject: [PATCH 08/16] refactor(listSecurableObjects): remove switch case --- .../relational/service/RoleMetaService.java | 117 +++++++----------- 1 file changed, 44 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index c25b98e1bb6..8903cbdb3ae 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -360,79 +360,50 @@ private static List listSecurableObjects(RolePO po) { .collect(Collectors.groupingBy(SecurableObjectPO::getType)) .forEach( (type, objects) -> { - // If the type is Fileset, use the batch retrieval interface; - // otherwise, use the single retrieval interface - switch (MetadataObject.Type.valueOf(type)) { - case METALAKE: - case CATALOG: - case FILESET: - case TABLE: - case MODEL: - case TOPIC: - List objectIds = - objects.stream() - .map(SecurableObjectPO::getMetadataObjectId) - .collect(Collectors.toList()); - - Map, Map>> - objectFullNameGetterFnMap = - ImmutableMap.of( - MetadataObject.Type.METALAKE, - MetadataObjectService::getMetalakeObjectFullNames, - MetadataObject.Type.CATALOG, - MetadataObjectService::getCatalogObjectFullNames, - MetadataObject.Type.FILESET, - MetadataObjectService::getFilesetObjectFullNames, - MetadataObject.Type.TABLE, - MetadataObjectService::getTableObjectFullNames, - MetadataObject.Type.MODEL, - MetadataObjectService::getModelObjectFullNames, - MetadataObject.Type.TOPIC, - MetadataObjectService::getTopicObjectFullNames); - - // dynamically calling getter function based on type - Map objectIdAndNameMap = - Optional.of(MetadataObject.Type.valueOf(type)) - .map(objectFullNameGetterFnMap::get) - .map(getter -> getter.apply(objectIds)) - .orElseThrow( - () -> - new IllegalArgumentException( - "Unsupported metadata object type: " + type)); - - for (SecurableObjectPO securableObjectPO : objects) { - String fullName = - objectIdAndNameMap.get(securableObjectPO.getMetadataObjectId()); - if (fullName != null) { - securableObjects.add( - POConverters.fromSecurableObjectPO( - fullName, securableObjectPO, getType(securableObjectPO.getType()))); - } else { - LOG.warn( - "The securable object {} {} may be deleted", - securableObjectPO.getMetadataObjectId(), - securableObjectPO.getType()); - } - } - break; - default: - - // todo：to get other securable object fullNames using batch retrieving - for (SecurableObjectPO securableObjectPO : objects) { - String fullName = - MetadataObjectService.getMetadataObjectFullName( - securableObjectPO.getType(), securableObjectPO.getMetadataObjectId()); - if (fullName != null) { - securableObjects.add( - POConverters.fromSecurableObjectPO( - fullName, securableObjectPO, getType(securableObjectPO.getType()))); - } else { - LOG.warn( - "The securable object {} {} may be deleted", - securableObjectPO.getMetadataObjectId(), - securableObjectPO.getType()); - } - } + List objectIds = + objects.stream() + .map(SecurableObjectPO::getMetadataObjectId) + .collect(Collectors.toList()); + + Map, Map>> + objectFullNameGetterFnMap = + ImmutableMap.of( + MetadataObject.Type.METALAKE, + MetadataObjectService::getMetalakeObjectFullNames, + MetadataObject.Type.CATALOG, + MetadataObjectService::getCatalogObjectFullNames, + MetadataObject.Type.FILESET, + MetadataObjectService::getFilesetObjectFullNames, + MetadataObject.Type.TABLE, + MetadataObjectService::getTableObjectFullNames, + MetadataObject.Type.MODEL, + MetadataObjectService::getModelObjectFullNames, + MetadataObject.Type.TOPIC, + MetadataObjectService::getTopicObjectFullNames); + + // dynamically calling getter function based on type + Map objectIdAndNameMap = + Optional.of(MetadataObject.Type.valueOf(type)) + .map(objectFullNameGetterFnMap::get) + .map(getter -> getter.apply(objectIds)) + .orElseThrow( + () -> + // For example: COLUMN doesn't have securable object + new IllegalArgumentException( + "Unsupported metadata object type: " + type)); + + for (SecurableObjectPO securableObjectPO : objects) { + String fullName = objectIdAndNameMap.get(securableObjectPO.getMetadataObjectId()); + if (fullName != null) { + securableObjects.add( + POConverters.fromSecurableObjectPO( + fullName, securableObjectPO, getType(securableObjectPO.getType()))); + } else { + LOG.warn( + "The securable object {} {} may be deleted", + securableObjectPO.getMetadataObjectId(), + securableObjectPO.getType()); + } } }); From b1e8e5588865d12b11a82f98a618da6d70d5e260 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Thu, 6 Mar 2025 20:58:06 +0800 Subject: [PATCH 09/16] fix(MetadataObjectService): fix wrong key in getCatalogObjectFullNames: catalogIdAndNameMap --- .../storage/relational/service/MetadataObjectService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index 7a539f42f89..e0db76cd26b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -499,7 +499,7 @@ public static Map getCatalogObjectFullNames(List ids) { catalogPOs.forEach( catalogPO -> { if (catalogPO.getCatalogId() == null) { - catalogIdAndNameMap.put(catalogPO.getMetalakeId(), null); + catalogIdAndNameMap.put(catalogPO.getCatalogId(), null); return; } catalogIdAndNameMap.put(catalogPO.getCatalogId(), catalogPO.getCatalogName()); From e8ca72dfdd595cf659c80d32e7342831812ca955 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Thu, 6 Mar 2025 21:27:38 +0800 Subject: [PATCH 10/16] feat(listSecurableObjects): add batch retrieval for SCHEMA --- .../service/MetadataObjectService.java | 38 +++++++++++++++++++ .../relational/service/RoleMetaService.java | 6 ++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index e0db76cd26b..496ae988165 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -508,6 +508,44 @@ public static Map getCatalogObjectFullNames(List ids) { return catalogIdAndNameMap; } + public static Map getSchemaObjectFullNames(List ids) { + List schemaPOs = + SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> mapper.listSchemaPOsBySchemaIds(ids)); + + if (schemaPOs == null || schemaPOs.isEmpty()) { + return new HashMap<>(); + } + + List catalogIds = + schemaPOs.stream().map(SchemaPO::getCatalogId).collect(Collectors.toList()); + + Map catalogIdAndNameMap = getCatalogIdAndNameMap(catalogIds); + + HashMap schemaIdAndNameMap = new HashMap<>(); + + schemaPOs.forEach( + schemaPO -> { + if (schemaPO.getSchemaId() == null) { + schemaIdAndNameMap.put(schemaPO.getSchemaId(), null); + return; + } + + String catalogName = catalogIdAndNameMap.getOrDefault(schemaPO.getCatalogId(), null); + if (catalogName == null) { + LOG.warn("The catalog of schema {} may be deleted", schemaPO.getSchemaId()); + schemaIdAndNameMap.put(schemaPO.getSchemaId(), null); + return; + } + + String fullName = DOT_JOINER.join(catalogName, schemaPO.getSchemaName()); + + schemaIdAndNameMap.put(schemaPO.getSchemaId(), fullName); + }); + + return schemaIdAndNameMap; + } + public static Map getCatalogIdAndNameMap(List catalogIds) { List catalogPOs = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 8903cbdb3ae..e3a043e5dd6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -372,10 +372,12 @@ private static List listSecurableObjects(RolePO po) { MetadataObjectService::getMetalakeObjectFullNames, MetadataObject.Type.CATALOG, MetadataObjectService::getCatalogObjectFullNames, - MetadataObject.Type.FILESET, - MetadataObjectService::getFilesetObjectFullNames, + MetadataObject.Type.SCHEMA, + MetadataObjectService::getSchemaObjectFullNames, MetadataObject.Type.TABLE, MetadataObjectService::getTableObjectFullNames, + MetadataObject.Type.FILESET, + MetadataObjectService::getFilesetObjectFullNames, MetadataObject.Type.MODEL, MetadataObjectService::getModelObjectFullNames, MetadataObject.Type.TOPIC, From 43018c25609326ab2af631451581c997b8d1d286 Mon Sep 17 00:00:00 2001 From: Eric Chang Date: Fri, 7 Mar 2025 09:00:38 +0800 Subject: [PATCH 11/16] feat(listSecurableObjects): add support for listing column object full names --- .../relational/mapper/TableColumnMapper.java | 3 +++ .../mapper/TableColumnSQLProviderFactory.java | 4 ++++ .../base/TableColumnBaseSQLProvider.java | 22 +++++++++++++++++++ .../relational/service/RoleMetaService.java | 2 ++ 4 files changed, 31 insertions(+) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java index 87b38ea482c..5cb72de4ff1 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnMapper.java @@ -36,6 +36,9 @@ public interface TableColumnMapper { List listColumnPOsByTableIdAndVersion( @Param("tableId") Long tableId, @Param("tableVersion") Long tableVersion); + @SelectProvider(type = TableColumnSQLProviderFactory.class, method = "listColumnPOsByColumnIds") + List listColumnPOsByColumnIds(@Param("columnIds") List columnIds); + @InsertProvider(type = TableColumnSQLProviderFactory.class, method = "insertColumnPOs") void insertColumnPOs(@Param("columnPOs") List columnPOs); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java index 11f0d5419f1..7f42af45820 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableColumnSQLProviderFactory.java @@ -57,6 +57,10 @@ public static String listColumnPOsByTableIdAndVersion( return getProvider().listColumnPOsByTableIdAndVersion(tableId, tableVersion); } + public static String listColumnPOsByColumnIds(@Param("columnIds") List columnIds) { + return getProvider().listColumnPOsByColumnIds(columnIds); + } + public static String insertColumnPOs(@Param("columnPOs") List columnPOs) { return getProvider().insertColumnPOs(columnPOs); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java index d6154c907b6..7090302706c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java @@ -47,6 +47,28 @@ public String listColumnPOsByTableIdAndVersion( + " ON t1.column_id = t2.column_id AND t1.table_version = t2.max_table_version"; } + public String listColumnPOsByColumnIds(@Param("columnIds") List columnIds) { + return ""; + } + public String insertColumnPOs(@Param("columnPOs") List columnPOs) { return "