Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#6474] improvement(storage): batch listing securable objects in RoleMetaService #6601

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,22 @@ public static SecurableObject ofTable(
return of(MetadataObject.Type.TABLE, names, privileges);
}

/**
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Column isn't the securable object. So you should remove this.

* Create the column {@link SecurableObject} with the given securable schema object, column name
* and privileges.
*
* @param table The table securable object
* @param column The column name
* @param privileges The privileges of the column
* @return The created column {@link SecurableObject}
*/
public static SecurableObject ofColumn(
SecurableObject table, String column, List<Privilege> privileges) {
List<String> names = Lists.newArrayList(DOT_SPLITTER.splitToList(table.fullName()));
names.add(column);
return of(MetadataObject.Type.TABLE, names, privileges);
}

/**
* Create the topic {@link SecurableObject} with the given securable schema object ,topic name and
* privileges.
Expand All @@ -105,7 +121,7 @@ public static SecurableObject ofTopic(
}

/**
* Create the table {@link SecurableObject} with the given securable schema object, fileset name
* Create the fileset {@link SecurableObject} with the given securable schema object, fileset name
* and privileges.
*
* @param schema The schema securable object
Expand All @@ -120,6 +136,22 @@ public static SecurableObject ofFileset(
return of(MetadataObject.Type.FILESET, names, privileges);
}

/**
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't support the privilege of the model now. You can remove this temporaly.

* Create the table {@link SecurableObject} with the given securable schema object, fileset name
* and privileges.
*
* @param schema The schema securable object
* @param model The model name
* @param privileges The privileges of the model
* @return The created model {@link SecurableObject}
*/
public static SecurableObject ofModel(
SecurableObject schema, String model, List<Privilege> privileges) {
List<String> names = Lists.newArrayList(DOT_SPLITTER.splitToList(schema.fullName()));
names.add(model);
return of(MetadataObject.Type.MODEL, names, privileges);
}

private static class SecurableObjectImpl extends MetadataObjectImpl implements SecurableObject {

private List<Privilege> privileges;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ public interface MetalakeMetaMapper {
@SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeMetaById")
MetalakePO selectMetalakeMetaById(@Param("metalakeId") Long metalakeId);

@SelectProvider(
type = MetalakeMetaSQLProviderFactory.class,
method = "listMetalakePOsByMetalakeIds")
List<MetalakePO> listMetalakePOsByMetalakeIds(@Param("metalakeIds") List<Long> metalakeIds);

@SelectProvider(
type = MetalakeMetaSQLProviderFactory.class,
method = "selectMetalakeIdMetaByName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.MetalakeMetaBaseSQLProvider;
Expand Down Expand Up @@ -68,6 +69,10 @@ public static String selectMetalakeIdMetaByName(@Param("metalakeName") String me
return getProvider().selectMetalakeIdMetaByName(metalakeName);
}

public static String listMetalakePOsByMetalakeIds(@Param("metalakeIds") List<Long> metalakeIds) {
return getProvider().listMetalakePOsByMetalakeIds(metalakeIds);
}

public static String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) {
return getProvider().insertMetalakeMeta(metalakePO);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public interface ModelMetaMapper {
@SelectProvider(type = ModelMetaSQLProviderFactory.class, method = "listModelPOsBySchemaId")
List<ModelPO> listModelPOsBySchemaId(@Param("schemaId") Long schemaId);

@SelectProvider(type = ModelMetaSQLProviderFactory.class, method = "listModelPOsByModelIds")
List<ModelPO> listModelPOsByModelIds(@Param("modelIds") List<Long> modelIds);

@SelectProvider(
type = ModelMetaSQLProviderFactory.class,
method = "selectModelMetaBySchemaIdAndModelName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.ModelMetaBaseSQLProvider;
Expand Down Expand Up @@ -62,6 +63,10 @@ public static String listModelPOsBySchemaId(@Param("schemaId") Long schemaId) {
return getProvider().listModelPOsBySchemaId(schemaId);
}

public static String listModelPOsByModelIds(@Param("modelIds") List<Long> modelIds) {
return getProvider().listModelPOsByModelIds(modelIds);
}

public static String selectModelMetaBySchemaIdAndModelName(
@Param("schemaId") Long schemaId, @Param("modelName") String modelName) {
return getProvider().selectModelMetaBySchemaIdAndModelName(schemaId, modelName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ public interface TableColumnMapper {
List<ColumnPO> listColumnPOsByTableIdAndVersion(
@Param("tableId") Long tableId, @Param("tableVersion") Long tableVersion);

@SelectProvider(type = TableColumnSQLProviderFactory.class, method = "listColumnPOsByColumnIds")
List<ColumnPO> listColumnPOsByColumnIds(@Param("columnIds") List<Long> columnIds);

@InsertProvider(type = TableColumnSQLProviderFactory.class, method = "insertColumnPOs")
void insertColumnPOs(@Param("columnPOs") List<ColumnPO> columnPOs);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ public static String listColumnPOsByTableIdAndVersion(
return getProvider().listColumnPOsByTableIdAndVersion(tableId, tableVersion);
}

public static String listColumnPOsByColumnIds(@Param("columnIds") List<Long> columnIds) {
return getProvider().listColumnPOsByColumnIds(columnIds);
}

public static String insertColumnPOs(@Param("columnPOs") List<ColumnPO> columnPOs) {
return getProvider().insertColumnPOs(columnPOs);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ public interface TableMetaMapper {
@SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsBySchemaId")
List<TablePO> listTablePOsBySchemaId(@Param("schemaId") Long schemaId);

@SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsByTableIds")
List<TablePO> listTablePOsByTableIds(@Param("tableIds") List<Long> tableIds);

@SelectProvider(
type = TableMetaSQLProviderFactory.class,
method = "selectTableIdBySchemaIdAndName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.TableMetaBaseSQLProvider;
Expand Down Expand Up @@ -54,6 +55,10 @@ public static String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) {
return getProvider().listTablePOsBySchemaId(schemaId);
}

public static String listTablePOsByTableIds(@Param("tableIds") List<Long> tableIds) {
return getProvider().listTablePOsByTableIds(tableIds);
}

public static String selectTableIdBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("tableName") String name) {
return getProvider().selectTableIdBySchemaIdAndName(schemaId, name);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public interface TopicMetaMapper {
@SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsBySchemaId")
List<TopicPO> listTopicPOsBySchemaId(@Param("schemaId") Long schemaId);

@SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsByTopicIds")
List<TopicPO> listTopicPOsByTopicIds(@Param("topicIds") List<Long> topicIds);

@SelectProvider(
type = TopicMetaSQLProviderFactory.class,
method = "selectTopicMetaBySchemaIdAndName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.TopicMetaBaseSQLProvider;
Expand Down Expand Up @@ -63,6 +64,10 @@ public static String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) {
return getProvider().listTopicPOsBySchemaId(schemaId);
}

public static String listTopicPOsByTopicIds(@Param("topicIds") List<Long> topicIds) {
return getProvider().listTopicPOsByTopicIds(topicIds);
}

public static String selectTopicMetaBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("topicName") String topicName) {
return getProvider().selectTopicMetaBySchemaIdAndName(schemaId, topicName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper.TABLE_NAME;

import java.util.List;
import org.apache.gravitino.storage.relational.po.MetalakePO;
import org.apache.ibatis.annotations.Param;

Expand Down Expand Up @@ -65,6 +66,24 @@ public String selectMetalakeIdMetaByName(@Param("metalakeName") String metalakeN
+ " WHERE metalake_name = #{metalakeName} and deleted_at = 0";
}

public String listMetalakePOsByMetalakeIds(@Param("metalakeIds") List<Long> metalakeIds) {
return "<script>"
+ " SELECT metalake_id as metalakeId, metalake_name as metalakeName,"
+ " metalake_comment as metalakeComment, properties,"
+ " audit_info as auditInfo, schema_version as schemaVersion,"
+ " current_version as currentVersion, last_version as lastVersion,"
+ " deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND metalake_id in ("
+ "<foreach collection='metalakeIds' item='metalakeId' separator=','>"
+ "#{metalakeId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) {
return "INSERT INTO "
+ TABLE_NAME
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.gravitino.storage.relational.mapper.provider.base;

import java.util.List;
import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper;
import org.apache.gravitino.storage.relational.po.ModelPO;
import org.apache.ibatis.annotations.Param;
Expand Down Expand Up @@ -66,6 +67,23 @@ public String listModelPOsBySchemaId(@Param("schemaId") Long schemaId) {
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}

public String listModelPOsByModelIds(List<Long> modelIds) {
return "<script>"
+ " SELECT model_id AS modelId, model_name AS modelName, metalake_id AS metalakeId,"
+ " catalog_id AS catalogId, schema_id AS schemaId, model_comment AS modelComment,"
+ " model_properties AS modelProperties, model_latest_version AS"
+ " modelLatestVersion, audit_info AS auditInfo, deleted_at AS deletedAt"
+ " FROM "
+ ModelMetaMapper.TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND model_id in ("
+ "<foreach collection='modelIds' item='modelId' separator=','>"
+ "#{modelId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String selectModelMetaBySchemaIdAndModelName(
@Param("schemaId") Long schemaId, @Param("modelName") String modelName) {
return "SELECT model_id AS modelId, model_name AS modelName, metalake_id AS metalakeId,"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,28 @@ public String listColumnPOsByTableIdAndVersion(
+ " ON t1.column_id = t2.column_id AND t1.table_version = t2.max_table_version";
}

public String listColumnPOsByColumnIds(@Param("columnIds") List<Long> columnIds) {
return "<script>"
+ " SELECT column_id AS columnId, column_name AS columnName,"
+ " column_position AS columnPosition,"
+ " metalake_id AS metalakeId, catalog_id AS catalogId,"
+ " schema_id AS schemaId, table_id AS tableId,"
+ " table_version AS tableVersion, column_type AS columnType,"
+ " column_comment AS columnComment, column_nullable AS nullable,"
+ " column_auto_increment AS autoIncrement,"
+ " column_default_value AS defaultValue, column_op_type AS columnOpType,"
+ " deleted_at AS deletedAt, audit_info AS auditInfo"
+ " FROM "
+ TableColumnMapper.COLUMN_TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND column_id in ("
+ "<foreach collection='columnIds' item='columnId' separator=','>"
+ "#{columnId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String insertColumnPOs(@Param("columnPOs") List<ColumnPO> columnPOs) {
return "<script>"
+ "INSERT INTO "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TABLE_NAME;

import java.util.List;
import org.apache.gravitino.storage.relational.po.TablePO;
import org.apache.ibatis.annotations.Param;

Expand All @@ -36,6 +37,24 @@ public String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) {
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}

public String listTablePOsByTableIds(List<Long> tableIds) {
return "<script>"
+ " SELECT table_id as tableId, table_name as tableName,"
+ " metalake_id as metalakeId, catalog_id as catalogId,"
+ " schema_id as schemaId, audit_info as auditInfo,"
+ " current_version as currentVersion, last_version as lastVersion,"
+ " deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND table_id in ("
+ "<foreach collection='tableIds' item='tableId' separator=','>"
+ "#{tableId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String selectTableIdBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("tableName") String name) {
return "SELECT table_id as tableId FROM "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import static org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TABLE_NAME;

import java.util.List;
import org.apache.gravitino.storage.relational.po.TopicPO;
import org.apache.ibatis.annotations.Param;

Expand Down Expand Up @@ -90,6 +91,24 @@ public String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) {
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}

public String listTopicPOsByTopicIds(@Param("topicIds") List<Long> topicIds) {
return "<script>"
+ " SELECT topic_id as topicId, topic_name as topicName, metalake_id as metalakeId,"
+ " catalog_id as catalogId, schema_id as schemaId,"
+ " comment as comment, properties as properties, audit_info as auditInfo,"
+ " current_version as currentVersion, last_version as lastVersion,"
+ " deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND topic_id in ("
+ "<foreach collection='topicIds' item='topicId' separator=','>"
+ "#{topicId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String selectTopicMetaBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("topicName") String topicName) {
return "SELECT topic_id as topicId, topic_name as topicName,"
Expand Down
Loading