diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 54881fb8293d..1ff5fba73b26 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -761,6 +761,104 @@ public void testConcurrentAutoCreateAndDropColumn() throws Exception { } } + @Test + public void testTableObjectCheck() throws Exception { + final Set illegal = new HashSet<>(Arrays.asList("./", ".", "..", ".\\", "../hack")); + for (final String single : illegal) { + testObject4SingleIllegalPath(single); + } + } + + private void testObject4SingleIllegalPath(final String illegal) throws Exception { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database db2"); + statement.execute("use db2"); + statement.execute(String.format("create table \"%s\" ()", illegal)); + + try { + statement.execute(String.format("alter table \"%s\" add column a object", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the tableName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + try { + statement.execute(String.format("create table test (\"%s\" object)", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + statement.execute("create table test (a tag, b attribute, c int32, d object)"); + try { + statement.execute(String.format("insert into test (a, b, c) values ('%s', 1, 1)", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "507: When there are object fields, the deviceId [%s] shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + try { + statement.execute(String.format("alter table test add column \"%s\" object", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + } + + // Test cache + TestUtils.restartCluster(EnvFactory.getEnv()); + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("use db2"); + + try { + statement.execute(String.format("insert into test (a, b, c) values ('%s', 1, 1)", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "507: When there are object fields, the deviceId [%s] shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + statement.execute("alter table test drop column d"); + statement.execute(String.format("insert into test (a, b, c) values ('%s', 1, 1)", illegal)); + try { + statement.execute("alter table test add column d object"); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the tag value %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + statement.execute("drop database db2"); + } + } + @Test public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java index 0d6d439583f9..da48b12bc4a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java @@ -125,6 +125,7 @@ public enum CnToDnAsyncRequestType { DELETE_DATA_FOR_TABLE_DEVICE, DELETE_TABLE_DEVICE_IN_BLACK_LIST, DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, + CHECK_DEVICE_ID_FOR_OBJECT, // audit log and event write-back INSERT_RECORD, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index cde9a174492d..47e66fd76f76 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -50,6 +50,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterEncodingCompressorReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq; +import org.apache.iotdb.mpp.rpc.thrift.TCheckDeviceIdForObjectReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; import org.apache.iotdb.mpp.rpc.thrift.TCleanDataNodeCacheReq; @@ -462,6 +463,11 @@ protected void initActionMapBuilder() { client.deleteTableDeviceInBlackList( (TTableDeviceDeletionWithPatternOrModReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.CHECK_DEVICE_ID_FOR_OBJECT, + (req, client, handler) -> + client.checkDeviceIdForObject( + (TCheckDeviceIdForObjectReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, (req, client, handler) -> diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java index 301bfc71f97d..548f91d7f898 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java @@ -232,6 +232,7 @@ public static DataNodeAsyncRequestRPCHandler buildHandler( case INVALIDATE_MATCHED_TABLE_DEVICE_CACHE: case DELETE_DATA_FOR_TABLE_DEVICE: case DELETE_TABLE_DEVICE_IN_BLACK_LIST: + case CHECK_DEVICE_ID_FOR_OBJECT: default: return new DataNodeTSStatusRPCHandler( requestType, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index bfe6e830cba9..bd8a9e940106 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -110,6 +110,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -123,6 +124,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -1331,9 +1333,14 @@ public synchronized Pair tableColumnCheckForColumnExtension( columnSchemaList.stream() .map(TsTableColumnSchema::getColumnName) .collect(Collectors.joining(", "))); + + final AtomicBoolean hasObject = new AtomicBoolean(false); columnSchemaList.removeIf( columnSchema -> { if (Objects.isNull(originalTable.getColumnSchema(columnSchema.getColumnName()))) { + if (columnSchema.getDataType().equals(TSDataType.OBJECT)) { + hasObject.set(true); + } expandedTable.addColumnSchema(columnSchema); return false; } @@ -1343,7 +1350,16 @@ public synchronized Pair tableColumnCheckForColumnExtension( if (columnSchemaList.isEmpty()) { return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null); } - return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); + + if (hasObject.get()) { + expandedTable.checkTableNameAndObjectNames4Object(); + } + final TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + // Check tag values only if the table does not have any object fields before + if (!originalTable.setNeedCheck4Object()) { + status.setMessage(""); + } + return new Pair<>(status, expandedTable); } public synchronized Pair tableColumnCheckForColumnRenaming( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java index fe92c802ce68..5ed820e2783b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java @@ -31,7 +31,9 @@ import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeTSStatusTaskExecutor; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.external.commons.lang3.function.TriFunction; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +43,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -187,6 +191,10 @@ public void deserialize(final ByteBuffer byteBuffer) { protected class TableRegionTaskExecutor extends DataNodeTSStatusTaskExecutor { private final String taskName; + private final Map failureMap = new HashMap<>(); + private final TriFunction< + TConsensusGroupId, Set, Map, Exception> + exceptionGenerator; protected TableRegionTaskExecutor( final String taskName, @@ -196,26 +204,84 @@ protected TableRegionTaskExecutor( final BiFunction, Q> dataNodeRequestGenerator) { super(env, targetRegionGroup, false, dataNodeRequestType, dataNodeRequestGenerator); this.taskName = taskName; + this.exceptionGenerator = this::getThrowable; + } + + protected TableRegionTaskExecutor( + final String taskName, + final ConfigNodeProcedureEnv env, + final Map targetRegionGroup, + final CnToDnAsyncRequestType dataNodeRequestType, + final BiFunction, Q> dataNodeRequestGenerator, + final TriFunction< + TConsensusGroupId, + Set, + Map, + Exception> + exceptionGenerator) { + super(env, targetRegionGroup, false, dataNodeRequestType, dataNodeRequestGenerator); + this.taskName = taskName; + this.exceptionGenerator = exceptionGenerator; + } + + @Override + protected List processResponseOfOneDataNode( + final TDataNodeLocation dataNodeLocation, + final List consensusGroupIdList, + final TSStatus response) { + final List failedRegionList = new ArrayList<>(); + if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return failedRegionList; + } + + if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + final List subStatus = response.getSubStatus(); + for (int i = 0; i < subStatus.size(); i++) { + if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedRegionList.add(consensusGroupIdList.get(i)); + } + } + } else { + failedRegionList.addAll(consensusGroupIdList); + } + if (!failedRegionList.isEmpty()) { + failureMap.put(dataNodeLocation, response); + } else { + failureMap.remove(dataNodeLocation); + } + return failedRegionList; } @Override protected void onAllReplicasetFailure( final TConsensusGroupId consensusGroupId, final Set dataNodeLocationSet) { + final Exception e = + exceptionGenerator.apply(consensusGroupId, dataNodeLocationSet, failureMap); setFailure( new ProcedureException( - new MetadataException( - String.format( - "[%s] for %s.%s failed when [%s] because failed to execute in all replicaset of %s %s. Failure nodes: %s", - this.getClass().getSimpleName(), - database, - tableName, - taskName, - consensusGroupId.type, - consensusGroupId.id, - dataNodeLocationSet)))); + Objects.nonNull(e) + ? e + : getThrowable(consensusGroupId, dataNodeLocationSet, failureMap))); interruptTask(); } + + protected Exception getThrowable( + final TConsensusGroupId consensusGroupId, + final Set dataNodeLocationSet, + final Map failureMap) { + return new MetadataException( + String.format( + "[%s] for %s.%s failed when [%s] because failed to execute in all replicaset of %s %s. Failure nodes: %s, Failures: %s", + this.getClass().getSimpleName(), + database, + tableName, + taskName, + consensusGroupId.type, + consensusGroupId.id, + dataNodeLocationSet, + failureMap)); + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index d5b99942721b..0a0ca6d26204 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -19,12 +19,15 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.view.AddTableViewColumnPlan; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; @@ -32,6 +35,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.table.view.AddViewColumnProcedure; import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TCheckDeviceIdForObjectReq; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; @@ -41,7 +45,9 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; public class AddTableColumnProcedure @@ -50,6 +56,9 @@ public class AddTableColumnProcedure private static final Logger LOGGER = LoggerFactory.getLogger(AddTableColumnProcedure.class); protected List addedColumnList; + // May be lost when deserialized, but just cause some redundant check + private transient boolean needCheck4Object = false; + public AddTableColumnProcedure(final boolean isGeneratedByPipe) { super(isGeneratedByPipe); } @@ -77,6 +86,12 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final AddTable case PRE_RELEASE: LOGGER.info("Pre release info of table {}.{} when adding column", database, tableName); preRelease(env); + if (needCheck4Object + && table.setNeedCheck4Object() + && !(this instanceof AddViewColumnProcedure)) { + checkObject(env, database, tableName); + } + setNextState(AddTableColumnState.ADD_COLUMN); break; case ADD_COLUMN: LOGGER.info("Add column to table {}.{}", database, tableName); @@ -115,6 +130,7 @@ protected void columnCheck(final ConfigNodeProcedureEnv env) { } table = result.getRight(); setNextState(AddTableColumnState.PRE_RELEASE); + needCheck4Object = status.isSetMessage(); } catch (final MetadataException e) { setFailure(new ProcedureException(e)); } @@ -123,7 +139,52 @@ protected void columnCheck(final ConfigNodeProcedureEnv env) { @Override protected void preRelease(final ConfigNodeProcedureEnv env) { super.preRelease(env); - setNextState(AddTableColumnState.ADD_COLUMN); + } + + private void checkObject( + final ConfigNodeProcedureEnv env, final String database, final String tableName) { + final Map relatedRegionGroup = + env.getConfigManager().getRelatedSchemaRegionGroup4TableModel(database); + + if (!relatedRegionGroup.isEmpty()) { + new TableRegionTaskExecutor<>( + "check deviceId for object", + env, + relatedRegionGroup, + CnToDnAsyncRequestType.CHECK_DEVICE_ID_FOR_OBJECT, + ((dataNodeLocation, consensusGroupIdList) -> + new TCheckDeviceIdForObjectReq(new ArrayList<>(consensusGroupIdList), tableName)), + ((tConsensusGroupId, tDataNodeLocations, failureMap) -> { + final String message = parseStatus(failureMap.values()); + // Shall not be SUCCESS here + return Objects.nonNull(message) + ? new IoTDBException(message, TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + : null; + })) + .execute(); + } + } + + // Success: "" + // All semantic: return last one + // Non-semantic error: return null + private String parseStatus(final Iterable statuses) { + String message = ""; + for (final TSStatus status : statuses) { + if (status.getCode() == TSStatusCode.SEMANTIC_ERROR.getStatusCode()) { + message = status.getMessage(); + } else if (status.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + final String tempMsg = parseStatus(status.getSubStatus()); + if (Objects.isNull(tempMsg)) { + return null; + } else if (!tempMsg.isEmpty()) { + message = tempMsg; + } + } else if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return null; + } + } + return message; } private void addColumn(final ConfigNodeProcedureEnv env) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 3f050b1184f4..9345b1f5b31f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -99,6 +99,7 @@ import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.client.cn.DnToCnInternalServiceAsyncRequestManager; @@ -215,6 +216,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCancelPlanFragmentReq; import org.apache.iotdb.mpp.rpc.thrift.TCancelQueryReq; import org.apache.iotdb.mpp.rpc.thrift.TCancelResp; +import org.apache.iotdb.mpp.rpc.thrift.TCheckDeviceIdForObjectReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateResp; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; @@ -1710,6 +1712,7 @@ public TSStatus updateTable(final TUpdateTableReq req) { case PRE_UPDATE_TABLE: final Pair pair = TsTableInternalRPCUtil.deserializeSingleTsTableWithDatabase(req.getTableInfo()); + pair.getRight().setNeedCheck4Object(); DataNodeTableCache.getInstance().preUpdateTable(pair.left, pair.right, req.oldName); break; case ROLLBACK_UPDATE_TABLE: @@ -1736,14 +1739,14 @@ public TSStatus updateTable(final TUpdateTableReq req) { @Override public TSStatus invalidateTableCache(final TInvalidateTableCacheReq req) { DataNodeSchemaLockManager.getInstance() - .takeWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .takeWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); try { TableDeviceSchemaCache.getInstance() .invalidate(PathUtils.unQualifyDatabaseName(req.getDatabase()), req.getTableName()); return StatusUtils.OK; } finally { DataNodeSchemaLockManager.getInstance() - .releaseWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .releaseWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); } } @@ -1820,7 +1823,7 @@ public TSStatus rollbackTableDeviceBlackList(final TTableDeviceDeletionWithPatte @Override public TSStatus invalidateMatchedTableDeviceCache(final TTableDeviceInvalidateCacheReq req) { DataNodeSchemaLockManager.getInstance() - .takeWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .takeWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); try { TableDeviceSchemaCache.getInstance() .invalidate( @@ -1831,7 +1834,7 @@ public TSStatus invalidateMatchedTableDeviceCache(final TTableDeviceInvalidateCa return StatusUtils.OK; } finally { DataNodeSchemaLockManager.getInstance() - .releaseWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .releaseWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); } } @@ -1927,7 +1930,7 @@ public TDeviceViewResp detectTreeDeviceViewFieldType(final TDeviceViewReq req) { @Override public TSStatus invalidateColumnCache(final TInvalidateColumnCacheReq req) { DataNodeSchemaLockManager.getInstance() - .takeWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .takeWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); try { TableDeviceSchemaCache.getInstance() .invalidate( @@ -1938,7 +1941,7 @@ public TSStatus invalidateColumnCache(final TInvalidateColumnCacheReq req) { return StatusUtils.OK; } finally { DataNodeSchemaLockManager.getInstance() - .releaseWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .releaseWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); } } @@ -1970,6 +1973,31 @@ public TSStatus deleteColumnData(final TDeleteColumnDataReq req) { .getStatus()); } + @Override + public TSStatus checkDeviceIdForObject(final TCheckDeviceIdForObjectReq req) { + // Take the lock to avoid concurrent alter + DataNodeSchemaLockManager.getInstance() + .takeWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); + try { + return executeInternalSchemaTask( + req.getRegionIdList(), + consensusGroupId -> { + final ISchemaRegion schemaRegion = + schemaEngine.getSchemaRegion(new SchemaRegionId(consensusGroupId.getId())); + try { + schemaRegion.checkTableDevice4Object(req.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (final SemanticException | MetadataException e) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage(e.getMessage()); + } + }); + } finally { + DataNodeSchemaLockManager.getInstance() + .releaseWriteLock(SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); + } + } + public TTestConnectionResp submitTestConnectionTask(final TNodeLocations nodeLocations) { return new TTestConnectionResp( new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/SchemaLockType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/SchemaLockType.java index da63e04d430f..8c36d588b070 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/SchemaLockType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/SchemaLockType.java @@ -59,5 +59,5 @@ public enum SchemaLockType { *
  • Release write lock after finishing invalidating schema cache. * */ - VALIDATE_VS_DELETION_TABLE, + AVOID_CONCURRENT_DEVICE_ALTER_TABLE, } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 0475a27c26da..a64f6171567a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.executable.ExecutableManager; import org.apache.iotdb.commons.path.PartialPath; @@ -549,10 +550,12 @@ private Pair parseTable4CreateTableOrView( // TODO: Place the check at statement analyzer boolean hasTimeColumn = false; final Set sourceNameSet = new HashSet<>(); + boolean hasObject = false; for (final ColumnDefinition columnDefinition : node.getElements()) { final TsTableColumnCategory category = columnDefinition.getColumnCategory(); final String columnName = columnDefinition.getName().getValue(); final TSDataType dataType = getDataType(columnDefinition.getType()); + hasObject |= dataType.equals(TSDataType.OBJECT); final String comment = columnDefinition.getComment(); if (checkTimeColumnIdempotent(category, columnName, dataType, comment, table) && !hasTimeColumn) { @@ -581,6 +584,13 @@ private Pair parseTable4CreateTableOrView( } table.addColumnSchema(schema); } + if (hasObject) { + try { + table.checkTableNameAndObjectNames4Object(); + } catch (final MetadataException e) { + throw new SemanticException(e.getMessage(), e.getErrorCode()); + } + } return new Pair<>(database, table); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 71b6f58f83e0..5f530b605fa4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -4273,6 +4273,8 @@ public SettableFuture alterTableAddColumn( || (TSStatusCode.COLUMN_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode() && columnIfExists)) { future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else if (tsStatus.getCode() == TSStatusCode.SEMANTIC_ERROR.getStatusCode()) { + future.setException(new SemanticException(tsStatus.getMessage())); } else { future.setException( new IoTDBException(getTableErrorMessage(tsStatus, database), tsStatus.getCode())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 01b216fe875d..58abdcf69964 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -231,6 +231,7 @@ import org.apache.tsfile.utils.Pair; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -268,6 +269,7 @@ import static java.util.Objects.requireNonNull; import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; +import static org.apache.iotdb.commons.schema.table.TsTable.getObjectStringError; import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinScalarFunction.DATE_BIN; import static org.apache.iotdb.db.queryengine.execution.warnings.StandardWarningCode.REDUNDANT_ORDER_BY; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.AggregationAnalyzer.verifyOrderByAggregations; @@ -4459,9 +4461,21 @@ protected Scope visitCreateOrUpdateDevice( final CreateOrUpdateDevice node, final Optional context) { queryContext.setQueryType(QueryType.WRITE); DataNodeSchemaLockManager.getInstance() - .takeReadLock(queryContext, SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .takeReadLock(queryContext, SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); // Check if the table exists - DataNodeTableCache.getInstance().getTable(node.getDatabase(), node.getTable()); + final TsTable table = + DataNodeTableCache.getInstance().getTable(node.getDatabase(), node.getTable()); + if (table.isNeedCheck4Object()) { + for (final Object[] deviceId : node.getDeviceIdList()) { + for (final Object part : deviceId) { + final String value = (String) part; + if (Objects.nonNull(value) && TsTable.isInvalid4ObjectType(value)) { + throw new SemanticException( + getObjectStringError("deviceId", Arrays.toString(deviceId))); + } + } + } + } return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java index b50a875d7b08..ec0ade6857d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; -import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.session.SessionManager; @@ -247,9 +247,8 @@ private void autoCreateOrUpdateDeviceSchema( Long.MAX_VALUE, false); if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new RuntimeException( - new IoTDBException( - executionResult.status.getMessage(), executionResult.status.getCode())); + throw new IoTDBRuntimeException( + executionResult.status.getMessage(), executionResult.status.getCode()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index 032b0d91cfd0..d36aa88f79bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -98,7 +98,7 @@ public Optional validateTableHeaderSchema( // The schema cache R/W and fetch operation must be locked together thus the cache clean // operation executed by delete timeSeries will be effective. DataNodeSchemaLockManager.getInstance() - .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .takeReadLock(context, SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); final List inputColumnList = tableSchema.getColumns(); if (inputColumnList == null || inputColumnList.isEmpty()) { @@ -275,7 +275,7 @@ private void autoCreateTable( "Auto create table column failed.", result.getStatusCode().getStatusCode())); } DataNodeSchemaLockManager.getInstance() - .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .takeReadLock(context, SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); } catch (final ExecutionException e) { throw new RuntimeException(e); } catch (final InterruptedException e) { @@ -387,7 +387,7 @@ private void autoCreateColumn( result.getStatusCode().getStatusCode())); } DataNodeSchemaLockManager.getInstance() - .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TABLE); + .takeReadLock(context, SchemaLockType.AVOID_CONCURRENT_DEVICE_ALTER_TABLE); } catch (final ExecutionException | InterruptedException e) { LOGGER.warn("Auto add table column failed.", e); throw new RuntimeException(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 916e52b4acab..cef65878a999 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -396,6 +396,8 @@ ISchemaReader getTableDeviceReader(final PartialPath pathPatt ISchemaReader getTableDeviceReader( final String table, final List devicePathList) throws MetadataException; + void checkTableDevice4Object(final String table) throws MetadataException; + // region Interfaces for AttributeUpdate Pair> getAttributeUpdateInfo( final AtomicInteger limit, final AtomicBoolean hasRemaining); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 3a80abba2b8c..52b523d7af56 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1736,6 +1736,11 @@ public ISchemaReader getTableDeviceReader( (pointer, name) -> deviceAttributeStore.getAttributes(pointer, name)); } + @Override + public void checkTableDevice4Object(final String table) throws MetadataException { + mTree.checkTableDevice4Object(table); + } + @Override public Pair> getAttributeUpdateInfo( final AtomicInteger limit, final AtomicBoolean hasRemaining) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 58e8fd786a5a..51cdf2228ee3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1542,6 +1542,11 @@ public ISchemaReader getTableDeviceReader( throw new UnsupportedOperationException(); } + @Override + public void checkTableDevice4Object(String table) { + throw new UnsupportedOperationException(); + } + @Override public Pair> getAttributeUpdateInfo( final AtomicInteger limit, final AtomicBoolean hasRemaining) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 53e8c2617bcb..6446d060b29a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -42,6 +43,7 @@ import org.apache.iotdb.db.exception.metadata.template.DifferentTemplateException; import org.apache.iotdb.db.exception.metadata.template.TemplateIsInUseException; import org.apache.iotdb.db.exception.quota.ExceedQuotaException; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.execution.operator.schema.source.DeviceAttributeUpdater; import org.apache.iotdb.db.queryengine.execution.operator.schema.source.DeviceBlackListConstructor; @@ -51,6 +53,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.Traverser; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.EntityCollector; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MNodeCollector; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MeasurementCollector; @@ -141,7 +144,7 @@ public class MTreeBelowSGMemoryImpl { private final MemMTreeStore store; @SuppressWarnings("java:S3077") - private volatile IMemMNode storageGroupMNode; + private volatile IMemMNode databaseMNode; private final IMemMNode rootNode; private final Function, Map> tagGetter; @@ -160,7 +163,7 @@ public MTreeBelowSGMemoryImpl( final SchemaRegionMemMetric metric) { store = new MemMTreeStore(storageGroupPath, regionStatistics, metric); this.regionStatistics = regionStatistics; - this.storageGroupMNode = store.getRoot(); + this.databaseMNode = store.getRoot(); this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupPath.getNodeLength() - 1; this.tagGetter = tagGetter; @@ -175,7 +178,7 @@ private MTreeBelowSGMemoryImpl( final MemSchemaRegionStatistics regionStatistics) { this.store = store; this.regionStatistics = regionStatistics; - this.storageGroupMNode = store.getRoot(); + this.databaseMNode = store.getRoot(); this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupPath.getNodeLength() - 1; this.tagGetter = tagGetter; @@ -184,7 +187,7 @@ private MTreeBelowSGMemoryImpl( public void clear() { store.clear(); - storageGroupMNode = null; + databaseMNode = null; } public synchronized boolean createSnapshot(final File snapshotDir) { @@ -409,7 +412,7 @@ private IMemMNode checkAndAutoCreateInternalPath(final PartialPath devicePath) if (nodeNames.length == levelOfSG + 1) { return null; } - IMemMNode cur = storageGroupMNode; + IMemMNode cur = databaseMNode; IMemMNode child; String childName; // e.g, path = root.sg.d1.s1, create internal nodes and set cur to sg node, parent of d1 @@ -433,13 +436,12 @@ private IMemMNode checkAndAutoCreateDeviceNode( throws PathAlreadyExistException, ExceedQuotaException { if (deviceParent == null) { // device is sg - return storageGroupMNode; + return databaseMNode; } IMemMNode device = store.getChild(deviceParent, deviceName); if (device == null) { if (IoTDBDescriptor.getInstance().getConfig().isQuotaEnable()) { - if (!DataNodeSpaceQuotaManager.getInstance() - .checkDeviceLimit(storageGroupMNode.getName())) { + if (!DataNodeSpaceQuotaManager.getInstance().checkDeviceLimit(databaseMNode.getName())) { throw new ExceedQuotaException( "The number of devices has reached the upper limit", TSStatusCode.SPACE_QUOTA_EXCEEDED.getStatusCode()); @@ -501,8 +503,7 @@ public Map checkMeasurementExistence( devicePath.getFullPath() + "." + measurementList.get(i), aliasList.get(i))); } if (IoTDBDescriptor.getInstance().getConfig().isQuotaEnable()) { - if (!DataNodeSpaceQuotaManager.getInstance() - .checkTimeSeriesNum(storageGroupMNode.getName())) { + if (!DataNodeSpaceQuotaManager.getInstance().checkTimeSeriesNum(databaseMNode.getName())) { failingMeasurementMap.put( i, new ExceedQuotaException( @@ -747,7 +748,7 @@ public IMemMNode getDeviceNodeWithAutoCreating(final PartialPath deviceId) throws MetadataException { MetaFormatUtils.checkTimeseries(deviceId); final String[] nodeNames = deviceId.getNodes(); - IMemMNode cur = storageGroupMNode; + IMemMNode cur = databaseMNode; IMemMNode child; for (int i = levelOfSG + 1; i < nodeNames.length; i++) { child = cur.getChild(nodeNames[i]); @@ -894,7 +895,7 @@ protected Void collectEntity(final IDeviceMNode node) { */ public IMemMNode getNodeByPath(final PartialPath path) throws PathNotExistException { final String[] nodes = path.getNodes(); - IMemMNode cur = storageGroupMNode; + IMemMNode cur = databaseMNode; IMemMNode next; for (int i = levelOfSG + 1; i < nodes.length; i++) { next = cur.getChild(nodes[i]); @@ -930,7 +931,7 @@ public IMeasurementMNode getMeasurementMNode(final PartialPath path) public void activateTemplate(final PartialPath activatePath, final Template template) throws MetadataException { final String[] nodes = activatePath.getNodes(); - IMemMNode cur = storageGroupMNode; + IMemMNode cur = databaseMNode; for (int i = levelOfSG + 1; i < nodes.length; i++) { cur = cur.getChild(nodes[i]); } @@ -1037,7 +1038,7 @@ protected void updateEntity(final IDeviceMNode node) { public void activateTemplateWithoutCheck( final PartialPath activatePath, final int templateId, final boolean isAligned) { final String[] nodes = activatePath.getNodes(); - IMemMNode cur = storageGroupMNode; + IMemMNode cur = databaseMNode; for (int i = levelOfSG + 1; i < nodes.length; i++) { cur = cur.getChild(nodes[i]); } @@ -1334,18 +1335,16 @@ public void close() {} private IMemMNode getTableDeviceNode(final String table, final Object[] deviceId) throws PathNotExistException { - IMemMNode cur = storageGroupMNode; + IMemMNode cur = databaseMNode; IMemMNode next; next = cur.getChild(table); if (next == null) { throw new PathNotExistException( - storageGroupMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), - true); + databaseMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), true); } else if (next.isMeasurement()) { throw new PathNotExistException( - storageGroupMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), - true); + databaseMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), true); } cur = next; @@ -1353,14 +1352,13 @@ private IMemMNode getTableDeviceNode(final String table, final Object[] deviceId next = cur.getChild(deviceId[i] == null ? null : String.valueOf(deviceId[i])); if (next == null) { throw new PathNotExistException( - storageGroupMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), - true); + databaseMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), true); } else if (next.isMeasurement()) { if (i == deviceId.length - 1) { return next; } else { throw new PathNotExistException( - storageGroupMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), + databaseMNode.getFullPath() + PATH_SEPARATOR + table + Arrays.toString(deviceId), true); } } @@ -1628,7 +1626,7 @@ protected Void collectMeasurement(final IMeasurementMNode node) { // region table device management public int getTableDeviceNotExistNum(final String tableName, final List deviceIdList) { - final IMemMNode tableNode = storageGroupMNode.getChild(tableName); + final IMemMNode tableNode = databaseMNode.getChild(tableName); int notExistNum = deviceIdList.size(); if (tableNode == null) { return notExistNum; @@ -1659,11 +1657,10 @@ public void createOrUpdateTableDevice( if (LOGGER.isDebugEnabled()) { LOGGER.debug("Start to create table device {}.{}", tableName, Arrays.toString(devicePath)); } - IMemMNode cur = storageGroupMNode.getChild(tableName); + IMemMNode cur = databaseMNode.getChild(tableName); if (cur == null) { cur = - store.addChild( - storageGroupMNode, tableName, nodeFactory.createInternalMNode(cur, tableName)); + store.addChild(databaseMNode, tableName, nodeFactory.createInternalMNode(cur, tableName)); } for (final String childName : devicePath) { @@ -1776,14 +1773,14 @@ protected void updateEntity(final IDeviceMNode node) { public boolean deleteTableDevice(final String tableName, final IntConsumer attributeDeleter) throws MetadataException { - if (!store.hasChild(storageGroupMNode, tableName)) { + if (!store.hasChild(databaseMNode, tableName)) { return false; } final AtomicInteger memoryReleased = new AtomicInteger(0); try (final MNodeCollector collector = new MNodeCollector( - storageGroupMNode, - new PartialPath(new String[] {storageGroupMNode.getName(), tableName}), + databaseMNode, + new PartialPath(new String[] {databaseMNode.getName(), tableName}), this.store, true, SchemaConstant.ALL_MATCH_SCOPE) { @@ -1805,7 +1802,7 @@ protected Void collectMNode(final IMemMNode node) { }) { collector.traverse(); } - storageGroupMNode.deleteChild(tableName); + databaseMNode.deleteChild(tableName); regionStatistics.resetTableDevice(tableName); store.releaseMemory(memoryReleased.get()); return true; @@ -1813,14 +1810,14 @@ protected Void collectMNode(final IMemMNode node) { public boolean dropTableAttribute(final String tableName, final IntConsumer attributeDropper) throws MetadataException { - if (!store.hasChild(storageGroupMNode, tableName)) { + if (!store.hasChild(databaseMNode, tableName)) { return false; } final AtomicInteger memoryReleased = new AtomicInteger(0); try (final EntityUpdater updater = new EntityUpdater( - storageGroupMNode, - new PartialPath(new String[] {storageGroupMNode.getName(), tableName}), + databaseMNode, + new PartialPath(new String[] {databaseMNode.getName(), tableName}), this.store, true, SchemaConstant.ALL_MATCH_SCOPE) { @@ -1837,5 +1834,55 @@ protected void updateEntity(final IDeviceMNode node) { return true; } + public void checkTableDevice4Object(final String tableName) throws MetadataException { + if (!store.hasChild(databaseMNode, tableName)) { + return; + } + try (final Traverser checker = + new Traverser( + databaseMNode, + new PartialPath(new String[] {databaseMNode.getName(), tableName}), + this.store, + true, + SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected boolean shouldVisitSubtreeOfInternalMatchedNode(final IMemMNode node) { + return true; + } + + @Override + protected boolean shouldVisitSubtreeOfFullMatchedNode(final IMemMNode node) { + return true; + } + + @Override + protected boolean acceptInternalMatchedNode(final IMemMNode node) { + return true; + } + + @Override + protected boolean acceptFullMatchedNode(final IMemMNode node) { + return true; + } + + @Override + protected Void generateResult(final IMemMNode nextMatchedNode) { + if (TsTable.isInvalid4ObjectType(nextMatchedNode.getName())) { + throw new SemanticException( + TsTable.getObjectStringError("tag value", nextMatchedNode.getName()), + TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + } + return null; + } + + @Override + protected boolean mayTargetNodeType(final IMemMNode node) { + return true; + } + }) { + checker.traverse(); + } + } + // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index fdbca98cc004..2fb30960de8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,6 +95,7 @@ public void init(final byte[] tableInitializationBytes) { TsTableInternalRPCUtil.deserializeTableInitializationInfo(tableInitializationBytes); final Map> usingMap = tableInfo.left; final Map> preCreateMap = tableInfo.right; + usingMap.values().forEach(list -> list.forEach(TsTable::setNeedCheck4Object)); usingMap.forEach( (key, value) -> databaseTableMap.put( @@ -105,6 +107,21 @@ public void init(final byte[] tableInitializationBytes) { Function.identity(), (v1, v2) -> v2, ConcurrentHashMap::new)))); + + preCreateMap.forEach( + (key, value) -> + value.stream() + .filter( + table -> + table.setNeedCheck4Object() + && databaseTableMap.containsKey(key) + && databaseTableMap.get(key).containsKey(table.getTableName())) + .forEach( + table -> + databaseTableMap + .get(key) + .get(table.getTableName()) + .setNeedCheck4Object(true))); preCreateMap.forEach( (key, value) -> preUpdateTableMap.put( @@ -140,6 +157,12 @@ public void preUpdateTable(String database, final TsTable table, final String ol return v; } }); + if (table.isNeedCheck4Object() && databaseTableMap.containsKey(database)) { + final TsTable existing = databaseTableMap.get(database).get(table.getTableName()); + if (Objects.nonNull(existing)) { + existing.setNeedCheck4Object(true); + } + } LOGGER.info("Pre-update table {}.{} successfully", database, table.getTableName()); // If rename table @@ -171,6 +194,13 @@ public void rollbackUpdateTable(String database, final String tableName, final S readWriteLock.writeLock().lock(); try { removeTableFromPreUpdateMap(database, tableName); + // If the "need check" flag is set by the "preUpdate" table, then it need to be cleared here + if (databaseTableMap.containsKey(database)) { + final TsTable existing = databaseTableMap.get(database).get(tableName); + if (Objects.nonNull(existing)) { + existing.setNeedCheck4Object(); + } + } LOGGER.info("Rollback-update table {}.{} successfully", database, tableName); // If rename table @@ -270,7 +300,12 @@ public void invalid(String database, final String tableName, final String column try { if (databaseTableMap.containsKey(database) && databaseTableMap.get(database).containsKey(tableName)) { - databaseTableMap.get(database).get(tableName).removeColumnSchema(columnName); + final TsTable table = databaseTableMap.get(database).get(tableName); + final TSDataType type = table.getColumnSchema(columnName).getDataType(); + table.removeColumnSchema(columnName); + if (type.equals(TSDataType.OBJECT)) { + table.setNeedCheck4Object(); + } } if (preUpdateTableMap.containsKey(database) && preUpdateTableMap.get(database).containsKey(tableName)) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 00ced1990eaf..c272fed1fd8b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -20,6 +20,7 @@ package org.apache.iotdb.commons.schema.table; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; @@ -29,6 +30,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.collect.ImmutableList; import org.apache.tsfile.enums.TSDataType; @@ -65,6 +67,8 @@ public class TsTable { public static final String TTL_PROPERTY = "ttl"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.singleton(TTL_PROPERTY); + private static final String OBJECT_STRING_ERROR = + "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'"; private String tableName; private final Map columnSchemaMap = new LinkedHashMap<>(); @@ -78,6 +82,7 @@ public class TsTable { private transient long ttlValue = Long.MIN_VALUE; private transient int tagNums = 0; private transient int fieldNum = 0; + private transient boolean needCheck4Object = false; public TsTable(final String tableName) { this.tableName = tableName; @@ -273,6 +278,25 @@ public Map getProps() { } } + public boolean setNeedCheck4Object() { + for (final TsTableColumnSchema schema : columnSchemaMap.values()) { + if (schema.getDataType().equals(TSDataType.OBJECT)) { + this.needCheck4Object = true; + return true; + } + } + this.needCheck4Object = false; + return false; + } + + public void setNeedCheck4Object(final boolean needCheck4Object) { + this.needCheck4Object = needCheck4Object; + } + + public boolean isNeedCheck4Object() { + return needCheck4Object; + } + public boolean containsPropWithoutLock(final String propKey) { return props != null && props.containsKey(propKey); } @@ -362,6 +386,33 @@ public void setProps(Map props) { } } + public void checkTableNameAndObjectNames4Object() throws MetadataException { + if (isInvalid4ObjectType(tableName)) { + throw new MetadataException( + getObjectStringError("tableName", tableName), + TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + } + for (final TsTableColumnSchema schema : columnSchemaMap.values()) { + if (schema.getDataType().equals(TSDataType.OBJECT) + && isInvalid4ObjectType(schema.getColumnName())) { + throw new MetadataException( + getObjectStringError("objectName", schema.getColumnName()), + TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + } + } + } + + public static boolean isInvalid4ObjectType(final String column) { + return column.equals(".") + || column.equals("..") + || column.contains("./") + || column.contains(".\\"); + } + + public static String getObjectStringError(final String columnType, final String columnName) { + return String.format(OBJECT_STRING_ERROR, columnType, columnName); + } + @Override public boolean equals(Object o) { return super.equals(o); diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 9416633a2c41..0dd5149b896c 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -367,6 +367,11 @@ struct TDeleteDataOrDevicesForDropTableReq { 2: required string tableName } +struct TCheckDeviceIdForObjectReq { + 1: required list regionIdList + 2: required string tableName +} + struct TTableDeviceDeletionWithPatternAndFilterReq { 1: required list schemaRegionIdList 2: required string tableName @@ -1210,6 +1215,10 @@ service IDataNodeRPCService { */ common.TSStatus deleteColumnData(TDeleteColumnDataReq req) + /** + * Check device ID for object + */ + common.TSStatus checkDeviceIdForObject(TCheckDeviceIdForObjectReq req) /** * Construct table device black list