Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Enhancement](nereids)support show backup in nereids #48818

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ supportedShowStatement
: SHOW statementScope? VARIABLES wildWhere? #showVariables
| SHOW AUTHORS #showAuthors
| SHOW CREATE (DATABASE | SCHEMA) name=multipartIdentifier #showCreateDatabase
| SHOW BACKUP ((FROM | IN) database=multipartIdentifier)? wildWhere? #showBackup
| SHOW BROKER #showBroker
| SHOW DYNAMIC PARTITION TABLES ((FROM | IN) database=multipartIdentifier)? #showDynamicPartition
| SHOW EVENTS ((FROM | IN) database=multipartIdentifier)? wildWhere? #showEvents
Expand Down Expand Up @@ -401,7 +402,6 @@ unsupportedShowStatement
| SHOW TABLET tabletId=INTEGER_VALUE #showTabletId
| SHOW TABLETS FROM tableName=multipartIdentifier partitionSpec?
wildWhere? sortClause? limitClause? #showTabletsFromTable
| SHOW BACKUP ((FROM | IN) database=multipartIdentifier)? wildWhere? #showBackup
| SHOW BRIEF? RESTORE ((FROM | IN) database=multipartIdentifier)? wildWhere? #showRestore
| SHOW RESOURCES wildWhere? sortClause? limitClause? #showResources
| SHOW WORKLOAD GROUPS wildWhere? #showWorkloadGroups
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,7 @@
import org.apache.doris.nereids.DorisParser.ShowAnalyzeContext;
import org.apache.doris.nereids.DorisParser.ShowAuthorsContext;
import org.apache.doris.nereids.DorisParser.ShowBackendsContext;
import org.apache.doris.nereids.DorisParser.ShowBackupContext;
import org.apache.doris.nereids.DorisParser.ShowBrokerContext;
import org.apache.doris.nereids.DorisParser.ShowCharsetContext;
import org.apache.doris.nereids.DorisParser.ShowCollationContext;
Expand Down Expand Up @@ -600,6 +601,7 @@
import org.apache.doris.nereids.trees.plans.commands.ShowAnalyzeCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowBackupCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCharsetCommand;
Expand Down Expand Up @@ -4960,6 +4962,20 @@ public LogicalPlan visitShowBackends(ShowBackendsContext ctx) {
return new ShowBackendsCommand();
}

@Override
public LogicalPlan visitShowBackup(ShowBackupContext ctx) {
String dbName = null;
Expression wildWhere = null;
if (ctx.database != null) {
List<String> nameParts = visitMultipartIdentifier(ctx.database);
dbName = nameParts.get(0);
}
if (ctx.wildWhere() != null) {
wildWhere = getWildWhere(ctx.wildWhere());
}
return new ShowBackupCommand(dbName, wildWhere);
}

@Override
public LogicalPlan visitShowPlugins(ShowPluginsContext ctx) {
return new ShowPluginsCommand();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ public enum PlanType {
SHOW_ANALYZE_COMMAND,
SHOW_QUEUED_ANALYZE_JOBS_COMMAND,
SHOW_BACKENDS_COMMAND,
SHOW_BACKUP_COMMAND,
SHOW_BLOCK_RULE_COMMAND,
SHOW_BROKER_COMMAND,
SHOW_CATALOG_COMMAND,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.nereids.trees.plans.commands;

import org.apache.doris.backup.AbstractJob;
import org.apache.doris.backup.BackupJob;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.CaseSensibility;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.PatternMatcherWrapper;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Like;
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.qe.ShowResultSetMetaData;
import org.apache.doris.qe.StmtExecutor;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;

import java.util.List;
import java.util.function.Predicate;
import java.util.stream.Collectors;

/**
* show backup command
*/
public class ShowBackupCommand extends ShowCommand {
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
.add("JobId").add("SnapshotName").add("DbName").add("State").add("BackupObjs").add("CreateTime")
.add("SnapshotFinishedTime").add("UploadFinishedTime").add("FinishedTime").add("UnfinishedTasks")
.add("Progress").add("TaskErrMsg").add("Status").add("Timeout")
.build();

private String dbName;
private Expression where;
private boolean isAccurateMatch;
private String snapshotName;

/**
* constructor
*/
public ShowBackupCommand(String dbName, Expression where) {
super(PlanType.SHOW_BACKUP_COMMAND);
this.dbName = dbName;
this.where = where;
}

public String getDbName() {
return dbName;
}

/**
* get metadata
*/
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
for (String title : TITLE_NAMES) {
builder.addColumn(new Column(title, ScalarType.createVarchar(30)));
}
return builder.build();
}

/**
* get label predicate for show backup
*/
public Predicate<String> getSnapshotPredicate() throws AnalysisException {
if (null == where) {
return label -> true;
}
if (isAccurateMatch) {
return CaseSensibility.LABEL.getCaseSensibility()
? label -> label.equals(snapshotName) : label -> label.equalsIgnoreCase(snapshotName);
} else {
PatternMatcher patternMatcher = PatternMatcherWrapper.createMysqlPattern(
snapshotName, CaseSensibility.LABEL.getCaseSensibility());
return patternMatcher::match;
}
}

/**
* validate
*/
private boolean validate(ConnectContext ctx) throws UserException {
if (Strings.isNullOrEmpty(dbName)) {
dbName = ctx.getDatabase();
if (Strings.isNullOrEmpty(dbName)) {
throw new AnalysisException("No database selected");
}
}

// check auth
if (!Env.getCurrentEnv().getAccessManager()
.checkDbPriv(ConnectContext.get(), InternalCatalog.INTERNAL_CATALOG_NAME, dbName, PrivPredicate.LOAD)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DBACCESS_DENIED_ERROR,
ConnectContext.get().getQualifiedUser(), dbName);
}

// SQL may be like : show backup from your_db_name; there is no where clause.
if (where == null) {
return true;
}

if (!(where instanceof Like) && !(where instanceof EqualTo)) {
return false;
}

if (where instanceof EqualTo) {
isAccurateMatch = true;
}

// left child
if (!(where.child(0) instanceof UnboundSlot)) {
return false;
}
String leftKey = ((UnboundSlot) where.child(0)).getName();
if (!"snapshotname".equalsIgnoreCase(leftKey)) {
return false;
}

// right child
if (!(where.child(1) instanceof StringLikeLiteral)) {
return false;
}
snapshotName = ((StringLikeLiteral) where.child(1)).getStringValue();
if (Strings.isNullOrEmpty(snapshotName)) {
return false;
}

return true;
}

/**
* handle show backup
*/
private ShowResultSet handleShowBackup(ConnectContext ctx, StmtExecutor executor) throws Exception {
boolean valid = validate(ctx);
if (!valid) {
throw new AnalysisException("Where clause should like: SnapshotName = \"your_snapshot_name\", "
+ " or SnapshotName LIKE \"matcher\"");
}

DatabaseIf database = ctx.getCurrentCatalog().getDbOrAnalysisException(dbName);
List<AbstractJob> jobs = Env.getCurrentEnv().getBackupHandler()
.getJobs(database.getId(), getSnapshotPredicate());
List<BackupJob> backupJobs = jobs.stream().filter(job -> job instanceof BackupJob)
.map(job -> (BackupJob) job).collect(Collectors.toList());
List<List<String>> infos = backupJobs.stream().map(BackupJob::getInfo).collect(Collectors.toList());

return new ShowResultSet(getMetaData(), infos);
}

@Override
public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
return handleShowBackup(ctx, executor);
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitShowBackupCommand(this, context);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
import org.apache.doris.nereids.trees.plans.commands.ShowAnalyzeCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowAuthorsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowBackendsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowBackupCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowBrokerCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCharsetCommand;
Expand Down Expand Up @@ -508,6 +509,10 @@ default R visitShowBackendsCommand(ShowBackendsCommand showBackendsCommand, C co
return visitCommand(showBackendsCommand, context);
}

default R visitShowBackupCommand(ShowBackupCommand showBackupCommand, C context) {
return visitCommand(showBackupCommand, context);
}

default R visitShowCreateTableCommand(ShowCreateTableCommand showCreateTableCommand, C context) {
return visitCommand(showCreateTableCommand, context);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th
"test_topn_fault_injection," +
"auto_partition_in_partition_prune," + // inserted data in too many tablets, txn to large. not suitable for cloud.
"one_col_range_partition," + // inserted data in too many tablets, txn to large. not suitable for cloud.
"test_nereids_show_backup," +
"zzz_the_end_sentinel_do_not_touch" // keep this line as the last line

excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_nereids_show_backup") {
String suiteName = "test_show_backup"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String snapshotName = "${suiteName}_snapshot"

def syncer = getSyncer()
syncer.createS3Repository(repoName)

sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
sql """
CREATE TABLE ${dbName}.${tableName} (
`id` LARGEINT NOT NULL,
`count` LARGEINT SUM DEFAULT "0")
AGGREGATE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 2
PROPERTIES
(
"replication_num" = "1"
)
"""

List<String> values = []
for (int i = 1; i <= 10; ++i) {
values.add("(${i}, ${i})")
}
sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
def result = sql "SELECT * FROM ${dbName}.${tableName}"
assertEquals(result.size(), values.size());

sql """
BACKUP SNAPSHOT ${dbName}.${snapshotName}
TO `${repoName}`
ON (${tableName})
"""

syncer.waitSnapshotFinish(dbName)

def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
assertTrue(snapshot != null)

checkNereidsExecute("show backup")
checkNereidsExecute("show backup from ${dbName}")
checkNereidsExecute("show backup in ${dbName}")
checkNereidsExecute("""show backup from ${dbName} where SnapshotName = "${snapshotName}" """)
checkNereidsExecute("""show backup in ${dbName} where SnapshotName = "${snapshotName}" """)
checkNereidsExecute("show backup from ${dbName} where SnapshotName like 'test%'")
checkNereidsExecute("show backup in ${dbName} where SnapshotName like 'test%'")

sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP DATABASE ${dbName} FORCE"
sql "DROP REPOSITORY `${repoName}`"
}