fix(restore): restore auth and service levels

As documented in https://opensource.docs.scylladb.com/master/cql/describe-schema.html#required-permissions,
schema restoration should be performed with superuser permissions.

Fixes #3869
Fixes #3875

Author: Michal-Leszczynski

pkg/service/restore/helper_integration_test.go

@@ -108,7 +108,7 @@ func newTestHelper(t *testing.T, srcHosts, dstHosts []string) *testHelper {
 	user := randomizedName("helper_user_")
 	pass := randomizedName("helper_pass_")
 
-	dropNonSuperUsers(t, dstCluster.rootSession)
+	dropNonCassandraUsers(t, dstCluster.rootSession)
 	createUser(t, dstCluster.rootSession, user, pass)
 
 	return &testHelper{
@@ -378,14 +378,14 @@ func filteredTables(t *testing.T, s gocqlx.Session, filter []string) []string {
 	return out
 }
 
-func dropNonSuperUsers(t *testing.T, s gocqlx.Session) {
+func dropNonCassandraUsers(t *testing.T, s gocqlx.Session) {
 	var (
 		name  string
 		super bool
 	)
 	iter := s.Query("LIST USERS", nil).Iter()
 	for iter.Scan(&name, &super) {
-		if !super {
+		if name != "cassandra" {
 			if err := s.ExecStmt(fmt.Sprintf("DROP USER '%s'", name)); err != nil {
 				t.Fatal(errors.Wrapf(err, "drop user %s", name))
 			}
@@ -434,7 +434,7 @@ func grantRestoreTablesPermissions(t *testing.T, s gocqlx.Session, restoredTable
 }
 
 func grantRestoreSchemaPermissions(t *testing.T, s gocqlx.Session, user string) {
-	ExecStmt(t, s, "GRANT CREATE ON ALL KEYSPACES TO "+user)
+	ExecStmt(t, s, fmt.Sprintf("ALTER USER %s SUPERUSER", user))
 }
 
 func validateCompleteProgress(t *testing.T, pr Progress, tables []table) {

pkg/service/restore/schema_worker.go

@@ -146,13 +146,6 @@ func (w *schemaWorker) restoreFromSchemaFile(ctx context.Context) error {
 
 	var createdKs []string
 	for _, row := range *w.describedSchema {
-		if row.Keyspace == "" {
-			// Scylla 6.3 added roles and service levels to the output of
-			// DESC SCHEMA WITH INTERNALS (https://github.com/scylladb/scylladb/pull/20168).
-			// Those entities do not live in any particular keyspace, so that's how we identify them.
-			// We are skipping them until we properly support their restoration.
-			continue
-		}
 		if row.Keyspace == "system_replicated_keys" {
 			// See https://github.com/scylladb/scylla-enterprise/issues/4168
 			continue

pkg/service/restore/service_restore_integration_test.go

@@ -731,7 +731,7 @@ func smokeRestore(t *testing.T, target Target, keyspace string, loadCnt, loadSiz
 	}
 
 	// Restore should be performed on user with limited permissions
-	dropNonSuperUsers(t, dstSession)
+	dropNonCassandraUsers(t, dstSession)
 	createUser(t, dstSession, user, "pass")
 	dstH = newRestoreTestHelper(t, mgrSession, cfg, target.Location[0], nil, user, "pass")
 
@@ -802,7 +802,7 @@ func restoreWithAgentRestart(t *testing.T, target Target, keyspace string, loadC
 	}
 
 	// Restore should be performed on user with limited permissions
-	dropNonSuperUsers(t, dstSession)
+	dropNonCassandraUsers(t, dstSession)
 	createUser(t, dstSession, user, "pass")
 	dstH = newRestoreTestHelper(t, mgrSession, cfg, target.Location[0], nil, user, "pass")
 
@@ -910,7 +910,7 @@ func restoreWithResume(t *testing.T, target Target, keyspace string, loadCnt, lo
 	}
 
 	// Restore should be performed on user with limited permissions
-	dropNonSuperUsers(t, dstSession)
+	dropNonCassandraUsers(t, dstSession)
 	createUser(t, dstSession, user, "pass")
 	dstH = newRestoreTestHelper(t, mgrSession, cfg, target.Location[0], nil, user, "pass")
 
@@ -1118,11 +1118,6 @@ func restoreWithVersions(t *testing.T, target Target, keyspace string, loadCnt,
 	// This also allows us to test scenario with mixed ID type SSTables.
 	srcH.Hrt.SetRespInterceptor(newRenameSnapshotSSTablesRespInterceptor(srcH.Client, srcSession, halfUUIDToIntIDGen()))
 
-	// Restore should be performed on user with limited permissions
-	//dropNonSuperUsers(t, dstSession)
-	//createUser(t, dstSession, user, "pass")
-	//dstH = newRestoreTestHelper(t, mgrSession, cfg, target.Location[0], nil, user, "pass")
-
 	if target.RestoreTables {
 		Print("Recreate schema on destination cluster")
 		WriteDataSecondClusterSchema(t, dstSession, keyspace, 0, 0)
@@ -1304,12 +1299,6 @@ func restoreWithVersions(t *testing.T, target Target, keyspace string, loadCnt,
 	Print("Restore 3-rd backup with versioned files")
 	target.SnapshotTag = tag3
 
-	if target.RestoreTables {
-		//	grantRestoreTablesPermissions(t, dstSession, target.Keyspace, user)
-	} else {
-		//	grantRestoreSchemaPermissions(t, dstSession, user)
-	}
-
 	if err = dstH.service.Restore(ctx, dstH.ClusterID, dstH.TaskID, dstH.RunID, dstH.targetToProperties(target)); err != nil {
 		t.Fatal(err)
 	}
@@ -1367,7 +1356,7 @@ func restoreViewCQLSchema(t *testing.T, target Target, keyspace string, loadCnt,
 	}
 
 	Print("When: Create Restore user")
-	dropNonSuperUsers(t, dstSession)
+	dropNonCassandraUsers(t, dstSession)
 	createUser(t, dstSession, user, "pass")
 	dstH = newRestoreTestHelper(t, mgrSession, cfg, target.Location[0], nil, user, "pass")
 
@@ -1459,7 +1448,7 @@ func restoreViewSSTableSchema(t *testing.T, schemaTarget, tablesTarget Target, k
 	dstH.skipImpossibleSchemaTest()
 
 	Print("When: Create Restore user")
-	dropNonSuperUsers(t, dstSession)
+	dropNonCassandraUsers(t, dstSession)
 	createUser(t, dstSession, user, "pass")
 	dstH = newRestoreTestHelper(t, mgrSession, cfg, schemaTarget.Location[0], nil, user, "pass")
 
@@ -1561,7 +1550,7 @@ func restoreAllTables(t *testing.T, schemaTarget, tablesTarget Target, keyspace
 	}
 
 	// Restore should be performed on user with limited permissions
-	dropNonSuperUsers(t, dstSession)
+	dropNonCassandraUsers(t, dstSession)
 	createUser(t, dstSession, user, "pass")
 	dstH = newRestoreTestHelper(t, mgrSession, cfg, schemaTarget.Location[0], nil, user, "pass")
 
@@ -1667,7 +1656,7 @@ func restoreAlternator(t *testing.T, schemaTarget, tablesTarget Target, testKeys
 	dstH.skipCQLSchemaTestAssumingSSTables()
 
 	// Restore should be performed on user with limited permissions
-	dropNonSuperUsers(t, dstSession)
+	dropNonCassandraUsers(t, dstSession)
 	createUser(t, dstSession, user, "pass")
 	dstH = newRestoreTestHelper(t, mgrSession, cfg, schemaTarget.Location[0], nil, user, "pass")