fix(redshift): detect existing DWH schema via svv_all_schemas (SCS-1193) (#2789)

The Diagnostics Warehouse existence check queried information_schema.schemata, which on Redshift only lists schemas the current user OWNS. A pre-provisioned DWH schema the service user merely has USAGE/CREATE on was therefore invisible, so Soda issued CREATE SCHEMA -- which needs CREATE-on-database and failed with 'permission denied for database' even though the schema already existed.

Override the schemas metadata query on Redshift to use pg_catalog.svv_all_schemas (visible by access, not ownership), matching the existing svv_tables/svv_columns overrides. Base dialect behaviour is unchanged via a new build_schemas_metadata_from_clause() hook.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Niels-b

soda-core/src/soda_core/common/sql_dialect.py

@@ -1576,6 +1576,17 @@ def build_all_columns_metadata_query_str(
             ]
         )
 
+    def build_schemas_metadata_from_clause(self, table_namespace: Optional[DataSourceNamespace] = None) -> FROM:
+        """
+        FROM clause for the schemas metadata query. Defaults to information_schema.schemata.
+
+        Override in dialects whose schema listing must not come from information_schema.
+        (e.g. Redshift, where information_schema.schemata is visibility-scoped to the schema
+        OWNER and therefore misses schemas the current user only has USAGE/CREATE on.)
+        """
+        information_schema_namespace_elements = self.information_schema_namespace_elements(table_namespace)
+        return FROM(self.table_schemata()).IN(information_schema_namespace_elements)
+
     def build_schemas_metadata_query_str(
         self, table_namespace: Optional[DataSourceNamespace] = None, filter_on_schema_name: Optional[str] = None
     ) -> str:
@@ -1587,11 +1598,9 @@ def build_schemas_metadata_query_str(
         """
         database_name: str | None = table_namespace.get_database_for_metadata_query() if table_namespace else None
 
-        information_schema_namespace_elements = self.information_schema_namespace_elements(table_namespace)
-
         select_elements = [
             SELECT([self.column_schema_name()]),
-            FROM(self.table_schemata()).IN(information_schema_namespace_elements),
+            self.build_schemas_metadata_from_clause(table_namespace),
         ]
 
         and_elements = [

soda-redshift/src/soda_redshift/common/data_sources/redshift_data_source.py

@@ -213,3 +213,20 @@ def build_columns_metadata_from_clause(self, table_namespace: DataSourceNamespac
 
     def _pg_catalog_schema(self) -> str:
         return "pg_catalog"
+
+    def table_schemata(self) -> str:
+        # information_schema.schemata on Redshift (Postgres-8 lineage) lists only schemas the
+        # current user OWNS, so a schema the DWH service user merely has USAGE/CREATE on is
+        # invisible. Soda's existence check then returns false and it issues CREATE SCHEMA,
+        # which needs CREATE-on-database and fails with "permission denied for database ..."
+        # even though the schema already exists (SCS-1193). SVV_ALL_SCHEMAS lists schemas by
+        # access, matching the svv_tables / svv_columns overrides above.
+        return self.default_casify("svv_all_schemas")
+
+    def column_schemata_catalog_name(self) -> str:
+        # SVV_ALL_SCHEMAS names the database column database_name (vs information_schema's catalog_name).
+        return self.default_casify("database_name")
+
+    def build_schemas_metadata_from_clause(self, table_namespace: Optional[DataSourceNamespace] = None) -> FROM:
+        # SVV_ALL_SCHEMAS is a system view in pg_catalog, mirroring the svv_columns FROM clause.
+        return FROM(self.table_schemata()).IN(self._pg_catalog_schema())

soda-redshift/tests/unit/test_redshift_dialect.py

@@ -1,3 +1,4 @@
+from soda_core.common.metadata_types import DbSchemaDataSourceNamespace
 from soda_core.common.sql_dialect import FROM, RANDOM, SELECT
 from soda_redshift.common.data_sources.redshift_data_source import RedshiftSqlDialect
 
@@ -8,6 +9,27 @@ def test_random():
     assert sql == 'SELECT RANDOM()\nFROM "a";'
 
 
+def test_schema_existence_query_uses_svv_all_schemas_not_information_schema():
+    # Regression for SCS-1193: information_schema.schemata on Redshift only lists schemas the
+    # current user OWNS, so a pre-provisioned DWH schema the service user merely has USAGE on is
+    # invisible -> Soda wrongly issues CREATE SCHEMA -> "permission denied for database". The
+    # existence check must query SVV_ALL_SCHEMAS (access-visible), like svv_tables / svv_columns.
+    sql_dialect: RedshiftSqlDialect = RedshiftSqlDialect()
+    sql = sql_dialect.build_schemas_metadata_query_str(
+        table_namespace=DbSchemaDataSourceNamespace(database="comm", schema="qe_gold"),
+        filter_on_schema_name="qe_gold",
+    )
+    lowered = sql.lower()
+    assert "svv_all_schemas" in lowered
+    assert "information_schema" not in lowered
+    # SVV_ALL_SCHEMAS exposes the database via database_name (not information_schema's catalog_name)
+    assert "database_name" in lowered
+    assert "catalog_name" not in lowered
+    # still filters on the target database + schema so it answers "does THIS schema exist"
+    assert "comm" in lowered
+    assert "qe_gold" in lowered
+
+
 def test_max_sql_statement_length_respects_redshift_16mb_cap():
     # Redshift's documented statement cap is 16 MB; the dialect reserves
     # 1 MB for multi-byte characters (we count chars, the server counts bytes).