diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java index 8b31afd8a5a..0e1f35bfc6e 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.gravitino.Catalog; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; @@ -44,6 +45,7 @@ import org.apache.gravitino.messaging.Topic; import org.apache.gravitino.messaging.TopicCatalog; import org.apache.gravitino.messaging.TopicChange; +import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; @@ -245,9 +247,15 @@ public void testDropTopic() { TopicCatalog topicCatalog = client.loadMetalake(METALAKE).loadCatalog(CATALOG).asTopicCatalog(); topicCatalog.dropTopic(NameIdentifier.of(SCHEMA, "topic1")); // check topics are dropped - NameIdentifier[] topicsList = topicCatalog.listTopics(Namespace.of(SCHEMA)); - assertArrayEquals(new NameIdentifier[] {}, topicsList); - NameIdentifier[] topicsListNormalUser = topicCatalogNormalUser.listTopics(Namespace.of(SCHEMA)); - assertArrayEquals(new NameIdentifier[] {}, topicsListNormalUser); + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .untilAsserted( + () -> { + NameIdentifier[] topicsList = topicCatalog.listTopics(Namespace.of(SCHEMA)); + assertArrayEquals(new NameIdentifier[] {}, topicsList); + NameIdentifier[] topicsListNormalUser = + topicCatalogNormalUser.listTopics(Namespace.of(SCHEMA)); + assertArrayEquals(new NameIdentifier[] {}, topicsListNormalUser); + }); } } diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index aa5790aeff4..196e0160f0c 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -107,17 +107,17 @@ The Gravitino Iceberg REST catalog service uses the memory catalog backend by de #### JDBC backend configuration -| Configuration item | Description | Default value | Required | Since Version | -|-----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|--------------------------|----------|---------------| -| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for the JDBC catalog backend. | `memory` | Yes | 0.2.0 | -| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.warehouse` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | -| `gravitino.iceberg-rest.jdbc-user` | The username of the JDBC connection. | (none) | No | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-password` | The password of the JDBC connection. | (none) | No | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | -| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog. Set to `V1` to enable view support. Once the underlying database is migrated to V1, this property is no longer required on subsequent restarts. | `V0` | No | 1.2.0 | +| Configuration item | Description | Default value | Required | Since Version | +|-----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------| +| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for the JDBC catalog backend. | `memory` | Yes | 0.2.0 | +| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.warehouse` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | +| `gravitino.iceberg-rest.jdbc-user` | The username of the JDBC connection. | (none) | No | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-password` | The password of the JDBC connection. | (none) | No | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | +| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog. Defaults to `V1` to enable view support. Set to `V0` only if you need to opt out of view support. Once the underlying database is migrated to V1, this property is no longer required on subsequent restarts. | `V1` | No | 1.2.0 | If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to keep consistent with your Jdbc Iceberg catalog name to operate the prior namespace and tables. @@ -458,11 +458,11 @@ Please set the `gravitino.iceberg-rest.warehouse` parameter to `{storage_prefix} ### View support -View operations are supported when using the JDBC catalog backend with schema version `V1`. Iceberg will automatically migrate the database schema on the first restart and detect the migration on all subsequent restarts. +View operations are supported when using the JDBC catalog backend with schema version `V1`. The default schema version is now `V1`, so view support is enabled out of the box. Iceberg will automatically migrate the database schema on the first restart and detect the migration on all subsequent restarts. -| Configuration item | Description | Default value | Required | Since Version | -|----------------------------------------------|--------------------------------------------------------------------------------------------|---------------|----------|---------------| -| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog backend. Set to `V1` to enable view operations. | `V0` | No | 1.2.0 | +| Configuration item | Description | Default value | Required | Since Version | +|----------------------------------------------|----------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------| +| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog backend. Defaults to `V1` to enable view operations. Set to `V0` to opt out. | `V1` | No | 1.2.0 | ### Other Apache Iceberg catalog properties diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java index 38f26433532..86508d77051 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java @@ -105,6 +105,9 @@ private static JdbcCatalog loadJdbcCatalog(IcebergConfig icebergConfig) { new JdbcCatalogWithMetadataLocationSupport( icebergConfig.get(IcebergConfig.JDBC_INIT_TABLES)); + // Default to V1 schema to support view operations; can be overridden by explicit config. + properties.putIfAbsent(IcebergConstants.ICEBERG_JDBC_SCHEMA_VERSION, "V1"); + HdfsConfiguration hdfsConfiguration = new HdfsConfiguration(); properties.forEach(hdfsConfiguration::set); jdbcCatalog.setConf(hdfsConfiguration); diff --git a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java index 9773480fdd2..5db205bfd72 100644 --- a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java +++ b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java @@ -29,6 +29,7 @@ import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.inmemory.InMemoryCatalog; import org.apache.iceberg.jdbc.JdbcCatalog; +import org.apache.iceberg.jdbc.JdbcCatalogWithMetadataLocationSupport; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -66,7 +67,7 @@ void testLoadCatalog() { properties.put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC"); properties.put(IcebergConstants.ICEBERG_JDBC_USER, "test"); properties.put(IcebergConstants.ICEBERG_JDBC_PASSWORD, "test"); - properties.put(IcebergConstants.ICEBERG_JDBC_INITIALIZE, "false"); + properties.put(IcebergConstants.ICEBERG_JDBC_INITIALIZE, "true"); catalog = IcebergCatalogUtil.loadCatalogBackend( IcebergCatalogBackend.JDBC, new IcebergConfig(properties)); @@ -79,6 +80,43 @@ void testLoadCatalog() { }); } + @Test + void testJdbcCatalogDefaultSchemaVersionIsV1() { + Map properties = new HashMap<>(); + properties.put(CatalogProperties.URI, "jdbc:sqlite::memory:"); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, "test"); + properties.put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC"); + properties.put(IcebergConstants.ICEBERG_JDBC_USER, "test"); + properties.put(IcebergConstants.ICEBERG_JDBC_PASSWORD, "test"); + // jdbc.schema-version is intentionally not set; default should be V1 + Catalog catalog = + IcebergCatalogUtil.loadCatalogBackend( + IcebergCatalogBackend.JDBC, new IcebergConfig(properties)); + Assertions.assertInstanceOf(JdbcCatalogWithMetadataLocationSupport.class, catalog); + Assertions.assertTrue( + ((JdbcCatalogWithMetadataLocationSupport) catalog).supportsViewsWithSchemaVersion(), + "JDBC catalog should default to V1 schema and support view operations"); + } + + @Test + void testJdbcCatalogExplicitSchemaVersionNotOverridden() { + Map properties = new HashMap<>(); + properties.put(CatalogProperties.URI, "jdbc:sqlite::memory:"); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, "test"); + properties.put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC"); + properties.put(IcebergConstants.ICEBERG_JDBC_USER, "test"); + properties.put(IcebergConstants.ICEBERG_JDBC_PASSWORD, "test"); + // Explicitly set V0; loadJdbcCatalog must not override it with V1 + properties.put(IcebergConstants.ICEBERG_JDBC_SCHEMA_VERSION, "V0"); + Catalog catalog = + IcebergCatalogUtil.loadCatalogBackend( + IcebergCatalogBackend.JDBC, new IcebergConfig(properties)); + Assertions.assertInstanceOf(JdbcCatalogWithMetadataLocationSupport.class, catalog); + Assertions.assertFalse( + ((JdbcCatalogWithMetadataLocationSupport) catalog).supportsViewsWithSchemaVersion(), + "Explicitly configured V0 schema should not be overridden to V1"); + } + @Test void testValidLoadCustomCatalog() { Catalog catalog;