materialize-iceberg: only run rest catalog tests

Author: mdibaiee

materialize-iceberg/.snapshots/TestIntegration-apply

@@ -1,11 +1,21 @@
 package connector
 
 import (
+	"flag"
+	"os"
 	"testing"
 
 	boilerplate "github.com/estuary/connectors/materialize-boilerplate/testutil"
 )
 
+// testAll controls whether integration tests run against every catalog
+// configuration (REST, Glue, S3 Tables, and their deprecated variants) or only
+// against the REST catalog. Running the full matrix is slow and requires
+// credentials for multiple cloud backends, so by default we only exercise the
+// REST catalog. Opt in to the full matrix by setting the `ICEBERG_TEST_ALL`
+// environment variable or passing `-iceberg.test-all` to `go test`.
+var testAll = flag.Bool("iceberg.test-all", false, "run integration tests against all catalog configurations (REST, Glue, S3 Tables) instead of REST only")
+
 func TestIntegration(t *testing.T) {
 	if testing.Short() {
 		t.Skip()
@@ -15,15 +25,26 @@ func TestIntegration(t *testing.T) {
 		return resource{Table: table}
 	}
 
+	all := *testAll || os.Getenv("ICEBERG_TEST_ALL") != ""
+
+	materializeSpec := "testdata/materialize-rest.flow.yaml"
+	applySpec := "testdata/apply-rest.flow.yaml"
+	migrateSpec := "testdata/migrate-rest.flow.yaml"
+	if all {
+		materializeSpec = "testdata/materialize.flow.yaml"
+		applySpec = "testdata/apply.flow.yaml"
+		migrateSpec = "testdata/migrate.flow.yaml"
+	}
+
 	t.Run("materialize", func(t *testing.T) {
-		boilerplate.RunMaterializationTestParallel(t, newMaterialization, "testdata/materialize.flow.yaml", makeResourceFn, nil)
+		boilerplate.RunMaterializationTestParallel(t, newMaterialization, materializeSpec, makeResourceFn, nil)
 	})
 
 	t.Run("apply", func(t *testing.T) {
-		boilerplate.RunApplyTestParallel(t, &Driver{}, newMaterialization, "testdata/apply.flow.yaml", makeResourceFn)
+		boilerplate.RunApplyTestParallel(t, &Driver{}, newMaterialization, applySpec, makeResourceFn)
 	})
 
 	t.Run("migrate", func(t *testing.T) {
-		boilerplate.RunMigrationTestParallel(t, newMaterialization, "testdata/migrate.flow.yaml", makeResourceFn, nil)
+		boilerplate.RunMigrationTestParallel(t, newMaterialization, migrateSpec, makeResourceFn, nil)
 	})
 }

materialize-iceberg/.snapshots/TestIntegration-materialize

@@ -0,0 +1,8 @@
+materializations:
+  acmeCo/tests/materialize-iceberg/rest:
+    endpoint:
+      local:
+        command: ["go", "run", "./cmd/connector"]
+        protobuf: true
+        config: config.rest.yaml
+    bindings: []

materialize-iceberg/.snapshots/TestIntegration-migrate

@@ -0,0 +1,28 @@
+import:
+  - ../../materialize-boilerplate/testdata/integration/collections.materialize.flow.yaml
+
+materializations:
+  acmeCo/tests/materialize-iceberg/rest:
+    endpoint:
+      local:
+        command: ["go", "run", "./cmd/connector"]
+        protobuf: true
+        config: config.rest.yaml
+    bindings:
+    - resource:
+        table: simple_standard
+      source: tests/simple
+      fields:
+        recommended: true
+    # The tests/not-simple collection is intentionally omitted for the REST
+    # (Polaris) catalog: several of its field names (notably the required key
+    # containing unicode supplementary characters and other punctuation) cause
+    # Polaris table creation to fail with a 500. Since the problematic column
+    # is part of the collection key, it can't be dropped via field selection,
+    # so we skip the whole binding here. It is still exercised by the Glue and
+    # S3 Tables tasks.
+    - resource:
+        table: data_types
+      source: tests/data-types
+      fields:
+        recommended: true

materialize-iceberg/driver_test.go

@@ -14,11 +14,9 @@ materializations:
       source: tests/simple
       fields:
         recommended: true
-    - resource:
-        table: not_simple
-      source: tests/not-simple
-      fields:
-        recommended: true
+    # tests/not-simple is skipped for REST (Polaris): its required unicode key
+    # field breaks Polaris table creation, and since it's a key it can't be
+    # dropped via field selection.
     - resource:
         table: data_types
       source: tests/data-types
@@ -37,11 +35,7 @@ materializations:
       source: tests/simple
       fields:
         recommended: true
-    - resource:
-        table: not_simple
-      source: tests/not-simple
-      fields:
-        recommended: true
+    # See note on the rest task above for why tests/not-simple is omitted.
     - resource:
         table: data_types
       source: tests/data-types

materialize-iceberg/testdata/apply-rest.flow.yaml

@@ -0,0 +1,16 @@
+import:
+  - ../../materialize-boilerplate/testdata/integration/collections.migrate-base.flow.yaml
+
+materializations:
+  acmeCo/tests/materialize-iceberg/rest:
+    endpoint:
+      local:
+        command: ["go", "run", "./cmd/connector"]
+        protobuf: true
+        config: config.rest.yaml
+    bindings:
+    - resource:
+        table: migrate
+      source: key/value
+      fields:
+        recommended: true