Tilt dev env & Tests: Go tests load .env no longer requiring variables injection

.env.example

@@ -16,8 +16,11 @@ CI_MYSQL_GTID_PORT=3306
 CI_MYSQL_POS_PORT=3307
 CI_MARIADB_PORT=3308
 CI_MYSQL_ROOT_PASSWORD=cipass
-CI_MYSQL_VERSION=mysql-gtid
+CI_MYSQL_GTID_VERSION=mysql-gtid
+CI_MYSQL_POS_VERSION=mysql-pos
+CI_MARIADB_VERSION=maria
 
+CI_MYSQL_VERSION=mysql-gtid
 CI_MYSQL_PORT=3306 # Default is gtid
 
 PG_HOST=host.docker.internal
@@ -39,4 +42,6 @@ AWS_ENDPOINT_URL_S3=http://host.docker.internal:9001
 AWS_EC2_METADATA_DISABLED="true"
 
 MINIO_ROOT_USER=_peerdb_minioadmin
-MINIO_ROOT_PASSWORD=_peerdb_minioadmin
+MINIO_ROOT_PASSWORD=_peerdb_minioadmin
+
+TZ='UTC'

README.md

@@ -100,7 +100,7 @@ For example:
 ```bash
 cd flow
 go clean -cache
-env -f ../.env go test -v -run TestGenericCH_MySQL ./e2e/
+go test -v -run TestGenericCH_MySQL ./e2e/
 ```
 
 Or local debugging sessions.
@@ -111,6 +111,8 @@ This is done through [Tilt](https://tilt.dev/) orchestrated Docker compose.
 
 To get the environment up you first need to specify the shared environment variables for both the test and the test environment in your local `.env` file. You can use the provided `.env.example` as a template: `cp .env.example .env `.
 
+If a `.env` file is present in the project root, tests will automatically load it. Any variable defined in `.env` can be overridden by user-provided environment variables.
+
 :memo: In the template, services URLs are set to `host.docker.internal`, which is the name for the default Docker gateway in Docker Desktop set-ups such as macOS and Windows. Using the default gateway address allows both test processes and services running inside Docker to access services on the host machine. In native Docker (Linux) this name is not resolved by default, you might replace it with the default gateway IP (e.g., `172.18.0.1`) or add a custom entry to your `/etc/hosts` file to resolve `host.docker.internal` to the appropriate IP address. e.g:
 
 ```bash
@@ -127,10 +129,10 @@ And follow the status of the services and access logs through the Tilt UI at htt
 
 <img width="1593" height="693" alt="image" src="https://github.com/user-attachments/assets/6c294dda-ca8f-45cc-b75c-11594118a641" />
 
-Since `.env` is the environment configuration source of truth, it can be used directly to inject the required variables to the test execution processes. e.g:
+Since `.env` is the environment configuration source of truth, tests automatically pick it up from the project root. For example:
 
 ```bash
-go clean -cache; env -f ../.env go test -v -run TestGenericCH_MySQL ./e2e/ # Some MySQL generic tests
+go clean -cache; go test -v -run TestGenericCH_MySQL ./e2e/ # Some MySQL generic tests
 ```
 
 ### Running tests from Tilt

Tiltfile

@@ -1,3 +1,5 @@
+update_settings(max_parallel_updates=10)
+
 allow_k8s_contexts(k8s_context()) # to unblock local() in local set-ups with a Kubernetes context configured, like Docker Desktop
 
 docker_compose('./docker-compose-dev.yml')
@@ -72,42 +74,42 @@ dc_resource('minio', labels=['PeerDB'])
 local_resource(
     'provision-mongodb',
     cmd='./local_provision_scripts/mongodb.sh',
-    labels=['Provisioning'],
+    labels=['Ancillary-DB-Provisioning'],
     resource_deps=['mongodb']
 )
 
 local_resource(
     'provision-clickhouse',
     cmd='./local_provision_scripts/clickhouse.sh',
-    labels=['Provisioning'],
+    labels=['Ancillary-DB-Provisioning'],
     resource_deps=['clickhouse']
 )
 
 local_resource(
     'provision-mysql-gtid',
     cmd='./local_provision_scripts/mysql.sh peerdb-mysql-gtid',
-    labels=['Provisioning'],
+    labels=['Ancillary-DB-Provisioning'],
     resource_deps=['mysql-gtid']
 )
 
 local_resource(
     'provision-mysql-pos',
     cmd='./local_provision_scripts/mysql.sh peerdb-mysql-pos',
-    labels=['Provisioning'],
+    labels=['Ancillary-DB-Provisioning'],
     resource_deps=['mysql-pos']
 )
 
 local_resource(
     'provision-mariadb',
     cmd='./local_provision_scripts/mysql.sh peerdb-mariadb',
-    labels=['Provisioning'],
+    labels=['Ancillary-DB-Provisioning'],
     resource_deps=['mariadb']
 )
 
 local_resource(
     'provision-postgres',
     cmd='./local_provision_scripts/postgres.sh',
-    labels=['Provisioning'],
+    labels=['Ancillary-DB-Provisioning'],
     resource_deps=['postgres']
 )
 
@@ -123,31 +125,43 @@ docker_compose('./ancillary-docker-compose.yml', env_file=tiltfile_dir + '/ancil
 # of their corresponding datastore.
 # This way, users can choose which ones to start and when, depending on the tests they want to run.
 
-dc_resource('clickhouse', labels=['DataStore'], links=[
+dc_resource('clickhouse', labels=['Ancillary-DB'], links=[
     link('http://localhost:11123', 'ClickHouse HTTP'),
     link('http://localhost:11000', 'ClickHouse TCP'),
 ], auto_init=False)
 
-dc_resource('mongodb', labels=['DataStore'], links=[
+dc_resource('mongodb', labels=['Ancillary-DB'], links=[
     link('http://localhost:11017', 'MongoDB'),
 ], auto_init=False)
 
-dc_resource('mysql-gtid', labels=['DataStore'], links=[
+dc_resource('mysql-gtid', labels=['Ancillary-DB'], links=[
     link('http://localhost:3306', 'MySQL GTID'),
 ], auto_init=False)
 
-dc_resource('mysql-pos', labels=['DataStore'], links=[
+dc_resource('mysql-pos', labels=['Ancillary-DB'], links=[
     link('http://localhost:3307', 'MySQL File-Pos'),
 ], auto_init=False)
 
-dc_resource('mariadb', labels=['DataStore'], links=[
+dc_resource('mariadb', labels=['Ancillary-DB'], links=[
     link('http://localhost:3308', 'MariaDB'),
 ], auto_init=False)
 
-dc_resource('postgres', labels=['DataStore'], links=[
+dc_resource('postgres', labels=['Ancillary-DB'], links=[
     link('http://localhost:5432', 'PostgreSQL'),
 ], auto_init=False)
 
+local_resource(
+    'all-test-resources',
+    cmd=' '.join([
+        "resources=$(tilt --port 10352 get uiresource -o json |",
+        "jq -r '.items[] | select(.metadata.labels[\"Ancillary-DB\"] or .metadata.labels[\"Ancillary-TestInfra\"]) | .metadata.name');",
+        "tilt --port 10352 enable $resources &&",
+        "for r in $resources; do tilt --port 10352 trigger $r; done",
+    ]),
+    labels=['Ancillary-Utilities'],
+    auto_init=False,
+)
+
 # Monitoring and utility tools
 
 dc_resource('dozzle', labels=['Monitoring'], links=[
@@ -156,8 +170,8 @@ dc_resource('dozzle', labels=['Monitoring'], links=[
 
 # Test services: Services supporting test execution that are not data stores, like proxies, mock servers, etc.
 
-dc_resource('toxiproxy', labels=['TestInfra'], auto_init=False)
-dc_resource('openssh', labels=['TestInfra'], auto_init=False)
+dc_resource('toxiproxy', labels=['Ancillary-TestInfra'], auto_init=False)
+dc_resource('openssh', labels=['Ancillary-TestInfra'], auto_init=False)
 
 # Cleanup
 
@@ -173,7 +187,16 @@ cmd_button(
     ])],
     text='Wipe ancillary volumes',
     icon_name='delete',
-    location=location.NAV,
+    location=location.NAV
+)
+
+cmd_button(
+    'Clean-up test code caches',
+    argv=['sh', '-c', 'cd flow && go clean -cache'],
+    text='Clean-up test code caches',
+    icon_name='delete_sweep',
+    resource='Test',
+    location=location.NAV
 )
 
 # Tests launchers
@@ -182,29 +205,93 @@ def e2e_test(name, test_run, extra_deps=[], vars_overrides={}):
     overrides_str = ' '.join(['%s=%s' % (var, value) for var, value in vars_overrides.items()])
     local_resource(
         'e2e_' + name,
-        cmd='cd flow && go clean -cache && env -f ../.env %s go test -v -run %s ./e2e/' % (overrides_str, test_run),
+        cmd='cd flow && %s go test -count=1 -v -run %s ./e2e/' % (overrides_str, test_run),
         labels=['Test'],
         auto_init=False,
         resource_deps=['flow-api', 'flow-worker', 'catalog', 'provision-clickhouse'] + extra_deps,
+        allow_parallel=True,
+    )
+
+def connector_test(connector, extra_deps=[], vars_overrides={}):
+    overrides_str = ' '.join(['%s=%s' % (var, value) for var, value in vars_overrides.items()])
+    local_resource(
+        'connector_' + connector,
+        cmd='cd flow && %s go test -count=1 -v ./connectors/%s/...' % (overrides_str, connector),
+        labels=['Test'],
+        auto_init=False,
+        resource_deps=['catalog'] + extra_deps,
+        allow_parallel=True,
     )
 
+# These are overrides to provide different MySQL flavors with the same test definitions.
+
+def resolve_env(var_name):
+    for line in str(read_file('.env')).splitlines():
+        if line.startswith(var_name + '='):
+            return line.strip().split('=', 1)[1]
+    return None
+
+mysql_gtid_vars = {
+    'CI_MYSQL_PORT': resolve_env('CI_MYSQL_GTID_PORT'),
+    'CI_MYSQL_VERSION': resolve_env('CI_MYSQL_GTID_VERSION'),
+}
+mysql_pos_vars = {
+    'CI_MYSQL_PORT': resolve_env('CI_MYSQL_POS_PORT'),
+    'CI_MYSQL_VERSION': resolve_env('CI_MYSQL_POS_VERSION'),
+}
+mariadb_vars = {
+    'CI_MYSQL_PORT': resolve_env('CI_MARIADB_PORT'),
+    'CI_MYSQL_VERSION': resolve_env('CI_MARIADB_VERSION'),
+}
+
 # Generic e2e tests
 
 # Postgres to ClickHouse generic tests
 e2e_test('postgres', 'TestGenericCH_PG', ['provision-postgres'])
 
 # MySQL GTID to ClickHouse generic tests
-e2e_test('mysql-gtid', 'TestGenericCH_MySQL', ['provision-mysql-gtid'], vars_overrides={'CI_MYSQL_PORT': '$CI_MYSQL_GTID_PORT'})
+e2e_test('mysql-gtid', 'TestGenericCH_MySQL', ['provision-mysql-gtid'], vars_overrides=mysql_gtid_vars)
 
 # MySQL Pos to ClickHouse generic tests
-e2e_test('mysql-pos', 'TestGenericCH_MySQL', ['provision-mysql-pos'], vars_overrides={'CI_MYSQL_PORT': '$CI_MYSQL_POS_PORT'})
+e2e_test('mysql-pos', 'TestGenericCH_MySQL', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars)
 
 # MariaDB to ClickHouse generic tests
-e2e_test('mariadb', 'TestGenericCH_MySQL', ['provision-mariadb'], vars_overrides={'CI_MYSQL_PORT': '$CI_MARIADB_PORT'})
+e2e_test('mariadb', 'TestGenericCH_MySQL', ['provision-mariadb'], vars_overrides=mariadb_vars)
 
 # MongoDB to ClickHouse test suite
 e2e_test('mongodb', 'TestMongoClickhouseSuite', ['provision-mongodb'])
 
+# Switchboard tests
+
+e2e_test('switchboard-postgres', 'TestSwitchboardPostgres', ['provision-postgres'])
+
+e2e_test('switchboard-mysql-gtid', 'TestSwitchboardMySQL', ['provision-mysql-gtid'], vars_overrides=mysql_gtid_vars)
+e2e_test('switchboard-mysql-pos', 'TestSwitchboardMySQL', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars)
+e2e_test('switchboard-mariadb', 'TestSwitchboardMySQL', ['provision-mariadb'], vars_overrides=mariadb_vars)
+
+e2e_test('switchboard-mongodb', 'TestSwitchboardMongo', ['provision-mongodb'])
+
+# Peer flow E2E
+
+e2e_test('peer-flow-postgres', '^TestPeerFlowE2ETestSuitePG_CH$', ['provision-postgres'])
+
+e2e_test('peer-flow-mysql-gtid', '^TestPeerFlowE2ETestSuiteMySQL_CH$', ['provision-mysql-gtid'], vars_overrides=mysql_gtid_vars)
+e2e_test('peer-flow-mysql-pos', '^TestPeerFlowE2ETestSuiteMySQL_CH$', ['provision-mysql-pos'], vars_overrides=mysql_pos_vars)
+e2e_test('peer-flow-mariadb', '^TestPeerFlowE2ETestSuiteMySQL_CH$', ['provision-mariadb'], vars_overrides=mariadb_vars)
+
 # API e2e tests
 
 e2e_test('api-postgres', 'TestApiPg', ['provision-postgres'])
+
+e2e_test('api-mysql-gtid', 'TestApiMy', ['provision-mysql-gtid', 'provision-postgres'], vars_overrides=mysql_gtid_vars)
+e2e_test('api-mysql-pos', 'TestApiMy', ['provision-mysql-pos', 'provision-postgres'], vars_overrides=mysql_pos_vars)
+e2e_test('api-mariadb', 'TestApiMy', ['provision-mariadb', 'provision-postgres'], vars_overrides=mariadb_vars)
+
+e2e_test('api-mongodb', 'TestApiMongo', ['provision-mongodb'])
+
+# Connectors tests
+
+connector_test('postgres', ['provision-postgres'])
+connector_test('mongo', ['provision-mongodb'])
+connector_test('mysql', ['provision-mysql-gtid'])
+connector_test('clickhouse', ['provision-clickhouse'])

flow/connectors/utils/autoenv.go

@@ -0,0 +1,13 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/PeerDB-io/peerdb/flow/internal/testutil"
+)
+
+func init() {
+	if testing.Testing() {
+		testutil.LoadEnv()
+	}
+}

flow/e2e/test_utils.go

@@ -14,7 +14,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/jackc/pgerrcode"
 	"github.com/jackc/pgx/v5"
-	"github.com/joho/godotenv"
 	"github.com/stretchr/testify/require"
 	"go.temporal.io/api/enums/v1"
 	"go.temporal.io/sdk/client"
@@ -30,6 +29,7 @@ import (
 	"github.com/PeerDB-io/peerdb/flow/generated/proto_conversions"
 	"github.com/PeerDB-io/peerdb/flow/generated/protos"
 	"github.com/PeerDB-io/peerdb/flow/internal"
+	"github.com/PeerDB-io/peerdb/flow/internal/testutil"
 	"github.com/PeerDB-io/peerdb/flow/model"
 	"github.com/PeerDB-io/peerdb/flow/shared"
 	"github.com/PeerDB-io/peerdb/flow/shared/types"
@@ -40,7 +40,7 @@ import (
 func init() {
 	// it's okay if the .env file is not present
 	// we will use the default values
-	_ = godotenv.Load()
+	testutil.LoadEnv()
 }
 
 type Suite interface {