fix: Expand env vars in stream_map_config (#421)

## Related

- meltano/meltano#7151
- meltano/meltano#9626

Author: edgarrmondragon

.github/workflows/ci.yml

@@ -103,6 +103,17 @@ jobs:
             select * from cust
             union
             select * from empl
+        - tap: tap-csv--multi
+          mapping: expand_env_vars
+          output_db: tap_csv__multi.db
+          query: |
+            with
+              cust_with_extra as (select *, 'CUSTOMER' as type from customers limit 10),
+              empl_with_extra as (select *, 'EMPLOYEE' as type from employees limit 10)
+
+            select * from cust_with_extra
+            union
+            select * from empl_with_extra
     steps:
     - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
@@ -126,6 +137,9 @@ jobs:
         meltano --version
 
     - name: Run
+      env:
+        MELTANO_RUN_ID: C95FD852-6766-4D11-A484-6CA858AA6939
+        ELT_RUN_STARTED_AT: "2025-01-01T00:00:00Z"
       run: |
         meltano run --install ${{ matrix.tap }} ${{ matrix.mapping }} target-sqlite
 

meltano.yml

@@ -46,6 +46,8 @@ plugins:
     settings:
     - name: stream_maps
       kind: object
+    - name: stream_maps_config
+      kind: object
     mappings:
     - name: hash_email
       config:
@@ -129,6 +131,17 @@ plugins:
           "*":
             name: first_name
             first_name: __NULL__
+    # Expand environment variables
+    - name: expand_env_vars
+      config:
+        stream_maps:
+          "*":
+            _extract__run_id: config['_extract__run_id']
+            # _extract__run_id: $MELTANO_RUN_ID
+            _extract__run_started_at: config['_extract__run_started_at']
+        stream_map_config:
+          _extract__run_id: $MELTANO_RUN_ID # tried with {}s, quotes, etc etc etc...
+          _extract__run_started_at: $ELT_RUN_STARTED_AT
 environments:
 - name: dev
   config:

meltano_map_transform/mapper.py

@@ -47,6 +47,12 @@ class StreamTransform(InlineMapper):
             required=True,
             description="Stream maps",
         ),
+        th.Property(
+            "stream_map_config",
+            th.ObjectType(),
+            title="User Stream Map Configuration",
+            description="User-defined config values to be used within map expressions.",
+        ),
         th.Property(
             "flattening_enabled",
             th.BooleanType(),

plugins/extractors/tap-csv--meltanolabs.lock

@@ -37,6 +37,18 @@
       "description": "Project-relative path to JSON file holding array of objects as described under [Files](#files) - with `entity`, `path`, `keys`, and other optional keys:\n\n```json\n[\n  {\n    \"entity\": \"<entity>\",\n    \"path\": \"<path>\",\n    \"keys\": [\"<key>\"],\n  },\n  // ...\n]\n```\n",
       "placeholder": "Ex. files-def.json"
     },
+    {
+      "name": "faker_config.locale",
+      "kind": "array",
+      "label": "Faker Locale",
+      "description": "One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization"
+    },
+    {
+      "name": "faker_config.seed",
+      "kind": "string",
+      "label": "Faker Seed",
+      "description": "Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator"
+    },
     {
       "name": "files",
       "kind": "array",
@@ -46,19 +58,19 @@
     {
       "name": "flattening_enabled",
       "kind": "boolean",
-      "label": "Flattening Enabled",
+      "label": "Enable Schema Flattening",
       "description": "'True' to enable schema flattening and automatically expand nested properties."
     },
     {
       "name": "flattening_max_depth",
       "kind": "integer",
-      "label": "Flattening Max Depth",
+      "label": "Max Flattening Depth",
       "description": "The max depth to flatten schemas."
     },
     {
       "name": "stream_map_config",
       "kind": "object",
-      "label": "Stream Map Config",
+      "label": "User Stream Map Configuration",
       "description": "User-defined config values to be used within map expressions."
     },
     {

plugins/mappers/meltano-map-transformer--meltano.lock

@@ -6,7 +6,7 @@
   "label": "Meltano Map Transformer",
   "docs": "https://hub.meltano.com/mappers/meltano-map-transformer--meltano",
   "repo": "https://github.com/MeltanoLabs/meltano-map-transform",
-  "pip_url": "git+https://github.com/MeltanoLabs/meltano-map-transform.git",
+  "pip_url": "meltano-map-transform",
   "executable": "meltano-map-transform",
   "description": "A map transformer which implements the Stream Maps capability from Meltano's tap and target SDK.",
   "logo_url": "https://hub.meltano.com/assets/logos/mappers/meltano.png",