source-appsflyer: move app_ids into document _meta fields

Author: nicolaslazo

source-appsflyer/acmeCo/daily_geo_aggregate_report.schema.yaml

@@ -15,17 +15,18 @@ $defs:
         description: Row ID of the Document, counting up from zero, or -1 if not known
         title: Row Id
         type: integer
+      app_id:
+        default: ''
+        description: AppsFlyer App ID this row was fetched for
+        title: App Id
+        type: string
     title: Meta
     type: object
 additionalProperties: true
 properties:
   _meta:
     $ref: '#/$defs/Meta'
     description: Document metadata
-  app_id:
-    description: AppsFlyer App ID this row was fetched for
-    title: App Id
-    type: string
   Date:
     title: Date
     type: string
@@ -39,7 +40,6 @@ properties:
     title: Campaign (C)
     type: string
 required:
-- app_id
 - Date
 - Country
 - Media Source (pid)

source-appsflyer/acmeCo/flow.yaml

@@ -2,7 +2,7 @@ collections:
   acmeCo/daily_geo_aggregate_report:
     schema: daily_geo_aggregate_report.schema.yaml
     key:
-    - /app_id
+    - /_meta/app_id
     - /Date
     - /Country
     - /Media Source (pid)

source-appsflyer/config.yaml

@@ -1,19 +1,14 @@
 credentials:
     credentials_title: Private App Credentials
     access_token_sops: ENC[AES256_GCM,data:ZkesZ9lRtjF1t8pqbDS77qtZdMAqYwQpaj440sWV5NeOUks3IW3Qg48umz+hU9IhlPmK7knYRhd+6Bk7eY1l3PnWuPIQkpYcBHnWT6G+UJYdtp8dWz/P3JnvMZt0q08EWri23OxulMtkaVk9K+9cHHqbJvKCiCP+6gU36Jf+T+D03khTCrUhBbwGxubK3+Yx+qlt6EjKQ+IJ8umTM9O93Ppgk3bRVN3dDqbF+W68YfMxLWefktWiaptIfGNWBVAjfDx7f4ngddxEjdzT6yuiECmbiVWXJuU6nTyeVplxU5/sbGeneoWBuywi178e0INS6pt0CIhojAdCuvNad1Ye3St8enDeZq5Yu4VvIdqsIY4WRgA0GmimxFRtl5XzScbgvfbvAok91BC98l75MMQFOdWw6wLWZyf+VrzWSN9mqQbrjB9fg0p0AUNL6FZsZEDuyw3WpzJxARbDSYsYzI4mnosI7Czb/Ep45ou/rXrzjoJpiZwU1O6C5gQNGlXYBe6TafSOudmUtJ8c2CpQB/vYHYDWglslWueSfvQbtQivRPC7Pl/KhWMJ07KMJzsdzAgipjbTNcyjAoZTdRjwQal56LRSGp/swtLIBnlWo651nHhcDRlebMqf2/1/OLdIvJG8eCl8bPirKZyloO/nVNVkZ1tBiA81V+IQbawfvImsRZwUJINWFR1ussoSNg2z321j/R8=,iv:i8/dwVUNry5gCKe8SjaMi32NBJrmdzyy+9YeASg6HR4=,tag:dY9vL48vlumDS3r8VD28fA==,type:str]
-app_ids_sops: ENC[AES256_GCM,data:vXAJcwxmoBY+j8fEfFh/,iv:U3hJh+J/iQLxMiU8hertkbk9B0VlO2R0SYdg667fu2s=,tag:BWWhHF2kwsvFzhShtSfvLQ==,type:str]
-start_date: "2026-04-15T00:00:00Z"
+app_ids_sops: ENC[AES256_GCM,data:gmndTfAWMX2ghKjDBUtrqQ==,iv:lf2W1vINpzcgfbF7vrdaH7GWaeTK/rDtFLxNBldN+kk=,tag:QsLi+uHB+tVfJElt3TaMkg==,type:str]
+start_date: "2026-04-05T00:00:00Z"
 sops:
-    kms: []
     gcp_kms:
         - resource_id: projects/estuary-theatre/locations/us-central1/keyRings/connector-keyring/cryptoKeys/connector-repository
           created_at: "2024-07-26T15:53:37Z"
           enc: CiQAdmEdwpmP0/Kcv6L1SnM3nPbsYLywIaUeOL1mz7cZjFUwWt0SSQD5lVw43Of1YI96anxUVo7d7ALi0DYnCNvxTHCgHX5ZeDBPlfXwLNpmYb6y3hJSsHG7NupwDQT/qIpUvlqQnq5IVkgJEFHPU1s=
-    azure_kv: []
-    hc_vault: []
-    age: []
-    lastmodified: "2026-04-17T20:56:07Z"
-    mac: ENC[AES256_GCM,data:Yq78TmkHNZajFC6sXVuJwsY5JJyrOimd3xPikP8l2Wmy4aBCdJ9zAfwgoYcl6UNh8T624wsJIaC9HnPD5wlO5yyMT44NNxO6JlODHLSEzgp+CV9qOHKE/OrA4MCRiZiNbIk0xT8Wf9pixWtkfkm3JZKJGIDrd82xWxx13lXPyy4=,iv:CdGQ0lVrxjMAN0pUj8zKFu2aXAoSk9mW2t2mXHjUtBU=,tag:ulFF0z+vMu1pRKp6YDQ+Bg==,type:str]
-    pgp: []
+    lastmodified: "2026-04-17T22:00:16Z"
+    mac: ENC[AES256_GCM,data:oUXiCPbmL84tL3+6Mz8rkI8tBVHlhUhEU4wI6QozsOJyKlrn+36p03S3Ykz617ocInPLMuAwR1ZMEUm5gOnU6+Grm39u9wC6SE+0GQhguV6lZ//wxr9U94yG9W43AgggnnQW6VUVpRvkkU0/zCQh+4C2NDGrUyqEL/0/OeBR+qY=,iv:Q/+saVbRXu2dwv+V05S9GTS8rwtM7oK3hZj2S9Y+IGI=,tag:xBx+hy788mOe27EwR94b8w==,type:str]
     encrypted_suffix: _sops
     version: 3.10.2

source-appsflyer/source_appsflyer/models.py

@@ -130,13 +130,30 @@ class PullApiDocument(BaseDocument, metaclass=ABCMeta):
     # returned by AppsFlyer exactly (including spaces, case, and parentheses).
     key_fields: ClassVar[tuple[str, ...]]
 
-    app_id: str = Field(description="AppsFlyer App ID this row was fetched for")
+    class Meta(BaseDocument.Meta):
+        model_config = ConfigDict(  # pyright: ignore[reportUnannotatedClassAttribute]
+            validate_assignment=True
+        )
+        app_id: str = Field(
+            description="AppsFlyer App ID this row was fetched for", default=""
+        )
+
+    meta_: Meta = Field(  # pyright: ignore[reportIncompatibleVariableOverride]
+        default_factory=Meta,
+        alias="_meta",
+        description="Document metadata",
+    )
 
     @model_validator(mode="before")
     @classmethod
     def _inject_app_id(cls, data: Any, info: ValidationInfo) -> Any:
-        if isinstance(data, dict) and info.context and "app_id" in info.context:
-            return {**data, "app_id": info.context["app_id"]}
+        if not isinstance(data, dict):
+            return data
+        if info.context and "app_id" in info.context:
+            meta = data.get("_meta") or {}
+            meta["app_id"] = info.context["app_id"]
+            data["_meta"] = meta
+            # return {**data, "app_id": info.context["app_id"]}
         return data
 
     @classmethod

source-appsflyer/source_appsflyer/resources.py

@@ -200,7 +200,7 @@ def make_resource(
     ) -> Resource[PullApiDocument, PullApiResourceConfig, ResourceState]:
         return Resource(
             name=doc_type.resource_name,
-            key=["/app_id", *[f"/{f}" for f in doc_type.key_fields]],
+            key=["/_meta/app_id", *[f"/{f}" for f in doc_type.key_fields]],
             model=doc_type,
             open=functools.partial(
                 open_report_binding,

source-appsflyer/test.flow.yaml

@@ -12,116 +12,116 @@ captures:
     bindings:
     - resource:
         type: webhook
-        name: re-attribution-in-app-event
+        name: in-app-event
         match_rule:
-          value: re-attribution-in-app-event
+          value: in-app-event
           key: event_name
           type: body
-      target: acmeCo/re-attribution-in-app-event
+      target: acmeCo/in-app-event
     - resource:
         type: webhook
-        name: install-in-app-event
+        name: organic-install-in-app-event
         match_rule:
-          value: install-in-app-event
+          value: organic-install-in-app-event
           key: event_name
           type: body
-      target: acmeCo/install-in-app-event
+      target: acmeCo/organic-install-in-app-event
     - resource:
         type: webhook
-        name: postback
+        name: organic-reinstall
         match_rule:
-          value: postback
+          value: organic-reinstall
           key: event_name
           type: body
-      target: acmeCo/postback
+      target: acmeCo/organic-reinstall
     - resource:
         type: webhook
-        name: organic-install-in-app-event
+        name: organic-install
         match_rule:
-          value: organic-install-in-app-event
+          value: organic-install
           key: event_name
           type: body
-      target: acmeCo/organic-install-in-app-event
+      target: acmeCo/organic-install
     - resource:
         type: webhook
-        name: reinstall
+        name: re-download
         match_rule:
-          value: reinstall
+          value: re-download
           key: event_name
           type: body
-      target: acmeCo/reinstall
+      target: acmeCo/re-download
     - resource:
         type: webhook
-        name: organic-reinstall
+        name: re-attribution-in-app-event
         match_rule:
-          value: organic-reinstall
+          value: re-attribution-in-app-event
           key: event_name
           type: body
-      target: acmeCo/organic-reinstall
+      target: acmeCo/re-attribution-in-app-event
     - resource:
         type: webhook
-        name: install
+        name: re-attribution
         match_rule:
-          value: install
+          value: re-attribution
           key: event_name
           type: body
-      target: acmeCo/install
+      target: acmeCo/re-attribution
     - resource:
         type: webhook
-        name: postbacks-copy
+        name: install
         match_rule:
-          value: postbacks-copy
+          value: install
           key: event_name
           type: body
-      target: acmeCo/postbacks-copy
+      target: acmeCo/install
     - resource:
         type: webhook
-        name: organic-install
+        name: install-in-app-event
         match_rule:
-          value: organic-install
+          value: install-in-app-event
           key: event_name
           type: body
-      target: acmeCo/organic-install
+      target: acmeCo/install-in-app-event
     - resource:
         type: webhook
-        name: re-engagement
+        name: re-engagement-in-app-event
         match_rule:
-          value: re-engagement
+          value: re-engagement-in-app-event
           key: event_name
           type: body
-      target: acmeCo/re-engagement
+      target: acmeCo/re-engagement-in-app-event
     - resource:
         type: webhook
-        name: re-engagement-in-app-event
+        name: postback
         match_rule:
-          value: re-engagement-in-app-event
+          value: postback
           key: event_name
           type: body
-      target: acmeCo/re-engagement-in-app-event
+      target: acmeCo/postback
     - resource:
         type: webhook
-        name: in-app-event
+        name: reinstall
         match_rule:
-          value: in-app-event
+          value: reinstall
           key: event_name
           type: body
-      target: acmeCo/in-app-event
+      target: acmeCo/reinstall
     - resource:
         type: webhook
-        name: re-attribution
+        name: re-engagement
         match_rule:
-          value: re-attribution
+          value: re-engagement
           key: event_name
           type: body
-      target: acmeCo/re-attribution
+      target: acmeCo/re-engagement
     - resource:
         type: webhook
-        name: re-download
+        name: postbacks-copy
         match_rule:
-          value: re-download
+          value: postbacks-copy
           key: event_name
           type: body
-      target: acmeCo/re-download
+      target: acmeCo/postbacks-copy
     - resource:
         name: daily_geo_aggregate_report
         interval: PT4H