diff --git a/gcp/README.md b/gcp/README.md index 1eae7a83..e99390fc 100644 --- a/gcp/README.md +++ b/gcp/README.md @@ -88,6 +88,7 @@ Please notice the escaping of quotes for job's argument. * `dataset` - BigQuery dataset id for writing results of ads queries (required) * `bq_dataset_location` - BigQuery dataset location, e.g. "europe", by default "us" (optional) * `cid` - Ads customer id, can be either a MCC or child account, without dashes (required), or a list of CIDs comma separated (required) +* `cid_ignore` - Ads customer id (or a comma separated list) to exclude from the resulting list * `customer_ids_query` - a path to a file with GAQL query that refines for which accounts to execute scripts (optional) * `customer_ids_batchsize` - a batch size for customer ids (cids), if not specified accounts will be processed by 1000 accounts (see gaarf-getids CF) * `customer_ids_offset` - an offset in resulting list of accounts if you need to implemented an external batching - i.e. execute workflow only for a subset of accounts from specified seed account(s). It differs from internal batching where accounts processed by batches to workaround the maximum steps limitation of Cloud Workflows (100K runtime steps). diff --git a/gcp/functions/build/src/gaarf-getcids.js b/gcp/functions/build/src/gaarf-getcids.js index bdbc43f1..bf8be36f 100644 --- a/gcp/functions/build/src/gaarf-getcids.js +++ b/gcp/functions/build/src/gaarf-getcids.js @@ -27,6 +27,16 @@ async function main_getcids_unsafe(req, res, logger) { const { refresh_token, ...ads_config_wo_token } = adsConfig; await logger.info('Ads API config', ads_config_wo_token); let customerIds = (0, google_ads_api_report_fetcher_1.parseCustomerIds)(req.query.customer_id, adsConfig); + let customerIdsIgnore = []; + if (req.query.customer_ids_ignore) { + const customer_ids_ignore = req.query.customer_ids_ignore; + if (customer_ids_ignore.includes(',')) { + customerIdsIgnore = customer_ids_ignore.split(','); + } + else { + customerIdsIgnore = [customer_ids_ignore]; + } + } if (!customerIds || customerIds.length === 0) { throw new Error("Customer id is not specified in either 'customer_id' query argument or google-ads.yaml"); } @@ -70,6 +80,7 @@ async function main_getcids_unsafe(req, res, logger) { await logger.info(`Reshaped customer ids array from ${cids_length} to ${customerIds.length} items`); } } + customerIds = customerIds.filter(cid => customerIdsIgnore.indexOf(cid) < 0); if (req.query.flatten) { res.json(customerIds); res.end(); diff --git a/gcp/functions/build/src/gaarf-getcids.js.map b/gcp/functions/build/src/gaarf-getcids.js.map index 2ebcabd3..9239ec02 100644 --- a/gcp/functions/build/src/gaarf-getcids.js.map +++ b/gcp/functions/build/src/gaarf-getcids.js.map @@ -1 +1 @@ -{"version":3,"file":"gaarf-getcids.js","sourceRoot":"","sources":["../../src/gaarf-getcids.ts"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;;;;GAcG;AACH,iFAMuC;AAGvC,mCAAkE;AAClE,qCAA+C;AAE/C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,KAAK,UAAU,mBAAmB,CAChC,GAAoB,EACpB,GAAqB,EACrB,MAAe;IAEf,6BAA6B;IAC7B,MAAM,SAAS,GAAuB,MAAM,IAAA,oBAAY,EAAC,GAAG,CAAC,CAAC;IAC9D,6DAA6D;IAC7D,MAAM,EAAC,aAAa,EAAE,GAAG,mBAAmB,EAAC,GAAG,SAAS,CAAC;IAC1D,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC;IAEzD,IAAI,WAAW,GAAG,IAAA,gDAAgB,EAAS,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IAE7E,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5C,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;KACH;IACD,IAAI,CAAC,SAAS,CAAC,iBAAiB,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;QAC3E,SAAS,CAAC,iBAAiB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;KAC9C;IACD,MAAM,SAAS,GAAG,IAAI,kDAAkB,CAAC,SAAS,CAAC,CAAC;IACpD,WAAW,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAC1D,IAAI,kBAAkB,GAAG,EAAE,CAAC;IAC5B,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,kBAAkB,EAAE;QAC3C,kBAAkB,GAAW,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC;KAC1D;SAAM,IAAI,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;QACvC,kBAAkB,GAAG,MAAM,IAAA,8CAAc,EAC/B,GAAG,CAAC,KAAK,CAAC,kBAAkB,CACrC,CAAC;KACH;IACD,IAAI,kBAAkB,EAAE;QACtB,MAAM,MAAM,CAAC,IAAI,CACf,4CAA4C,kBAAkB,EAAE,CACjE,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,gDAAgB,CAAC,SAAS,CAAC,CAAC;QACjD,WAAW,GAAG,MAAM,QAAQ,CAAC,cAAc,CACzC,WAAW,EACX,kBAAkB,CACnB,CAAC;QACF,MAAM,MAAM,CAAC,IAAI,CACf,UAAU,WAAW,CAAC,MAAM,WAAW,CACxC,CAAC;KACH;IACD,WAAW,GAAG,WAAW,IAAI,EAAE,CAAC;IAChC,WAAW,CAAC,IAAI,EAAE,CAAC;IAEnB,qDAAqD;IACrD,IAAI,SAAS,GAAG,kBAAkB,CAAC;IACnC,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,EAAE;QACpC,SAAS,GAAG,MAAM,CAAS,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE;YACpB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC9D;KACF;IACD,IAAI,GAAG,CAAC,KAAK,CAAC,mBAAmB,EAAE;QACjC,kDAAkD;QAClD,MAAM,MAAM,GAAG,MAAM,CAAS,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;SAC3D;QACD,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;QACvC,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;QAC5D,IAAI,WAAW,KAAK,WAAW,CAAC,MAAM,EAAE;YACtC,MAAM,MAAM,CAAC,IAAI,CACf,oCAAoC,WAAW,OAAO,WAAW,CAAC,MAAM,QAAQ,CACjF,CAAC;SACH;KACF;IAED,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE;QACrB,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtB,GAAG,CAAC,GAAG,EAAE,CAAC;KACX;SAAM;QACL,8DAA8D;QAC9D,qEAAqE;QACrE,MAAM,kBAAkB,GAAG,IAAA,uBAAe,EAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACnE,GAAG,CAAC,IAAI,CAAC;YACP,UAAU,EAAE,kBAAkB,CAAC,MAAM;YACrC,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,kBAAkB;SAC7B,CAAC,CAAC;QACH,GAAG,CAAC,GAAG,EAAE,CAAC;KACX;AACH,CAAC;AAEM,MAAM,YAAY,GAAiB,KAAK,EAC7C,GAAoB,EACpB,GAAqB,EACrB,EAAE;IACF,MAAM,SAAS,GAAG,MAAM,IAAA,kBAAU,GAAE,CAAC;IACrC,MAAM,MAAM,GAAG,IAAA,qBAAY,EACzB,GAAG,EACH,SAAS,EACT,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,eAAe,CACzC,CAAC;IACF,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAC,CAAC,CAAC;IAEjE,IAAI;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;KAC7C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,EAAC,KAAK,EAAE,CAAC,EAAC,CAAC,CAAC;QAC1C,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;KACvC;AACH,CAAC,CAAC;AAlBW,QAAA,YAAY,gBAkBvB"} \ No newline at end of file +{"version":3,"file":"gaarf-getcids.js","sourceRoot":"","sources":["../../src/gaarf-getcids.ts"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;;;;GAcG;AACH,iFAMuC;AAGvC,mCAAkE;AAClE,qCAA+C;AAE/C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,KAAK,UAAU,mBAAmB,CAChC,GAAoB,EACpB,GAAqB,EACrB,MAAe;IAEf,6BAA6B;IAC7B,MAAM,SAAS,GAAuB,MAAM,IAAA,oBAAY,EAAC,GAAG,CAAC,CAAC;IAC9D,6DAA6D;IAC7D,MAAM,EAAC,aAAa,EAAE,GAAG,mBAAmB,EAAC,GAAG,SAAS,CAAC;IAC1D,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC;IAEzD,IAAI,WAAW,GAAG,IAAA,gDAAgB,EAAS,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IAC7E,IAAI,iBAAiB,GAAkB,EAAE,CAAC;IAC1C,IAAI,GAAG,CAAC,KAAK,CAAC,mBAAmB,EAAE;QACjC,MAAM,mBAAmB,GAAW,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC;QAClE,IAAI,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YACrC,iBAAiB,GAAG,mBAAmB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACpD;aAAM;YACL,iBAAiB,GAAG,CAAC,mBAAmB,CAAC,CAAC;SAC3C;KACF;IACD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5C,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;KACH;IACD,IAAI,CAAC,SAAS,CAAC,iBAAiB,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;QAC3E,SAAS,CAAC,iBAAiB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;KAC9C;IACD,MAAM,SAAS,GAAG,IAAI,kDAAkB,CAAC,SAAS,CAAC,CAAC;IACpD,WAAW,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAC1D,IAAI,kBAAkB,GAAG,EAAE,CAAC;IAC5B,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,kBAAkB,EAAE;QAC3C,kBAAkB,GAAW,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC;KAC1D;SAAM,IAAI,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;QACvC,kBAAkB,GAAG,MAAM,IAAA,8CAAc,EAC/B,GAAG,CAAC,KAAK,CAAC,kBAAkB,CACrC,CAAC;KACH;IACD,IAAI,kBAAkB,EAAE;QACtB,MAAM,MAAM,CAAC,IAAI,CACf,4CAA4C,kBAAkB,EAAE,CACjE,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,gDAAgB,CAAC,SAAS,CAAC,CAAC;QACjD,WAAW,GAAG,MAAM,QAAQ,CAAC,cAAc,CACzC,WAAW,EACX,kBAAkB,CACnB,CAAC;QACF,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,MAAM,WAAW,CAAC,CAAC;KAC5D;IACD,WAAW,GAAG,WAAW,IAAI,EAAE,CAAC;IAChC,WAAW,CAAC,IAAI,EAAE,CAAC;IAEnB,qDAAqD;IACrD,IAAI,SAAS,GAAG,kBAAkB,CAAC;IACnC,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,EAAE;QACpC,SAAS,GAAG,MAAM,CAAS,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE;YACpB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC9D;KACF;IACD,IAAI,GAAG,CAAC,KAAK,CAAC,mBAAmB,EAAE;QACjC,kDAAkD;QAClD,MAAM,MAAM,GAAG,MAAM,CAAS,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;SAC3D;QACD,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;QACvC,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;QAC5D,IAAI,WAAW,KAAK,WAAW,CAAC,MAAM,EAAE;YACtC,MAAM,MAAM,CAAC,IAAI,CACf,oCAAoC,WAAW,OAAO,WAAW,CAAC,MAAM,QAAQ,CACjF,CAAC;SACH;KACF;IAED,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5E,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE;QACrB,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtB,GAAG,CAAC,GAAG,EAAE,CAAC;KACX;SAAM;QACL,8DAA8D;QAC9D,qEAAqE;QACrE,MAAM,kBAAkB,GAAG,IAAA,uBAAe,EAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACnE,GAAG,CAAC,IAAI,CAAC;YACP,UAAU,EAAE,kBAAkB,CAAC,MAAM;YACrC,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,kBAAkB;SAC7B,CAAC,CAAC;QACH,GAAG,CAAC,GAAG,EAAE,CAAC;KACX;AACH,CAAC;AAEM,MAAM,YAAY,GAAiB,KAAK,EAC7C,GAAoB,EACpB,GAAqB,EACrB,EAAE;IACF,MAAM,SAAS,GAAG,MAAM,IAAA,kBAAU,GAAE,CAAC;IACrC,MAAM,MAAM,GAAG,IAAA,qBAAY,EACzB,GAAG,EACH,SAAS,EACT,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,eAAe,CACzC,CAAC;IACF,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAC,CAAC,CAAC;IAEjE,IAAI;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;KAC7C;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,EAAC,KAAK,EAAE,CAAC,EAAC,CAAC,CAAC;QAC1C,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;KACvC;AACH,CAAC,CAAC;AAlBW,QAAA,YAAY,gBAkBvB"} \ No newline at end of file diff --git a/gcp/functions/src/gaarf-getcids.ts b/gcp/functions/src/gaarf-getcids.ts index cf229cfe..ce4be14a 100644 --- a/gcp/functions/src/gaarf-getcids.ts +++ b/gcp/functions/src/gaarf-getcids.ts @@ -39,7 +39,15 @@ async function main_getcids_unsafe( await logger.info('Ads API config', ads_config_wo_token); let customerIds = parseCustomerIds(req.query.customer_id, adsConfig); - + let customerIdsIgnore: Array = []; + if (req.query.customer_ids_ignore) { + const customer_ids_ignore = req.query.customer_ids_ignore; + if (customer_ids_ignore.includes(',')) { + customerIdsIgnore = customer_ids_ignore.split(','); + } else { + customerIdsIgnore = [customer_ids_ignore]; + } + } if (!customerIds || customerIds.length === 0) { throw new Error( "Customer id is not specified in either 'customer_id' query argument or google-ads.yaml" @@ -67,9 +75,7 @@ async function main_getcids_unsafe( customerIds, customer_ids_query ); - await logger.info( - `Loaded ${customerIds.length} accounts` - ); + await logger.info(`Loaded ${customerIds.length} accounts`); } customerIds = customerIds || []; customerIds.sort(); @@ -97,6 +103,8 @@ async function main_getcids_unsafe( } } + customerIds = customerIds.filter(cid => customerIdsIgnore.indexOf(cid) < 0); + if (req.query.flatten) { res.json(customerIds); res.end(); diff --git a/gcp/workflow/workflow-ads.yaml b/gcp/workflow/workflow-ads.yaml index 5123e444..9b39aa84 100644 --- a/gcp/workflow/workflow-ads.yaml +++ b/gcp/workflow/workflow-ads.yaml @@ -14,7 +14,6 @@ main: - macros: ${map.get(args, "macros")} - bq_writer_options: ${map.get(args, "bq_writer_options")} - concurrency_limit: ${default(map.get(args, "concurrency_limit"), 20)} - - disable_strict_views: ${default(map.get(args,"disable_strict_views"), false)} # get CF 'gaarf' function's URL - get_function: @@ -29,18 +28,9 @@ main: #args: # name: ${"projects/" + project + "/locations/" + location + "/functions/" + cloud_function} #result: function - # get CF 'gaarf-bq-view' function's URL - - get_function_view: - call: http.get - args: - url: ${"https://cloudfunctions.googleapis.com/v2/projects/" + project + "/locations/" + location + "/functions/" + cloud_function + "-bq-view"} - auth: - type: OAuth2 - result: function_view - initialize_functions_uris: assign: - gaarf_main_cf_uri: ${function.body.serviceConfig.uri} - - gaarf_bqview_cf_uri: ${function_view.body.serviceConfig.uri} - iterate_over_scripts: parallel: @@ -91,22 +81,6 @@ main: bq_dataset_location: ${bq_dataset_location} ads_config_path: ${ads_config_path} bq_writer_options: ${bq_writer_options} - # create a view in BQ to combine all account tables into a single view - - call_create_view_cf: - call: http.post - args: - url: ${gaarf_bqview_cf_uri} - timeout: 1800 - query: - project_id: ${project} - dataset: ${bq_dataset} - dataset_location: ${bq_dataset_location} - script_path: ${script_item} - body: - accounts: ${if(disable_strict_views, [], accounts)} - auth: - type: OIDC - result: create_view_response executeAdsQuery: params: [cf_uri, script_path, account, macros, project, bq_dataset, bq_dataset_location, ads_config_path, bq_writer_options, is_constant: false] diff --git a/gcp/workflow/workflow.yaml b/gcp/workflow/workflow.yaml index 905c2902..7c5679c7 100644 --- a/gcp/workflow/workflow.yaml +++ b/gcp/workflow/workflow.yaml @@ -30,6 +30,7 @@ main: queries_path: ${args.ads_queries_path} ads_config_path: ${args.ads_config_path} cid: ${args.cid} + cid_ignore: ${map.get(args, "cid_ignore")} customer_ids_query: ${map.get(args, "customer_ids_query")} customer_ids_offset: ${map.get(args, "customer_ids_offset")} customer_ids_batchsize: ${map.get(args, "customer_ids_batchsize")} @@ -69,7 +70,7 @@ main: return: ${accounts} runAdsQueries: - params: [project, location, function_name, gcs_bucket, queries_path, ads_config_path, cid, customer_ids_query, customer_ids_offset, customer_ids_batchsize, bq_dataset, bq_dataset_location, macros, bq_writer_options, concurrency_limit, workflow_ads_id, disable_strict_views] + params: [project, location, function_name, gcs_bucket, queries_path, ads_config_path, cid, cid_ignore, customer_ids_query, customer_ids_offset, customer_ids_batchsize, bq_dataset, bq_dataset_location, macros, bq_writer_options, concurrency_limit, workflow_ads_id, disable_strict_views] # NOTE: currently it's assumed that CF's project is the same as project for BQ datasets steps: # get CF 'gaarf-getcids' function's URL @@ -80,6 +81,14 @@ runAdsQueries: auth: type: OAuth2 result: function_cids + # get CF 'gaarf-bq-view' function's URL + - get_function_view: + call: http.get + args: + url: ${"https://cloudfunctions.googleapis.com/v2/projects/" + project + "/locations/" + location + "/functions/" + function_name + "-bq-view"} + auth: + type: OAuth2 + result: function_view #call 'gaarf-getcids' CF to get a list of customer ids for further processing - call_gaarf_cids_cf: @@ -90,6 +99,7 @@ runAdsQueries: query: ads_config_path: ${ads_config_path} customer_id: ${cid} + customer_ids_ignore: ${default(cid_ignore, "")} customer_ids_query: ${default(customer_ids_query, "")} customer_ids_offset: ${default(customer_ids_offset, "")} customer_ids_batchsize: ${default(customer_ids_batchsize, "")} @@ -172,9 +182,53 @@ runAdsQueries: macros: ${macros} bq_writer_options: ${bq_writer_options} concurrency_limit: ${concurrency_limit} - disable_strict_views: ${disable_strict_views} result: execution_result + # collection account ids from `accounts` where they are slit into batched into a flatten list + - initialize_accounts_flatten: + assign: + - accounts_flatten: [] + - flatten_accounts: + for: + value: accounts_batch + in: ${accounts} + steps: + - loop_over_batch: + for: + value: account + in: ${accounts_batch} + steps: + - append: + assign: + - accounts_flatten: ${list.concat(accounts_flatten, account)} + + # for each script (excluding constants) create a unified view + - create_views: + for: + value: script_item + in: ${scripts} + steps: + - check_for_constant: + switch: + - condition: ${text.match_regex(script_item, "_constant")} + next: continue + next: call_create_view_cf + - call_create_view_cf: + call: http.post + args: + url: ${function_view.body.serviceConfig.uri} + timeout: 1800 + query: + project_id: ${project} + dataset: ${bq_dataset} + dataset_location: ${bq_dataset_location} + script_path: ${script_item} + body: + accounts: ${if(disable_strict_views, [], accounts_flatten)} + auth: + type: OIDC + result: create_view_response + - return_result: return: ${accounts}