|
| 1 | +{% comment %} |
| 2 | + -- using the DummyJSON service for this demonstration, with dummy data for 194 products |
| 3 | + -- using a page limit of 50 should result in 4 calls to the API to get all of the products |
| 4 | +{% endcomment %} |
| 5 | + |
| 6 | +{% assign api_endpoint = "https://dummyjson.com/products" %} |
| 7 | +{% assign limit = 50 %} |
| 8 | + |
| 9 | +{% if event.topic == "mechanic/user/trigger" %} |
| 10 | + {% comment %} |
| 11 | + -- use query parameters to limit the results and only return the id (default), sku, and stock |
| 12 | + {% endcomment %} |
| 13 | + |
| 14 | + {%- capture api_endpoint_with_parameters -%} |
| 15 | + {{ api_endpoint }}?select=sku,stock&limit={{ limit }} |
| 16 | + {%- endcapture -%} |
| 17 | + |
| 18 | + {% comment %} |
| 19 | + -- make the first http get call to the external api |
| 20 | + -- note: on this first call we can use the standard http action format since we don't yet need to pass meta values (e.g. products) |
| 21 | + {% endcomment %} |
| 22 | + |
| 23 | + {% action "http" %} |
| 24 | + { |
| 25 | + "method": "get", |
| 26 | + "url": {{ api_endpoint_with_parameters | json }}, |
| 27 | + "verify": "true", |
| 28 | + "error_on_5xx": "true" |
| 29 | + } |
| 30 | + {% endaction %} |
| 31 | + |
| 32 | +{% elsif event.topic == "mechanic/actions/perform" %} |
| 33 | + {% comment %} |
| 34 | + -- create preview stub data to simulate results that indicate there is more data to query |
| 35 | + {% endcomment %} |
| 36 | + |
| 37 | + {% if event.preview %} |
| 38 | + {% capture action_json %} |
| 39 | + { |
| 40 | + "type": "http", |
| 41 | + "meta": null, |
| 42 | + "run": { |
| 43 | + "ok": true, |
| 44 | + "result": { |
| 45 | + "status": 200, |
| 46 | + "body": { |
| 47 | + "products": [ |
| 48 | + { |
| 49 | + "id": 1, |
| 50 | + "sku": "RCH45Q1A", |
| 51 | + "stock": 5 |
| 52 | + } |
| 53 | + ], |
| 54 | + "total": 194, |
| 55 | + "skip": 0, |
| 56 | + "limit": 50 |
| 57 | + } |
| 58 | + } |
| 59 | + } |
| 60 | + } |
| 61 | + {% endcapture %} |
| 62 | + |
| 63 | + {% assign action = action_json | parse_json %} |
| 64 | + {% endif %} |
| 65 | + |
| 66 | + {% comment %} |
| 67 | + -- for demonstration purposes, only process http actions that ran successfully |
| 68 | + -- typically, some error checking would be done here on the status code to see if a different action should be taken (e.g. 429 - Too many requests) |
| 69 | + {% endcomment %} |
| 70 | + |
| 71 | + {% unless action.type == "http" and action.run.ok and action.run.result.status == 200 %} |
| 72 | + {% break %} |
| 73 | + {% endunless %} |
| 74 | + |
| 75 | + {% comment %} |
| 76 | + -- assign api results to variables |
| 77 | + {% endcomment %} |
| 78 | + |
| 79 | + {% assign result_products = action.run.result.body.products %} |
| 80 | + {% assign result_total = action.run.result.body.total %} |
| 81 | + {% assign result_skip = action.run.result.body.skip %} |
| 82 | + {% assign result_limit = action.run.result.body.limit %} |
| 83 | + |
| 84 | + {% comment %} |
| 85 | + -- get products passed in meta if this is not the first call, otherwise default to an empty array |
| 86 | + {% endcomment %} |
| 87 | + |
| 88 | + {% assign fetched_products = action.meta.fetched_products | default: array %} |
| 89 | + |
| 90 | + {% comment %} |
| 91 | + -- concatenate the new results to the fetched products array |
| 92 | + {% endcomment %} |
| 93 | + |
| 94 | + {% assign fetched_products = fetched_products | concat: result_products %} |
| 95 | + |
| 96 | + {% comment %} |
| 97 | + -- check to see if another API call should be made |
| 98 | + -- the DummyJSON API uses limit based pagination with a skip parameter, and the limit returned will indicate the current size |
| 99 | + -- this means the total objects retrieved so far should be the sum of the returned skip and limit values |
| 100 | + {% endcomment %} |
| 101 | + |
| 102 | + {% assign count_products_retrieved = result_skip | plus: result_limit %} |
| 103 | + |
| 104 | + {% if count_products_retrieved < result_total %} |
| 105 | + {% comment %} |
| 106 | + -- we do not yet have the full set of products; log out how many have been retrieved and make the next api call |
| 107 | + -- note: when accessing an api with low rate limiting, then you may want to use a delay strategy [see tutorial for this demo task for more detail] |
| 108 | + {% endcomment %} |
| 109 | + |
| 110 | + {% log |
| 111 | + count_products_retrieved: count_products_retrieved, |
| 112 | + total_products_expected: result_total |
| 113 | + %} |
| 114 | + |
| 115 | + {%- capture api_endpoint_with_parameters -%} |
| 116 | + {{ api_endpoint }}?select=sku,stock&limit={{ limit }}&skip={{ count_products_retrieved }} |
| 117 | + {%- endcapture -%} |
| 118 | + |
| 119 | + {% comment %} |
| 120 | + -- in order to pass the array of products retrieved so far, we need to use the http action format that supports passing meta values |
| 121 | + {% endcomment %} |
| 122 | + |
| 123 | + {% action %} |
| 124 | + { |
| 125 | + "type": "http", |
| 126 | + "options": { |
| 127 | + "method": "get", |
| 128 | + "url": {{ api_endpoint_with_parameters | json }}, |
| 129 | + "verify": "true", |
| 130 | + "error_on_5xx": "true" |
| 131 | + }, |
| 132 | + "meta": { |
| 133 | + "fetched_products": {{ fetched_products | json }} |
| 134 | + } |
| 135 | + } |
| 136 | + {% endaction %} |
| 137 | + |
| 138 | + {% break %} |
| 139 | + {% endif %} |
| 140 | + |
| 141 | + {% comment %} |
| 142 | + -- we've reached the end of the api results, pass the fetched products array to a custom event for additional processing |
| 143 | + -- note: the products could be processed here, but using a custom event is a useful indicator in the event logs that the task has finished fetching products |
| 144 | + {% endcomment %} |
| 145 | + |
| 146 | + {% action "event" %} |
| 147 | + { |
| 148 | + "topic": "user/demo_paginated_api/fetched_products", |
| 149 | + "task_id": {{ task.id | json }}, |
| 150 | + "data": { |
| 151 | + "fetched_products":{{ fetched_products | json }} |
| 152 | + } |
| 153 | + } |
| 154 | + {% endaction %} |
| 155 | + |
| 156 | +{% elsif event.topic == "user/demo_paginated_api/fetched_products" %} |
| 157 | + {% comment %} |
| 158 | + -- for this demo, just log out the fetched products |
| 159 | + {% endcomment %} |
| 160 | + |
| 161 | + {% assign fetched_products = event.data.fetched_products %} |
| 162 | + |
| 163 | + {% log %} |
| 164 | + "Fetched {{ fetched_products.size }} products from the external API." |
| 165 | + {% endlog %} |
| 166 | + |
| 167 | + {% log fetched_products: fetched_products %} |
| 168 | + |
| 169 | + {% log "Fetched products would be processed here..." %} |
| 170 | +{% endif %} |
0 commit comments