update the bookmark value even if the record is not retrieved (#247)

* update the bookmark value even if the record is not retrieved

* setup and changelog update

* add discount applications in the output

* fix the test

* fix copilot review comments

Author: sgandhi1311

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+### 3.8.1
+  * Update the bookmark value even if the record is not retrieved [#247](https://github.com/singer-io/tap-shopify/pull/247)
+
 ### 3.8.0
   * Exponential backoff for Shopify bulk operations in progress [#246](https://github.com/singer-io/tap-shopify/pull/246)
 

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name="tap-shopify",
-    version="3.8.0",
+    version="3.8.1",
     description="Singer.io tap for extracting Shopify data",
     author="Stitch",
     url="http://github.com/singer-io/tap-shopify",

tap_shopify/streams/orders.py

@@ -1004,6 +1004,7 @@ def get_query(self):
                         code
                     }
                     ... on ManualDiscountApplication {
+                        __typename
                         title
                         description
                     }
@@ -1020,6 +1021,14 @@ def get_query(self):
         }
         """
 
+    def is_discount_application(self, rec):
+        if '__typename' in rec and rec['__typename'] in ['AutomaticDiscountApplication',
+                                                       'DiscountCodeApplication',
+                                                       'ManualDiscountApplication',
+                                                       'ScriptDiscountApplication']:
+            return True
+        return False
+
     def update_bookmark(self, bookmark_value, bookmark_key=None, bulk_op_metadata=None):
         # Standard Singer bookmark
         singer.write_bookmark(
@@ -1163,6 +1172,7 @@ def parse_bulk_jsonl(self, url):
         resp = requests.get(url, stream=True, timeout=60)
         current_order = None
         current_line_items = []
+        current_discount_applications = []
 
         for line in resp.iter_lines():
             if not line:
@@ -1173,18 +1183,25 @@ def parse_bulk_jsonl(self, url):
                 continue
             # Detect line item (child) or order (parent)
             if '__parentId' in rec:
-                # It's a line item belonging to current_order
-                current_line_items.append(rec)
+                if self.is_discount_application(rec):
+                    # It's a discount application belonging to current_order
+                    current_discount_applications.append(rec)
+                else:
+                    # It's a line item belonging to current_order
+                    current_line_items.append(rec)
             else:
                 if current_order:
                     current_order["lineItems"] = current_line_items
+                    current_order["discountApplications"] = current_discount_applications
                     yield current_order
                 # Start tracking new parent group
                 current_order = rec
                 current_line_items = []
+                current_discount_applications = []
         # Yield the last parent group (if exists)
         if current_order:
             current_order["lineItems"] = current_line_items
+            current_order["discountApplications"] = current_discount_applications
             yield current_order
 
     def transform_object(self, obj):
@@ -1254,6 +1271,7 @@ def get_objects(self):
             else:
                 LOGGER.info("No data returned for the date range: %s to %s",
                                last_updated_at, query_end)
+                current_bookmark = max(current_bookmark, query_end)
 
             self.clear_bulk_operation_state()
             last_updated_at = query_end

tests/test_all_fields.py

@@ -12,8 +12,7 @@
         'canDelete',
         'rawMessage',
         'canEdit',
-    },
-    'orders': {'discountApplications'}
+    }
 }
 
 class AllFieldsTest(BaseTapTest):