[SAC-27536] Frontapp App libary changes

Author: gl-romil

.circleci/config.yml

@@ -2,7 +2,8 @@ version: 2
 jobs:
   build:
     docker:
-      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:tap-tester-v4
+      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester
+
     steps:
       - checkout
       - run:

.gitignore

@@ -102,3 +102,6 @@ tags
 singer-check-tap-data
 state.json
 catalog.json
+output.json
+convert_json_csv.py
+notes_frontapp.txt

example.config.json

@@ -10,11 +10,11 @@
     url="http://singer.io",
     classifiers=["Programming Language :: Python :: 3 :: Only"],
     install_requires=[
-        "singer-python==5.4.0",
-        "pendulum",
-        "ratelimit",
-        "backoff==1.3.2",
-        "requests==2.31.0",
+        "singer-python==6.1.1",
+        "pendulum==3.1.0",
+        "ratelimit==2.2.1",
+        "backoff==2.2.1",
+        "requests==2.32.3",
     ],
     entry_points="""
     [console_scripts]
@@ -25,4 +25,4 @@
         "schemas": ["tap_frontapp/schemas/*.json"]
     },
     include_package_data=True
-)
+)

setup.py

@@ -23,40 +23,117 @@
 #  call to the api but with the odd frontapp structure, we won't do that
 #  here we never use atx in here since the schema is from file but we
 #  would use it if we pulled schema from the API def discover(atx):
+
+
 def discover():
     catalog = Catalog([])
+    
+    # Build initial catalog from schema files
     for tap_stream_id in schemas.STATIC_SCHEMA_STREAM_IDS:
-        #print("tap stream id=",tap_stream_id)
+        LOGGER.info("tap stream id=%s", tap_stream_id)
         schema = Schema.from_dict(schemas.load_schema(tap_stream_id))
         metadata = []
+        
+        # Stream-level metadata select the stream
+        metadata.append({
+            "metadata": {
+                "selected": True  # Make sure to select every stream
+            },
+            "breadcrumb": []
+        })
+        
+        # Field level metadata with inclusion type
         for field_name in schema.properties.keys():
-            #print("field name=",field_name)
             if field_name in schemas.PK_FIELDS[tap_stream_id]:
                 inclusion = 'automatic'
             else:
                 inclusion = 'available'
             metadata.append({
-                'metadata': {
-                    'inclusion': inclusion
+                "metadata": {
+                    "inclusion": inclusion
                 },
-                'breadcrumb': ['properties', field_name]
+                "breadcrumb": ['properties', field_name]
             })
+        
         catalog.streams.append(CatalogEntry(
             stream=tap_stream_id,
             tap_stream_id=tap_stream_id,
             key_properties=schemas.PK_FIELDS[tap_stream_id],
             schema=schema,
             metadata=metadata
         ))
-    return catalog
+    
+    # Creating a dict to change before converting
+    catalog_dict = catalog.to_dict()
+    
+    required_streams = [
+        "accounts_table",
+        "channels_table",
+        "inboxes_table",
+        "tags_table",
+        "teammates_table",
+        "teams_table"
+    ]
+    
+    # We verify this to ensure all mandatory streams are available even if schema files are missing
+    present_streams = {stream['tap_stream_id'] for stream in catalog_dict['streams']}
+    
+    # Ensure all required streams are included even if schema is missing
+    for stream_name in required_streams:
+        if stream_name not in present_streams:
+
+            LOGGER.info("Adding missing required stream: %s", stream_name)
+            
+            # Create a minimal stream entry that will be visible in the output
+            catalog_dict['streams'].append({
+                "stream": stream_name,
+                "tap_stream_id": stream_name,
+                "schema": {
+                    "type": ['null', 'object'],
+                    "properties": {},
+                    "additionalProperties": False
+                },
+                "key_properties": [],
+                "metadata": [
+                    {
+                        "metadata": {
+                            "selected": True
+                        },
+                        "breadcrumb": []
+                    }
+                ]
+            })
+    
+    # This ensure singer tools recognize all streams in the catalog dictionary
+    for stream in catalog_dict['streams']:
+        has_selection = False
+        for metadata_item in stream.get('metadata', []):
+            if metadata_item.get('breadcrumb') == [] and metadata_item.get('metadata', {}).get('selected') is True:
+                has_selection = True
+                break
+                
+        if not has_selection:
+            LOGGER.info("Adding selection metadata to stream: %s", stream['tap_stream_id'])
+            stream.setdefault('metadata', []).insert(0, {
+                "metadata": {
+                    "selected": True
+                },
+                "breadcrumb": []
+            })
+    
+    # Convert back to a Catalog object
+    modified_catalog = Catalog.from_dict(catalog_dict)
+
+    
+    return modified_catalog
 
 
 def get_abs_path(path: str):
     """Returns absolute path for URL."""
     return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
 
 
-# this is already defined in schemas.py though w/o dependencies.  do we keep this for the sync?
+# this is already defined in schemas.py though w/o dependencies
 def load_schema(tap_stream_id):
     path = "schemas/{}.json".format(tap_stream_id)
     schema = utils.load_json(get_abs_path(path))
@@ -90,4 +167,4 @@ def main():
         sync(atx)
 
 if __name__ == "__main__":
-    main()
+    main()

tap_frontapp/__init__.py

@@ -0,0 +1,81 @@
+import unittest
+from unittest.mock import patch
+import requests
+from tap_frontapp.http import Client, RateLimitException, MetricsRateLimitException
+from requests.exceptions import Timeout, ConnectionError
+import json
+
+
+class MockResponse:
+    """Mock response object to simulate API calls."""
+    def __init__(self, status_code, headers=None, json_data=None, raise_for_status=False):
+        self.status_code = status_code
+        self.headers = headers or {}
+        self._json_data = json_data or {}
+        self._raise_for_status = raise_for_status
+        self.content = json.dumps(self._json_data)
+        self.text = json.dumps(self._json_data)
+
+    def json(self):
+        return self._json_data
+
+    def raise_for_status(self):
+        if self._raise_for_status:
+            raise requests.HTTPError("Mocker Error")
+        return self.status_code
+
+
+def get_mock_response(status_code=200, raise_for_status=False, json_data=None, headers=None):
+    return MockResponse(status_code, headers=headers, json_data=json_data, raise_for_status=raise_for_status)
+
+
+class TestFrontAppClient(unittest.TestCase):
+
+    @patch("requests.request")
+    def test_successful_request(self, mock_request):
+        mock_request.return_value = get_mock_response(
+            status_code=200,
+            headers={"X-Ratelimit-Remaining": "100", "X-Ratelimit-Reset": "1000"},
+            json_data={"metrics": [{"id": "m1"}]}
+        )
+        client = Client(config={"token": "test-token"})
+        response = client.get_report_metrics("https://api2.frontapp.com/analytics/reports/xyz")
+        self.assertEqual(response, [{"id": "m1"}])
+
+    @patch("requests.request")
+    def test_rate_limit_429(self, mock_request):
+        mock_request.return_value = get_mock_response(
+            status_code=429,
+            headers={"X-Ratelimit-Remaining": "0", "X-Ratelimit-Reset": "999"},
+            raise_for_status=True
+        )
+        client = Client(config={"token": "test-token"})
+        with self.assertRaises(RateLimitException):
+            client.get_report_metrics("https://api2.frontapp.com/analytics/reports/xyz")
+
+    @patch("requests.request")
+    def test_metrics_rate_limit_423(self, mock_request):
+        mock_request.return_value = get_mock_response(
+            status_code=423,
+            headers={"X-Ratelimit-Remaining": "10", "X-Ratelimit-Reset": "999"},
+            raise_for_status=True
+        )
+        client = Client(config={"token": "test-token"})
+        with self.assertRaises(MetricsRateLimitException):
+            client.get_report_metrics("https://api2.frontapp.com/analytics/reports/xyz")
+
+    @patch("requests.request", side_effect=Timeout)
+    def test_timeout_handling(self, mock_request):
+        client = Client(config={"token": "test-token"})
+        with self.assertRaises(Timeout):
+            client.get_report_metrics("https://api2.frontapp.com/analytics/reports/xyz")
+
+    @patch("requests.request", side_effect=ConnectionError)
+    def test_connection_error_handling(self, mock_request):
+        client = Client(config={"token": "test-token"})
+        with self.assertRaises(ConnectionError):
+            client.get_report_metrics("https://api2.frontapp.com/analytics/reports/xyz")
+
+
+if __name__ == "__main__":
+    unittest.main()