Added stream access check in discovery

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 1.3.0
+  * Exclude inaccessible streams during discovery and fail discovery when no streams are accessible.
+
 ## 1.2.0
   * Upgrade python version to 3.12 [#12](https://github.com/singer-io/tap-saasoptics/pull/12)
   * Add mock-integration tests

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-saasoptics',
-      version='1.2.0',
+    version='1.3.0',
       description='Singer.io tap for extracting data from the SaaSOptics v1.0 API',
       author='jeff.huth@bytecode.io',
       classifiers=['Programming Language :: Python :: 3 :: Only'],

tap_saasoptics/__init__.py

@@ -19,10 +19,10 @@
     'user_agent'
 ]
 
-def do_discover():
+def do_discover(client):
 
     LOGGER.info('Starting discover')
-    catalog = discover()
+    catalog = discover(client)
     json.dump(catalog.to_dict(), sys.stdout, indent=2)
     LOGGER.info('Finished discover')
 
@@ -42,7 +42,7 @@ def main():
             state = parsed_args.state
 
         if parsed_args.discover:
-            do_discover()
+            do_discover(client)
         elif parsed_args.catalog:
             sync(client=client,
                  config=parsed_args.config,

tap_saasoptics/discover.py

@@ -1,18 +1,55 @@
+import singer
 from singer.catalog import Catalog, CatalogEntry, Schema
-from tap_saasoptics.schema import get_schemas, STREAMS
 
-def discover():
+from tap_saasoptics.client import SaaSOpticsForbiddenError
+from tap_saasoptics.schema import STREAMS, get_schemas
+
+LOGGER = singer.get_logger()
+
+
+def _check_stream_access(client, stream_name, stream_config):
+    path = stream_config.get('path', stream_name)
+    client.get(path=path)
+
+
+def _apply_access_checks(client, streams):
+    if client is None:
+        return streams
+
+    accessible = []
+    inaccessible = []
+
+    for stream_name, stream_config in streams:
+        try:
+            _check_stream_access(client, stream_name, stream_config)
+            accessible.append((stream_name, stream_config))
+        except SaaSOpticsForbiddenError:
+            inaccessible.append(stream_name)
+
+    if inaccessible:
+        LOGGER.warning('Skipping inaccessible streams: %s', ', '.join(inaccessible))
+
+    if not accessible:
+        raise SaaSOpticsForbiddenError(
+            'No streams are accessible. Verify API permissions for the configured token.'
+        )
+
+    return accessible
+
+
+def discover(client=None):
     schemas, field_metadata = get_schemas()
     catalog = Catalog([])
 
-    for stream_name, schema_dict in schemas.items():
-        schema = Schema.from_dict(schema_dict)
+    stream_items = list(STREAMS.items())
+    for stream_name, stream_metadata in _apply_access_checks(client, stream_items):
+        schema = Schema.from_dict(schemas[stream_name])
         mdata = field_metadata[stream_name]
 
         catalog.streams.append(CatalogEntry(
             stream=stream_name,
             tap_stream_id=stream_name,
-            key_properties=STREAMS[stream_name]['key_properties'],
+            key_properties=stream_metadata['key_properties'],
             schema=schema,
             metadata=mdata
         ))

tests/unittests/test_discover.py

@@ -1,18 +1,46 @@
 import unittest
+from unittest.mock import MagicMock
 
 from tap_saasoptics.discover import discover
+from tap_saasoptics.client import SaaSOpticsForbiddenError
 from tap_saasoptics.streams import STREAMS
 
 
 class TestDiscover(unittest.TestCase):
     """Unit tests for discover()."""
 
-    def test_discover_returns_catalog_with_all_streams(self):
+    def test_discover_returns_catalog_with_all_streams_without_client(self):
         """discover() must return a Catalog containing all STREAMS entries."""
         catalog = discover()
         stream_names = {s.tap_stream_id for s in catalog.streams}
         self.assertEqual(stream_names, set(STREAMS.keys()))
 
+    def test_discover_excludes_inaccessible_streams(self):
+        """discover(client) must exclude streams that raise forbidden."""
+        restricted_streams = {'customers', 'contracts'}
+
+        mock_client = MagicMock()
+
+        def _get_side_effect(path, **_kwargs):
+            if path in restricted_streams:
+                raise SaaSOpticsForbiddenError('Forbidden')
+            return {'results': []}
+
+        mock_client.get.side_effect = _get_side_effect
+
+        catalog = discover(mock_client)
+        stream_names = {s.tap_stream_id for s in catalog.streams}
+
+        self.assertEqual(stream_names, set(STREAMS.keys()) - restricted_streams)
+
+    def test_discover_raises_when_no_stream_is_accessible(self):
+        """discover(client) must fail if all streams are inaccessible."""
+        mock_client = MagicMock()
+        mock_client.get.side_effect = SaaSOpticsForbiddenError('Forbidden')
+
+        with self.assertRaises(SaaSOpticsForbiddenError):
+            discover(mock_client)
+
     def test_discover_catalog_entry_stream_equals_tap_stream_id(self):
         """CatalogEntry.stream must equal CatalogEntry.tap_stream_id."""
         catalog = discover()