Fix discoverable metadata

Author: romilqlik

tap_frontapp/__init__.py

@@ -15,132 +15,88 @@
 
 LOGGER = singer.get_logger()
 
-#def check_authorization(atx):
-#    atx.client.get('/settings')
-
-
-# Some taps do discovery dynamically where the catalog is read in from a
-#  call to the api but with the odd frontapp structure, we won't do that
-#  here we never use atx in here since the schema is from file but we
-#  would use it if we pulled schema from the API def discover(atx):
-
 
 def discover():
     catalog = Catalog([])
-    
+
     # Build initial catalog from schema files
     for tap_stream_id in schemas.STATIC_SCHEMA_STREAM_IDS:
         LOGGER.info("tap stream id=%s", tap_stream_id)
         schema = Schema.from_dict(schemas.load_schema(tap_stream_id))
         metadata = []
-        
-        # Stream-level metadata select the stream
+
+        # Add discoverable stream-level metadata
         metadata.append({
             "metadata": {
-                "selected": True  # Make sure to select every stream
+                "inclusion": "available",
+                "selected-by-default": True,
+                "inclusion-reason": "automatic"
             },
             "breadcrumb": []
         })
-        
-        # Field level metadata with inclusion type
+
+        # Add discoverable field-level metadata
         for field_name in schema.properties.keys():
-            if field_name in schemas.PK_FIELDS[tap_stream_id]:
-                inclusion = 'automatic'
-            else:
-                inclusion = 'available'
+            inclusion = "automatic" if field_name in schemas.PK_FIELDS[tap_stream_id] else "available"
             metadata.append({
                 "metadata": {
-                    "inclusion": inclusion
+                    "inclusion": inclusion,
+                    "selected-by-default": True,
+                    "inclusion-reason": "manual"
                 },
-                "breadcrumb": ['properties', field_name]
+                "breadcrumb": ["properties", field_name]
             })
-        
+
         catalog.streams.append(CatalogEntry(
             stream=tap_stream_id,
             tap_stream_id=tap_stream_id,
             key_properties=schemas.PK_FIELDS[tap_stream_id],
             schema=schema,
             metadata=metadata
         ))
-    
-    # Creating a dict to change before converting
+
     catalog_dict = catalog.to_dict()
-    
-    required_streams = [
-        "accounts_table",
-        "channels_table",
-        "inboxes_table",
-        "tags_table",
-        "teammates_table",
-        "teams_table"
-    ]
-    
-    # We verify this to ensure all mandatory streams are available even if schema files are missing
-    present_streams = {stream['tap_stream_id'] for stream in catalog_dict['streams']}
-    
-    # Ensure all required streams are included even if schema is missing
-    for stream_name in required_streams:
-        if stream_name not in present_streams:
+    present_streams = {stream["tap_stream_id"] for stream in catalog_dict["streams"]}
 
+    # Use METRIC_API_DESCRIPTION_KEY keys instead of hardcoded list
+    for stream_name in streams.METRIC_API_DESCRIPTION_KEY.keys():
+        if stream_name not in present_streams:
             LOGGER.info("Adding missing required stream: %s", stream_name)
-            
-            # Create a minimal stream entry that will be visible in the output
-            catalog_dict['streams'].append({
+            catalog_dict["streams"].append({
                 "stream": stream_name,
                 "tap_stream_id": stream_name,
                 "schema": {
-                    "type": ['null', 'object'],
+                    "type": ["null", "object"],
                     "properties": {},
                     "additionalProperties": False
                 },
                 "key_properties": [],
                 "metadata": [
                     {
                         "metadata": {
-                            "selected": True
+                            "inclusion": "available",
+                            "selected-by-default": True,
+                            "inclusion-reason": "automatic"
                         },
                         "breadcrumb": []
                     }
                 ]
             })
-    
-    # This ensure singer tools recognize all streams in the catalog dictionary
-    for stream in catalog_dict['streams']:
-        has_selection = False
-        for metadata_item in stream.get('metadata', []):
-            if metadata_item.get('breadcrumb') == [] and metadata_item.get('metadata', {}).get('selected') is True:
-                has_selection = True
-                break
-                
-        if not has_selection:
-            LOGGER.info("Adding selection metadata to stream: %s", stream['tap_stream_id'])
-            stream.setdefault('metadata', []).insert(0, {
-                "metadata": {
-                    "selected": True
-                },
-                "breadcrumb": []
-            })
-    
-    # Convert back to a Catalog object
-    modified_catalog = Catalog.from_dict(catalog_dict)
 
-    
-    return modified_catalog
+    return Catalog.from_dict(catalog_dict)
 
 
 def get_abs_path(path: str):
-    """Returns absolute path for URL."""
+    """Returns absolute path for a given relative path."""
     return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
 
 
-# this is already defined in schemas.py though w/o dependencies
 def load_schema(tap_stream_id):
-    path = "schemas/{}.json".format(tap_stream_id)
+    """Loads schema from JSON file, resolving dependencies."""
+    path = f"schemas/{tap_stream_id}.json"
     schema = utils.load_json(get_abs_path(path))
     dependencies = schema.pop("tap_schema_dependencies", [])
-    refs = {}
-    for sub_stream_id in dependencies:
-        refs[sub_stream_id] = load_schema(sub_stream_id)
+    refs = {sub_id: load_schema(sub_id) for sub_id in dependencies}
     if refs:
         singer.resolve_schema_references(schema, refs)
     return schema
@@ -149,7 +105,6 @@ def load_schema(tap_stream_id):
 def sync(atx):
     for tap_stream_id in schemas.STATIC_SCHEMA_STREAM_IDS:
         schemas.load_and_write_schema(tap_stream_id)
-
     streams.sync_selected_streams(atx)
 
 
@@ -158,13 +113,12 @@ def main():
     args = utils.parse_args(REQUIRED_CONFIG_KEYS)
     atx = Context(args.config, args.state)
     if args.discover:
-        # the schema is static from file so we don't need to pass in atx for connection info.
         catalog = discover()
         json.dump(catalog.to_dict(), sys.stdout)
     else:
-        atx.catalog = Catalog.from_dict(args.properties) \
-            if args.properties else discover()
+        atx.catalog = Catalog.from_dict(args.properties) if args.properties else discover()
         sync(atx)
 
+
 if __name__ == "__main__":
-    main()
+    main()