From ddfde29559f2cd08c4d43de0a658eb520dd957d0 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 16 Apr 2025 13:04:57 +0300 Subject: [PATCH 1/8] add --- .../connector_builder/test_reader/reader.py | 25 +++- .../sources/declarative/declarative_source.py | 11 +- .../manifest_declarative_source.py | 4 + .../models/base_model_with_deprecations.py | 113 ++++++++++++++++++ .../parsers/model_to_component_factory.py | 26 +++- bin/generate_component_manifest_files.py | 69 +++++++++++ .../test_connector_builder_handler.py | 5 +- 7 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py diff --git a/airbyte_cdk/connector_builder/test_reader/reader.py b/airbyte_cdk/connector_builder/test_reader/reader.py index a125047b4..d8b94cd34 100644 --- a/airbyte_cdk/connector_builder/test_reader/reader.py +++ b/airbyte_cdk/connector_builder/test_reader/reader.py @@ -112,11 +112,16 @@ def run_test_read( record_limit = self._check_record_limit(record_limit) # The connector builder currently only supports reading from a single stream at a time stream = source.streams(config)[0] + + # get any deprecation warnings during the component creation + deprecation_warnings: List[AirbyteLogMessage] = source.deprecation_warnings() + schema_inferrer = SchemaInferrer( self._pk_to_nested_and_composite_field(stream.primary_key), self._cursor_field_to_nested_and_composite_field(stream.cursor_field), ) datetime_format_inferrer = DatetimeFormatInferrer() + message_group = get_message_groups( self._read_stream(source, config, configured_catalog, state), schema_inferrer, @@ -125,7 +130,7 @@ def run_test_read( ) slices, log_messages, auxiliary_requests, latest_config_update = self._categorise_groups( - message_group + message_group, deprecation_warnings ) schema, log_messages = self._get_infered_schema( configured_catalog, schema_inferrer, log_messages @@ -238,7 +243,11 @@ def _check_record_limit(self, record_limit: Optional[int] = None) -> int: return record_limit - def _categorise_groups(self, message_groups: MESSAGE_GROUPS) -> GROUPED_MESSAGES: + def _categorise_groups( + self, + message_groups: MESSAGE_GROUPS, + deprecation_warnings: Optional[List[Any]] = None, + ) -> GROUPED_MESSAGES: """ Categorizes a sequence of message groups into slices, log messages, auxiliary requests, and the latest configuration update. @@ -269,6 +278,7 @@ def _categorise_groups(self, message_groups: MESSAGE_GROUPS) -> GROUPED_MESSAGES auxiliary_requests = [] latest_config_update: Optional[AirbyteControlMessage] = None + # process the message groups first for message_group in message_groups: match message_group: case AirbyteLogMessage(): @@ -298,6 +308,17 @@ def _categorise_groups(self, message_groups: MESSAGE_GROUPS) -> GROUPED_MESSAGES case _: raise ValueError(f"Unknown message group type: {type(message_group)}") + # process deprecation warnings, if present + if deprecation_warnings is not None: + for deprecation in deprecation_warnings: + match deprecation: + case AirbyteLogMessage(): + log_messages.append( + LogMessage(message=deprecation.message, level=deprecation.level.value) + ) + case _: + raise ValueError(f"Unknown message group type: {type(deprecation)}") + return slices, log_messages, auxiliary_requests, latest_config_update def _get_infered_schema( diff --git a/airbyte_cdk/sources/declarative/declarative_source.py b/airbyte_cdk/sources/declarative/declarative_source.py index 77bf427a1..55f425e50 100644 --- a/airbyte_cdk/sources/declarative/declarative_source.py +++ b/airbyte_cdk/sources/declarative/declarative_source.py @@ -4,8 +4,11 @@ import logging from abc import abstractmethod -from typing import Any, Mapping, Tuple +from typing import Any, List, Mapping, Tuple +from airbyte_cdk.models import ( + AirbyteLogMessage, +) from airbyte_cdk.sources.abstract_source import AbstractSource from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker @@ -34,3 +37,9 @@ def check_connection( The error object will be cast to string to display the problem to the user. """ return self.connection_checker.check_connection(self, logger, config) + + def deprecation_warnings(self) -> List[AirbyteLogMessage]: + """ + Returns a list of deprecation warnings for the source. + """ + return [] diff --git a/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte_cdk/sources/declarative/manifest_declarative_source.py index cfd258c6c..6041b4362 100644 --- a/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -17,6 +17,7 @@ from airbyte_cdk.models import ( AirbyteConnectionStatus, + AirbyteLogMessage, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, @@ -123,6 +124,9 @@ def dynamic_streams(self) -> List[Dict[str, Any]]: manifest=self._source_config, config=self._config, with_dynamic_stream_name=True ) + def deprecation_warnings(self) -> List[AirbyteLogMessage]: + return self._constructor.get_model_deprecations() or [] + @property def connection_checker(self) -> ConnectionChecker: check = self._source_config["check"] diff --git a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py new file mode 100644 index 000000000..a0fde745b --- /dev/null +++ b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py @@ -0,0 +1,113 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. + +# THIS IS A STATIC CLASS MODEL USED TO DISPLAY DEPRECATION WARNINGS +# WHEN DEPRECATED FIELDS ARE ACCESSED + +import warnings +from typing import Any, List + +from pydantic.v1 import BaseModel + +from airbyte_cdk.models import ( + AirbyteLogMessage, + Level, +) + +# format the warning message +warnings.formatwarning = ( + lambda message, category, *args, **kwargs: f"{category.__name__}: {message}" +) + +FIELDS_TAG = "__fields__" +DEPRECATED = "deprecated" +DEPRECATION_MESSAGE = "deprecation_message" +DEPRECATION_LOGS_TAG = "_deprecation_logs" + + +class BaseModelWithDeprecations(BaseModel): + """ + Pydantic BaseModel that warns when deprecated fields are accessed. + The deprecation message is stored in the field's extra attributes. + This class is used to create models that can have deprecated fields + and show warnings when those fields are accessed or initialized. + + The `_deprecation_logs` attribute is storred in the model itself. + The collected deprecation warnings are further proparated to the Airbyte log messages, + during the component creation process, in `model_to_component._collect_model_deprecations()`. + + The component implementation is not responsible for handling the deprecation warnings, + since the deprecation warnings are already handled in the model itself. + """ + + class Config: + """ + Allow extra fields in the model. In case the model restricts extra fields. + """ + + extra = "allow" + + _deprecation_logs: List[AirbyteLogMessage] = [] + + def __init__(self, **data: Any) -> None: + """ + Show warnings for deprecated fields during component initialization. + """ + model_fields = self.__fields__ + + for field_name in data: + if field_name in model_fields: + is_deprecated_field = model_fields[field_name].field_info.extra.get( + DEPRECATED, False + ) + if is_deprecated_field: + deprecation_message = model_fields[field_name].field_info.extra.get( + DEPRECATION_MESSAGE, "" + ) + self._deprecated_warning(field_name, deprecation_message) + + # Call the parent constructor + super().__init__(**data) + + def __getattribute__(self, name: str) -> Any: + """ + Show warnings for deprecated fields during field usage. + """ + + value = super().__getattribute__(name) + + try: + model_fields = super().__getattribute__(FIELDS_TAG) + field_info = model_fields.get(name) + is_deprecated_field = ( + field_info.field_info.extra.get(DEPRECATED, False) if field_info else False + ) + if is_deprecated_field: + deprecation_message = field_info.extra.get(DEPRECATION_MESSAGE, "") + self._deprecated_warning(name, deprecation_message) + except (AttributeError, KeyError): + pass + + return value + + def _deprecated_warning(self, field_name: str, message: str) -> None: + """ + Show a warning message for deprecated fields (to stdout). + Args: + field_name (str): Name of the deprecated field. + message (str): Warning message to be displayed. + """ + + # Emit a warning message for deprecated fields (to stdout) (Python Default behavior) + warnings.warn( + f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}", + DeprecationWarning, + ) + + # Add the deprecation message to the Airbyte log messages, + # this logs are displayed in the Connector Builder. + self._deprecation_logs.append( + AirbyteLogMessage( + level=Level.WARN, + message=f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}", + ), + ) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 09b20b028..ce5b15e73 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -27,7 +27,7 @@ from isodate import parse_duration from pydantic.v1 import BaseModel -from airbyte_cdk.models import FailureType, Level +from airbyte_cdk.models import AirbyteLogMessage, FailureType, Level from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator from airbyte_cdk.sources.declarative.async_job.job_tracker import JobTracker @@ -108,6 +108,10 @@ CustomStateMigration, GzipDecoder, ) +from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( + DEPRECATION_LOGS_TAG, + BaseModelWithDeprecations, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( AddedFieldDefinition as AddedFieldDefinitionModel, ) @@ -584,6 +588,8 @@ def __init__( self._connector_state_manager = connector_state_manager or ConnectorStateManager() self._api_budget: Optional[Union[APIBudget, HttpAPIBudget]] = None self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) + # placeholder for deprecation warnings + self._deprecation_logs: List[AirbyteLogMessage] = [] def _init_mappings(self) -> None: self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { @@ -730,8 +736,26 @@ def _create_component_from_model(self, model: BaseModel, config: Config, **kwarg component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) if not component_constructor: raise ValueError(f"Could not find constructor for {model.__class__}") + + # collect deprecation warnings for supported models. + if isinstance(model, BaseModelWithDeprecations): + self._collect_model_deprecations(model) + return component_constructor(model=model, config=config, **kwargs) + def get_model_deprecations(self) -> List[Any]: + """ + Returns the deprecation warnings that were collected during the creation of components. + """ + return self._deprecation_logs + + def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: + if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: + for log in model._deprecation_logs: + # avoid duplicates for deprecation logs observed. + if log not in self._deprecation_logs: + self._deprecation_logs.append(log) + @staticmethod def create_added_field_definition( model: AddedFieldDefinitionModel, config: Config, **kwargs: Any diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 43f9b568e..51b3d8efb 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -1,5 +1,6 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import re import sys from glob import glob from pathlib import Path @@ -28,6 +29,63 @@ def generate_init_module_content() -> str: return header +def replace_base_model_for_classes_with_deprecated_fields(post_processed_content: str) -> str: + """ + Replace the base model for classes with deprecated fields. + This function looks for classes that inherit from `BaseModel` and have fields marked as deprecated. + It replaces the base model with `BaseModelWithDeprecations` for those classes. + """ + + # Find classes with deprecated fields + classes_with_deprecated_fields = set() + class_matches = re.finditer(r"class (\w+)\(BaseModel\):", post_processed_content) + + for class_match in class_matches: + class_name = class_match.group(1) + class_start = class_match.start() + # Find the next class definition or end of file + next_class_match = re.search( + r"class \w+\(", + post_processed_content[class_start + len(class_match.group(0)) :], + ) + class_end = ( + len(post_processed_content) + if next_class_match is None + else class_start + len(class_match.group(0)) + next_class_match.start() + ) + class_content = post_processed_content[class_start:class_end] + + # Check if any field has deprecated=True + if re.search(r"deprecated\s*=\s*True", class_content): + classes_with_deprecated_fields.add(class_name) + + # update the imports to include the new base model with deprecation warinings + # only if there are classes with the fields marked as deprecated. + if len(classes_with_deprecated_fields) > 0: + # Find where to insert the base model - after imports but before class definitions + imports_end = post_processed_content.find( + "\n\n", + post_processed_content.find("from pydantic.v1 import"), + ) + if imports_end > 0: + post_processed_content = ( + post_processed_content[:imports_end] + + "\n\n" + + "from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import (\n" + + " BaseModelWithDeprecations,\n" + + ")" + + post_processed_content[imports_end:] + ) + + # Use the `BaseModelWithDeprecations` base model for the classes with deprecated fields + for class_name in classes_with_deprecated_fields: + pattern = rf"class {class_name}\(BaseModel\):" + replacement = f"class {class_name}(BaseModelWithDeprecations):" + post_processed_content = re.sub(pattern, replacement, post_processed_content) + + return post_processed_content + + async def post_process_codegen(codegen_container: dagger.Container): codegen_container = codegen_container.with_exec( ["mkdir", "/generated_post_processed"], use_entrypoint=True @@ -41,6 +99,11 @@ async def post_process_codegen(codegen_container: dagger.Container): post_processed_content = original_content.replace( " _parameters:", " parameters:" ).replace("from pydantic", "from pydantic.v1") + + post_processed_content = replace_base_model_for_classes_with_deprecated_fields( + post_processed_content + ) + codegen_container = codegen_container.with_new_file( f"/generated_post_processed/{generated_file}", contents=post_processed_content ) @@ -75,6 +138,12 @@ async def main(): "--set-default-enum-member", "--use-double-quotes", "--remove-special-field-name-prefix", + # allow usage of the extra key such as `deprecated`, etc. + "--field-extra-keys", + # account the `deprecated` flag provided for the field. + "deprecated", + # account the `deprecation_message` provided for the field. + "deprecation_message", ], use_entrypoint=True, ) diff --git a/unit_tests/connector_builder/test_connector_builder_handler.py b/unit_tests/connector_builder/test_connector_builder_handler.py index d98a49a8c..4abff8f19 100644 --- a/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/unit_tests/connector_builder/test_connector_builder_handler.py @@ -7,7 +7,7 @@ import json import logging import os -from typing import Literal +from typing import List, Literal from unittest import mock from unittest.mock import MagicMock, patch @@ -818,6 +818,9 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: connector_specification.connectionSpecification = {} return connector_specification + def deprecation_warnings(self) -> List[AirbyteLogMessage]: + return [] + @property def check_config_against_spec(self) -> Literal[False]: return False From a3086744c34f8617c9ddcdb010e64dac6367b216 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 16 Apr 2025 15:31:32 +0300 Subject: [PATCH 2/8] updated --- .../declarative/manifest_declarative_source.py | 2 +- .../models/base_model_with_deprecations.py | 11 +++++------ .../parsers/model_to_component_factory.py | 16 ++++++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte_cdk/sources/declarative/manifest_declarative_source.py index 6041b4362..252e99d2f 100644 --- a/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -125,7 +125,7 @@ def dynamic_streams(self) -> List[Dict[str, Any]]: ) def deprecation_warnings(self) -> List[AirbyteLogMessage]: - return self._constructor.get_model_deprecations() or [] + return self._constructor.get_model_deprecations() @property def connection_checker(self) -> ConnectionChecker: diff --git a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py index a0fde745b..4083ce931 100644 --- a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py +++ b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py @@ -31,8 +31,8 @@ class BaseModelWithDeprecations(BaseModel): This class is used to create models that can have deprecated fields and show warnings when those fields are accessed or initialized. - The `_deprecation_logs` attribute is storred in the model itself. - The collected deprecation warnings are further proparated to the Airbyte log messages, + The `_deprecation_logs` attribute is stored in the model itself. + The collected deprecation warnings are further propagated to the Airbyte log messages, during the component creation process, in `model_to_component._collect_model_deprecations()`. The component implementation is not responsible for handling the deprecation warnings, @@ -46,14 +46,14 @@ class Config: extra = "allow" - _deprecation_logs: List[AirbyteLogMessage] = [] - def __init__(self, **data: Any) -> None: """ Show warnings for deprecated fields during component initialization. """ - model_fields = self.__fields__ + self._deprecation_logs: List[AirbyteLogMessage] = [] + + model_fields = self.__fields__ for field_name in data: if field_name in model_fields: is_deprecated_field = model_fields[field_name].field_info.extra.get( @@ -74,7 +74,6 @@ def __getattribute__(self, name: str) -> Any: """ value = super().__getattribute__(name) - try: model_fields = super().__getattribute__(FIELDS_TAG) field_info = model_fields.get(name) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index ce5b15e73..56bfc7085 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -589,7 +589,7 @@ def __init__( self._api_budget: Optional[Union[APIBudget, HttpAPIBudget]] = None self._job_tracker: JobTracker = JobTracker(max_concurrent_async_job_count or 1) # placeholder for deprecation warnings - self._deprecation_logs: List[AirbyteLogMessage] = [] + self._collected_deprecation_logs: List[AirbyteLogMessage] = [] def _init_mappings(self) -> None: self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { @@ -747,14 +747,22 @@ def get_model_deprecations(self) -> List[Any]: """ Returns the deprecation warnings that were collected during the creation of components. """ - return self._deprecation_logs + return self._collected_deprecation_logs def _collect_model_deprecations(self, model: BaseModelWithDeprecations) -> None: + """ + Collects deprecation logs from the given model and appends any new logs to the internal collection. + + This method checks if the provided model has deprecation logs (identified by the presence of the DEPRECATION_LOGS_TAG attribute and a non-None `_deprecation_logs` property). It iterates through each deprecation log in the model and appends it to the `_collected_deprecation_logs` list if it has not already been collected, ensuring that duplicate logs are avoided. + + Args: + model (BaseModelWithDeprecations): The model instance from which to collect deprecation logs. + """ if hasattr(model, DEPRECATION_LOGS_TAG) and model._deprecation_logs is not None: for log in model._deprecation_logs: # avoid duplicates for deprecation logs observed. - if log not in self._deprecation_logs: - self._deprecation_logs.append(log) + if log not in self._collected_deprecation_logs: + self._collected_deprecation_logs.append(log) @staticmethod def create_added_field_definition( From f63f195ba067ea187bc94c162b7bc29b34f4bbc4 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 16 Apr 2025 15:55:44 +0300 Subject: [PATCH 3/8] updated --- .../models/base_model_with_deprecations.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py index 4083ce931..3276c39f4 100644 --- a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py +++ b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py @@ -50,7 +50,7 @@ def __init__(self, **data: Any) -> None: """ Show warnings for deprecated fields during component initialization. """ - + # placeholder for deprecation logs self._deprecation_logs: List[AirbyteLogMessage] = [] model_fields = self.__fields__ @@ -81,7 +81,7 @@ def __getattribute__(self, name: str) -> Any: field_info.field_info.extra.get(DEPRECATED, False) if field_info else False ) if is_deprecated_field: - deprecation_message = field_info.extra.get(DEPRECATION_MESSAGE, "") + deprecation_message = field_info.field_info.extra.get(DEPRECATION_MESSAGE, "") self._deprecated_warning(name, deprecation_message) except (AttributeError, KeyError): pass @@ -96,17 +96,13 @@ def _deprecated_warning(self, field_name: str, message: str) -> None: message (str): Warning message to be displayed. """ + message = f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}" + # Emit a warning message for deprecated fields (to stdout) (Python Default behavior) - warnings.warn( - f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}", - DeprecationWarning, - ) + warnings.warn(message, DeprecationWarning) # Add the deprecation message to the Airbyte log messages, # this logs are displayed in the Connector Builder. self._deprecation_logs.append( - AirbyteLogMessage( - level=Level.WARN, - message=f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}", - ), + AirbyteLogMessage(level=Level.WARN, message=message), ) From 8dd12006ca48b06fc3a8c933e406f1b46db79375 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 16 Apr 2025 17:01:55 +0300 Subject: [PATCH 4/8] added HttpRequester deprecations --- .../declarative_component_schema.yaml | 31 +++++++- .../models/base_model_with_deprecations.py | 70 +++++++++------- .../models/declarative_component_schema.py | 46 +++++++---- .../parsers/model_to_component_factory.py | 29 +++++-- .../declarative/requesters/http_requester.py | 79 +++++++++++++++---- .../declarative/requesters/requester.py | 12 +++ 6 files changed, 195 insertions(+), 72 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 2d36004a3..8bfb0d4f4 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -1863,14 +1863,16 @@ definitions: type: object required: - type - - url_base properties: type: type: string enum: [HttpRequester] url_base: + deprecated: true + deprecation_message: "Use `url` field instead." + sharable: true title: API Base URL - description: Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. type: string interpolation_context: - config @@ -1886,9 +1888,30 @@ definitions: - "{{ config['base_url'] or 'https://app.posthog.com'}}/api" - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups" - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}" + url: + sharable: true + title: API URL + description: The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + type: string + interpolation_context: + - config + - next_page_token + - stream_interval + - stream_partition + - stream_slice + - creation_response + - polling_response + - download_target + examples: + - "https://connect.squareup.com/v2" + - "{{ config['url'] or 'https://app.posthog.com'}}/api" + - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups" + - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}" path: + deprecated: true + deprecation_message: "Use `url` field instead." title: URL Path - description: Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. type: string interpolation_context: - config @@ -4158,4 +4181,4 @@ interpolation: regex: The regular expression to search for. It must include a capture group. return_type: str examples: - - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"' + - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"' \ No newline at end of file diff --git a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py index 3276c39f4..9fbd290ac 100644 --- a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py +++ b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py @@ -46,15 +46,39 @@ class Config: extra = "allow" - def __init__(self, **data: Any) -> None: + def __init__(self, **model_data: Any) -> None: """ Show warnings for deprecated fields during component initialization. """ - # placeholder for deprecation logs + + # call the parent constructor first to initialize Pydantic internals + super().__init__(**model_data) + + # set the placeholder for the deprecation logs self._deprecation_logs: List[AirbyteLogMessage] = [] + # process deprecated fields, if present + self._process_fields(model_data) + + # set the deprecation logs attribute to the model + self._set_deprecation_logs_attr_to_model() + + def _process_fields(self, model_data: Any) -> None: + """ + Processes the fields in the provided model data, checking for deprecated fields. + + For each field in the input `model_data`, this method checks if the field exists in the model's defined fields. + If the field is marked as deprecated (using the `DEPRECATED` flag in its metadata), it triggers a deprecation warning + by calling the `_deprecated_warning` method with the field name and an optional deprecation message. + + Args: + model_data (Any): The data containing fields to be processed. + + Returns: + None + """ model_fields = self.__fields__ - for field_name in data: + for field_name in model_data.keys(): if field_name in model_fields: is_deprecated_field = model_fields[field_name].field_info.extra.get( DEPRECATED, False @@ -65,28 +89,18 @@ def __init__(self, **data: Any) -> None: ) self._deprecated_warning(field_name, deprecation_message) - # Call the parent constructor - super().__init__(**data) - - def __getattribute__(self, name: str) -> Any: - """ - Show warnings for deprecated fields during field usage. + def _set_deprecation_logs_attr_to_model(self) -> None: """ + Sets the deprecation logs attribute on the model instance. - value = super().__getattribute__(name) - try: - model_fields = super().__getattribute__(FIELDS_TAG) - field_info = model_fields.get(name) - is_deprecated_field = ( - field_info.field_info.extra.get(DEPRECATED, False) if field_info else False - ) - if is_deprecated_field: - deprecation_message = field_info.field_info.extra.get(DEPRECATION_MESSAGE, "") - self._deprecated_warning(name, deprecation_message) - except (AttributeError, KeyError): - pass - - return value + This method attaches the current instance's deprecation logs to the model by setting + an attribute named by `DEPRECATION_LOGS_TAG` to the value of `self._deprecation_logs`. + This is typically used to track or log deprecated features or configurations within the model. + + Returns: + None + """ + setattr(self, DEPRECATION_LOGS_TAG, self._deprecation_logs) def _deprecated_warning(self, field_name: str, message: str) -> None: """ @@ -97,12 +111,12 @@ def _deprecated_warning(self, field_name: str, message: str) -> None: """ message = f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}" - # Emit a warning message for deprecated fields (to stdout) (Python Default behavior) warnings.warn(message, DeprecationWarning) - + # Create an Airbyte deprecation log message + deprecation_log_message = AirbyteLogMessage(level=Level.WARN, message=message) # Add the deprecation message to the Airbyte log messages, # this logs are displayed in the Connector Builder. - self._deprecation_logs.append( - AirbyteLogMessage(level=Level.WARN, message=message), - ) + if deprecation_log_message not in self._deprecation_logs: + # Avoid duplicates in the deprecation logs + self._deprecation_logs.append(deprecation_log_message) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 7f56ab92d..38da5d13d 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1,5 +1,3 @@ -# Copyright (c) 2025 Airbyte, Inc., all rights reserved. - # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -10,6 +8,10 @@ from pydantic.v1 import BaseModel, Extra, Field +from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( + BaseModelWithDeprecations, +) + class AuthFlowType(Enum): oauth2_0 = "oauth2.0" @@ -880,20 +882,17 @@ class FlattenFields(BaseModel): class KeyTransformation(BaseModel): - prefix: Optional[Union[str, None]] = Field( + type: Literal["KeyTransformation"] + prefix: Optional[str] = Field( None, description="Prefix to add for object keys. If not provided original keys remain unchanged.", - examples=[ - "flattened_", - ], + examples=["flattened_"], title="Key Prefix", ) - suffix: Optional[Union[str, None]] = Field( + suffix: Optional[str] = Field( None, description="Suffix to add for object keys. If not provided original keys remain unchanged.", - examples=[ - "_flattened", - ], + examples=["_flattened"], title="Key Suffix", ) @@ -916,7 +915,7 @@ class DpathFlattenFields(BaseModel): description="Whether to replace the origin record or not. Default is False.", title="Replace Origin Record", ) - key_transformation: Optional[Union[KeyTransformation, None]] = Field( + key_transformation: Optional[KeyTransformation] = Field( None, description="Transformation for object keys. If not provided, original key will be used.", title="Key transformation", @@ -2171,11 +2170,13 @@ class SessionTokenAuthenticator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class HttpRequester(BaseModel): +class HttpRequester(BaseModelWithDeprecations): type: Literal["HttpRequester"] - url_base: str = Field( - ..., - description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + url_base: Optional[str] = Field( + None, + deprecated=True, + deprecation_message="Use `url` field instead.", + description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ "https://connect.squareup.com/v2", "{{ config['base_url'] or 'https://app.posthog.com'}}/api", @@ -2184,9 +2185,22 @@ class HttpRequester(BaseModel): ], title="API Base URL", ) + url: Optional[str] = Field( + None, + description="The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + examples=[ + "https://connect.squareup.com/v2", + "{{ config['url'] or 'https://app.posthog.com'}}/api", + "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", + "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", + ], + title="API URL", + ) path: Optional[str] = Field( None, - description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + deprecated=True, + deprecation_message="Use `url` field instead.", + description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ "/products", "/quotes/{{ stream_partition['id'] }}/quote_line_groups", diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 56bfc7085..ae3b4b150 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2194,7 +2194,7 @@ def create_http_requester( self._create_component_from_model( model=model.authenticator, config=config, - url_base=model.url_base, + url_base=model.url or model.url_base, name=name, decoder=decoder, ) @@ -2231,6 +2231,7 @@ def create_http_requester( return HttpRequester( name=name, + url=model.url, url_base=model.url_base, path=model.path, authenticator=authenticator, @@ -2928,6 +2929,25 @@ def create_simple_retriever( use_cache: Optional[bool] = None, **kwargs: Any, ) -> SimpleRetriever: + def _get_url() -> str: + """ + Closure to get the URL from the requester. This is used to get the URL in the case of a lazy retriever. + This is needed because the URL is not set until the requester is created. + """ + + _url = ( + model.requester.url + if hasattr(model.requester, "url") and model.requester.url is not None + else requester.get_url() + ) + _url_base = ( + model.requester.url_base + if hasattr(model.requester, "url_base") and model.requester.url_base is not None + else requester.get_url_base() + ) + + return _url or _url_base + decoder = ( self._create_component_from_model(model=model.decoder, config=config) if model.decoder @@ -2992,11 +3012,6 @@ def create_simple_retriever( use_cache=use_cache, config=config, ) - url_base = ( - model.requester.url_base - if hasattr(model.requester, "url_base") - else requester.get_url_base() - ) # Define cursor only if per partition or common incremental support is needed cursor = stream_slicer if isinstance(stream_slicer, DeclarativeCursor) else None @@ -3020,7 +3035,7 @@ def create_simple_retriever( self._create_component_from_model( model=model.paginator, config=config, - url_base=url_base, + url_base=_get_url(), extractor_model=model.record_selector.extractor, decoder=decoder, cursor_used_for_stop_condition=cursor_used_for_stop_condition, diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py index 78c07b725..6b0e65aab 100644 --- a/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -3,7 +3,6 @@ # import logging -import os from dataclasses import InitVar, dataclass, field from typing import Any, Callable, Mapping, MutableMapping, Optional, Union from urllib.parse import urljoin @@ -53,10 +52,11 @@ class HttpRequester(Requester): """ name: str - url_base: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] + url: Optional[Union[InterpolatedString, str]] = None + url_base: Optional[Union[InterpolatedString, str]] = None path: Optional[Union[InterpolatedString, str]] = None authenticator: Optional[DeclarativeAuthenticator] = None http_method: Union[str, HttpMethod] = HttpMethod.GET @@ -71,7 +71,14 @@ class HttpRequester(Requester): decoder: Decoder = field(default_factory=lambda: JsonDecoder(parameters={})) def __post_init__(self, parameters: Mapping[str, Any]) -> None: - self._url_base = InterpolatedString.create(self.url_base, parameters=parameters) + self._url = InterpolatedString.create( + self.url if self.url else EmptyString, parameters=parameters + ) + # deprecated + self._url_base = InterpolatedString.create( + self.url_base if self.url_base else EmptyString, parameters=parameters + ) + # deprecated self._path = InterpolatedString.create( self.path if self.path else EmptyString, parameters=parameters ) @@ -120,6 +127,51 @@ def exit_on_rate_limit(self, value: bool) -> None: def get_authenticator(self) -> DeclarativeAuthenticator: return self._authenticator + def get_url( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: + interpolation_context = get_interpolation_context( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + + return str(self._url.eval(self.config, **interpolation_context)) + + def _get_url( + self, + *, + path: Optional[str] = None, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: + url = self.get_url( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + + url_base = self.get_url_base( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + + path = path or self.get_path( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + + full_url = self._join_url(url_base, path) if url_base else url + path if path else url + + return full_url + def get_url_base( self, *, @@ -349,7 +401,7 @@ def _request_body_json( return options @classmethod - def _join_url(cls, url_base: str, path: str) -> str: + def _join_url(cls, url_base: str, path: Optional[str] = None) -> str: """ Joins a base URL with a given path and returns the resulting URL with any trailing slash removed. @@ -358,7 +410,7 @@ def _join_url(cls, url_base: str, path: str) -> str: Args: url_base (str): The base URL to which the path will be appended. - path (str): The path to join with the base URL. + path (Optional[str]): The path to join with the base URL. Returns: str: The resulting joined URL. @@ -399,18 +451,11 @@ def send_request( ) -> Optional[requests.Response]: request, response = self._http_client.send_request( http_method=self.get_method().value, - url=self._join_url( - self.get_url_base( - stream_state=stream_state, - stream_slice=stream_slice, - next_page_token=next_page_token, - ), - path - or self.get_path( - stream_state=stream_state, - stream_slice=stream_slice, - next_page_token=next_page_token, - ), + url=self._get_url( + path=path, + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, ), request_kwargs={"stream": self.stream_response}, headers=self._request_headers( diff --git a/airbyte_cdk/sources/declarative/requesters/requester.py b/airbyte_cdk/sources/declarative/requesters/requester.py index ddda1ddba..97b31e884 100644 --- a/airbyte_cdk/sources/declarative/requesters/requester.py +++ b/airbyte_cdk/sources/declarative/requesters/requester.py @@ -34,6 +34,18 @@ def get_authenticator(self) -> DeclarativeAuthenticator: """ pass + @abstractmethod + def get_url( + self, + *, + stream_state: Optional[StreamState], + stream_slice: Optional[StreamSlice], + next_page_token: Optional[Mapping[str, Any]], + ) -> str: + """ + :return: URL base for the API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "https://myapi.com/v1/" + """ + @abstractmethod def get_url_base( self, From 2448d377c87a6f824857a9ff956324a1abc03e95 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 16 Apr 2025 18:08:58 +0300 Subject: [PATCH 5/8] test change --- .../models/declarative_component_schema.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 38da5d13d..849c57182 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -882,17 +882,20 @@ class FlattenFields(BaseModel): class KeyTransformation(BaseModel): - type: Literal["KeyTransformation"] - prefix: Optional[str] = Field( + prefix: Optional[Union[str, None]] = Field( None, description="Prefix to add for object keys. If not provided original keys remain unchanged.", - examples=["flattened_"], + examples=[ + "flattened_", + ], title="Key Prefix", ) - suffix: Optional[str] = Field( + suffix: Optional[Union[str, None]] = Field( None, description="Suffix to add for object keys. If not provided original keys remain unchanged.", - examples=["_flattened"], + examples=[ + "_flattened", + ], title="Key Suffix", ) @@ -915,7 +918,7 @@ class DpathFlattenFields(BaseModel): description="Whether to replace the origin record or not. Default is False.", title="Replace Origin Record", ) - key_transformation: Optional[KeyTransformation] = Field( + key_transformation: Optional[Union[KeyTransformation, None]] = Field( None, description="Transformation for object keys. If not provided, original key will be used.", title="Key transformation", @@ -2637,4 +2640,4 @@ class DynamicDeclarativeStream(BaseModel): ParentStreamConfig.update_forward_refs() PropertiesFromEndpoint.update_forward_refs() SimpleRetriever.update_forward_refs() -AsyncRetriever.update_forward_refs() +AsyncRetriever.update_forward_refs() \ No newline at end of file From baf1058f18228450d55b2ad8aeb8c20b91d8ebf8 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 16 Apr 2025 18:16:18 +0300 Subject: [PATCH 6/8] updated models --- .../models/declarative_component_schema.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 849c57182..38da5d13d 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -882,20 +882,17 @@ class FlattenFields(BaseModel): class KeyTransformation(BaseModel): - prefix: Optional[Union[str, None]] = Field( + type: Literal["KeyTransformation"] + prefix: Optional[str] = Field( None, description="Prefix to add for object keys. If not provided original keys remain unchanged.", - examples=[ - "flattened_", - ], + examples=["flattened_"], title="Key Prefix", ) - suffix: Optional[Union[str, None]] = Field( + suffix: Optional[str] = Field( None, description="Suffix to add for object keys. If not provided original keys remain unchanged.", - examples=[ - "_flattened", - ], + examples=["_flattened"], title="Key Suffix", ) @@ -918,7 +915,7 @@ class DpathFlattenFields(BaseModel): description="Whether to replace the origin record or not. Default is False.", title="Replace Origin Record", ) - key_transformation: Optional[Union[KeyTransformation, None]] = Field( + key_transformation: Optional[KeyTransformation] = Field( None, description="Transformation for object keys. If not provided, original key will be used.", title="Key transformation", @@ -2640,4 +2637,4 @@ class DynamicDeclarativeStream(BaseModel): ParentStreamConfig.update_forward_refs() PropertiesFromEndpoint.update_forward_refs() SimpleRetriever.update_forward_refs() -AsyncRetriever.update_forward_refs() \ No newline at end of file +AsyncRetriever.update_forward_refs() From c94363e91294a5ac36e0491ce5410161367b0a96 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 17 Apr 2025 19:40:10 +0300 Subject: [PATCH 7/8] updated the BaseModelWithDeprecations class --- .../models/base_model_with_deprecations.py | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py index 9fbd290ac..f3e6db445 100644 --- a/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py +++ b/airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py @@ -46,48 +46,57 @@ class Config: extra = "allow" - def __init__(self, **model_data: Any) -> None: + def __init__(self, **model_fields: Any) -> None: """ Show warnings for deprecated fields during component initialization. """ - # call the parent constructor first to initialize Pydantic internals - super().__init__(**model_data) - + super().__init__(**model_fields) # set the placeholder for the deprecation logs self._deprecation_logs: List[AirbyteLogMessage] = [] - # process deprecated fields, if present - self._process_fields(model_data) - + self._process_fields(model_fields) # set the deprecation logs attribute to the model self._set_deprecation_logs_attr_to_model() - def _process_fields(self, model_data: Any) -> None: + def _is_deprecated_field(self, field_name: str) -> bool: + return ( + self.__fields__[field_name].field_info.extra.get(DEPRECATED, False) + if field_name in self.__fields__.keys() + else False + ) + + def _get_deprecation_message(self, field_name: str) -> str: + return ( + self.__fields__[field_name].field_info.extra.get( + DEPRECATION_MESSAGE, "" + ) + if field_name in self.__fields__.keys() + else "" + ) + + def _process_fields(self, model_fields: Any) -> None: """ Processes the fields in the provided model data, checking for deprecated fields. - For each field in the input `model_data`, this method checks if the field exists in the model's defined fields. + For each field in the input `model_fields`, this method checks if the field exists in the model's defined fields. If the field is marked as deprecated (using the `DEPRECATED` flag in its metadata), it triggers a deprecation warning - by calling the `_deprecated_warning` method with the field name and an optional deprecation message. + by calling the `_create_warning` method with the field name and an optional deprecation message. Args: - model_data (Any): The data containing fields to be processed. + model_fields (Any): The data containing fields to be processed. Returns: None """ - model_fields = self.__fields__ - for field_name in model_data.keys(): - if field_name in model_fields: - is_deprecated_field = model_fields[field_name].field_info.extra.get( - DEPRECATED, False - ) - if is_deprecated_field: - deprecation_message = model_fields[field_name].field_info.extra.get( - DEPRECATION_MESSAGE, "" + + if hasattr(self, FIELDS_TAG): + for field_name in model_fields.keys(): + if self._is_deprecated_field(field_name): + self._create_warning( + field_name, + self._get_deprecation_message(field_name), ) - self._deprecated_warning(field_name, deprecation_message) def _set_deprecation_logs_attr_to_model(self) -> None: """ @@ -102,7 +111,7 @@ def _set_deprecation_logs_attr_to_model(self) -> None: """ setattr(self, DEPRECATION_LOGS_TAG, self._deprecation_logs) - def _deprecated_warning(self, field_name: str, message: str) -> None: + def _create_warning(self, field_name: str, message: str) -> None: """ Show a warning message for deprecated fields (to stdout). Args: From 97c83d939cbb8431dc9bc1a25456eae32857d118 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 21 Apr 2025 23:12:55 +0300 Subject: [PATCH 8/8] fixed the leftovers from un-merging process (PoC) --- .../sources/declarative/declarative_component_schema.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 8bfb0d4f4..6c3a10aff 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -1870,7 +1870,6 @@ definitions: url_base: deprecated: true deprecation_message: "Use `url` field instead." - sharable: true title: API Base URL description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. type: string @@ -1889,8 +1888,7 @@ definitions: - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups" - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}" url: - sharable: true - title: API URL + title: The URL of an API endpoint description: The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. type: string interpolation_context: