Skip to content

Bug: Cloud API interfaces do not gracefully handle "too many requests" (Status 429) #797

@aaronsteers

Description

@aaronsteers

This is appearing in CI checks:

______________________ test_create_and_delete_destination ______________________

workspace_id = '19d7a891-8e0e-40ac-8a8c-5faf8d11e47c'
airbyte_cloud_api_root = 'https://api.airbyte.com/v1'
motherduck_api_key = <SecretString: ****>
airbyte_cloud_client_id = <SecretString: ****>
airbyte_cloud_client_secret = <SecretString: ****>

    def test_create_and_delete_destination(
        workspace_id: str,
        airbyte_cloud_api_root: str,
        motherduck_api_key: str,
        airbyte_cloud_client_id: SecretString,
        airbyte_cloud_client_secret: SecretString,
    ) -> None:
        new_resource_name = (
            "deleteme-destination-faker" + text_util.generate_random_suffix()
        )
        destination_config = DestinationDuckdb(
            destination_path="temp_db",
            motherduck_api_key=motherduck_api_key,
        )
    
        destination = api_util.create_destination(
            name=new_resource_name,
            api_root=airbyte_cloud_api_root,
            workspace_id=workspace_id,
            config=destination_config,
            client_id=airbyte_cloud_client_id,
            client_secret=airbyte_cloud_client_secret,
        )
        assert destination.name == new_resource_name
        assert destination.destination_type == "duckdb"
        assert destination.destination_id
    
>       api_util.delete_destination(
            destination_id=destination.destination_id,
            api_root=airbyte_cloud_api_root,
            workspace_id=workspace_id,
            client_id=airbyte_cloud_client_id,
            client_secret=airbyte_cloud_client_secret,
        )

tests/integration_tests/cloud/test_cloud_api_util.py:151: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
airbyte/_util/api_util.py:672: in delete_destination
    response = airbyte_instance.destinations.delete_destination(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <airbyte_api.destinations.Destinations object at 0x7f73e1309780>
request = DeleteDestinationRequest(destination_id='249bc314-b66d-484f-b766-5b5608854263')

    def delete_destination(self, request: api.DeleteDestinationRequest) -> api.DeleteDestinationResponse:
        r"""Delete a Destination"""
        hook_ctx = HookContext(operation_id='deleteDestination', oauth2_scopes=[], security_source=self.sdk_configuration.security)
        base_url = utils.template_url(*self.sdk_configuration.get_server_details())
    
        url = utils.generate_url(base_url, '/destinations/***destinationId***', request)
    
        if callable(self.sdk_configuration.security):
            headers, query_params = utils.get_security(self.sdk_configuration.security())
        else:
            headers, query_params = utils.get_security(self.sdk_configuration.security)
    
        headers['Accept'] = '*/*'
        headers['user-agent'] = self.sdk_configuration.user_agent
        client = self.sdk_configuration.client
    
        try:
            req = client.prepare_request(requests_http.Request('DELETE', url, params=query_params, headers=headers))
            req = self.sdk_configuration.get_hooks().before_request(BeforeRequestContext(hook_ctx), req)
            http_res = client.send(req)
        except Exception as e:
            _, e = self.sdk_configuration.get_hooks().after_error(AfterErrorContext(hook_ctx), None, e)
            if e is not None:
                raise e
    
        if utils.match_status_codes(['403','404','4XX','5XX'], http_res.status_code):
            result, e = self.sdk_configuration.get_hooks().after_error(AfterErrorContext(hook_ctx), http_res, None)
            if e is not None:
                raise e
            if result is not None:
                http_res = result
        else:
            http_res = self.sdk_configuration.get_hooks().after_success(AfterSuccessContext(hook_ctx), http_res)
    
    
    
        res = api.DeleteDestinationResponse(status_code=http_res.status_code, content_type=http_res.headers.get('Content-Type') or '', raw_response=http_res)
    
        if http_res.status_code == 204:
            pass
        elif http_res.status_code == 403 or http_res.status_code == 404 or http_res.status_code >= 400 and http_res.status_code < 500 or http_res.status_code >= 500 and http_res.status_code < 600:
>           raise errors.SDKError('API error occurred', http_res.status_code, http_res.text, http_res)
E           airbyte_api.errors.sdkerror.SDKError: API error occurred: Status 429
E           <!doctype html><meta charset="utf-8"><meta name=viewport content="width=device-width, initial-scale=1"><title>429</title>429 Too Many Requests

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions