From 937ec478326854e3c82986cb79b8866b58967574 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Mon, 9 Oct 2023 09:42:20 +0000 Subject: [PATCH 01/24] Update LinkedIn API version from 202302 to 202308 --- tap_linkedin_ads/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index 3248426..733b87f 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -11,7 +11,7 @@ BASE_URL = 'https://api.linkedin.com/rest' LINKEDIN_TOKEN_URI = 'https://www.linkedin.com/oauth/v2/accessToken' INTROSPECTION_URI = 'https://www.linkedin.com/oauth/v2/introspectToken' -LINKEDIN_VERSION = '202302' +LINKEDIN_VERSION = '202308' # set default timeout of 300 seconds REQUEST_TIMEOUT = 300 From 84ce4dcdbd5fcd0413abcfb66779cbe3870afafb Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Fri, 20 Oct 2023 10:03:25 +0000 Subject: [PATCH 02/24] Changes: 1) Changed LinkedIn API version to 202309. 2) Removed logic to add account parameter for streams with account filter search_account_param. --- tap_linkedin_ads/client.py | 2 +- tap_linkedin_ads/sync.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index 733b87f..a198311 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -11,7 +11,7 @@ BASE_URL = 'https://api.linkedin.com/rest' LINKEDIN_TOKEN_URI = 'https://www.linkedin.com/oauth/v2/accessToken' INTROSPECTION_URI = 'https://www.linkedin.com/oauth/v2/introspectToken' -LINKEDIN_VERSION = '202308' +LINKEDIN_VERSION = '202309' # set default timeout of 300 seconds REQUEST_TIMEOUT = 300 diff --git a/tap_linkedin_ads/sync.py b/tap_linkedin_ads/sync.py index 85a5b08..92dd280 100644 --- a/tap_linkedin_ads/sync.py +++ b/tap_linkedin_ads/sync.py @@ -109,9 +109,6 @@ def sync(client, config, catalog, state): for idx, account in enumerate(account_list): if account_filter == 'search_id_values_param': params['search.id.values[{}]'.format(idx)] = int(account) - elif account_filter == 'search_account_values_param': - params['search.account.values[{}]'.format(idx)] = \ - 'urn:li:sponsoredAccount:{}'.format(account) elif account_filter == 'accounts_param': params['accounts[{}]'.format(idx)] = \ 'urn:li:sponsoredAccount:{}'.format(account) @@ -128,6 +125,7 @@ def sync(client, config, catalog, state): total_records, max_bookmark_value = stream_obj.sync_endpoint( client=client, catalog=catalog, + config=config, state=state, page_size=page_size, start_date=start_date, From 661f1eeb87d4cf5e74ed3172301de39fa5614e25 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Fri, 20 Oct 2023 10:10:11 +0000 Subject: [PATCH 03/24] Changes: 1) Added logic to iterate over each account and use modified url for campaigns, campaigngroups & creatives APIs. 2) Added X-Restli-Protocol-Version header & modified params fir campaigns & campaigngroups APIs. 3 ) Removed account_filter for campaigns & campaigngroups APIs. 4) Added a new parameter config for sync_endpoint method. --- tap_linkedin_ads/streams.py | 267 +++++++++++++++++++----------------- 1 file changed, 141 insertions(+), 126 deletions(-) diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index e13ee36..b15b530 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -20,6 +20,7 @@ 'creative', 'creativeId', } +NEW_PATH_STREAMS = ["campaign_groups", "campaigns", "creatives"] def write_bookmark(state, value, stream_name): """ @@ -150,7 +151,7 @@ class LinkedInAds: key_properties : Primary keys for a given stream path : API endpoint relative path, when added to the base URL, creates the full path account_filter : Method for Account filtering. Each uses a different query pattern/parameter: - search_id_values_param, search_account_values_param, accounts_param + search_id_values_param, accounts_param params : Query, sort, and other endpoint specific parameters data_key : JSON element containing the records for the endpoint bookmark_query_field : Typically a date-time field is used for filtering the query @@ -261,6 +262,7 @@ def process_records(self, def sync_endpoint(self, client, catalog, + config, state, page_size, start_date, @@ -308,126 +310,139 @@ def sync_endpoint(self, } querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in endpoint_params.items()]) - next_url = 'https://api.linkedin.com/rest/{}?{}'.format(self.path, querystring) - - while next_url: #pylint: disable=too-many-nested-blocks - LOGGER.info('URL for %s: %s', self.tap_stream_id, next_url) - - # Get data, API request - data = client.get( - url=next_url, - endpoint=self.tap_stream_id, - headers=self.headers) - # time_extracted: datetime when the data was extracted from the API - time_extracted = utils.now() - - # Transform data with transform_json from transform.py - # This function converts unix datetimes, de-nests audit fields, - # tranforms URNs to IDs, tranforms/abstracts variably named fields, - # converts camelCase to snake_case for fieldname keys. - # For the Linkedin Ads API, 'elements' is always the root data_key for records. - # The data_key identifies the collection of records below the element - transformed_data = [] # initialize the record list - if self.data_key in data: - transformed_data = transform_json(data, self.tap_stream_id)[self.data_key] - if not transformed_data or transformed_data is None: - LOGGER.info('No transformed_data') - break # No data results - - pre_singer_transformed_data = copy.deepcopy(transformed_data) - if self.tap_stream_id in selected_streams: - # Process records and gets the max_bookmark_value and record_count for the set of records - max_bookmark_value, record_count = self.process_records( - catalog=catalog, - records=transformed_data, - time_extracted=time_extracted, - bookmark_field=bookmark_field, - max_bookmark_value=max_bookmark_value, - last_datetime=last_datetime, - parent_id=parent_id) - LOGGER.info('%s, records processed: %s', self.tap_stream_id, record_count) - total_records = total_records + record_count - - # Loop thru parent batch records for each children objects - for child_stream_name in children: - if child_stream_name in selected_streams: - # For each parent record - child_obj = STREAMS[child_stream_name]() - - for record in pre_singer_transformed_data: - - parent_id = record.get(child_obj.foreign_key) - - child_stream_params = child_obj.params - # Add children filter params based on parent IDs - if self.tap_stream_id == 'accounts': - account = 'urn:li:sponsoredAccount:{}'.format(parent_id) - owner_id = record.get('reference_organization_id', None) - owner = 'urn:li:organization:{}'.format(owner_id) - if child_stream_name == 'video_ads' and owner_id is not None: - child_stream_params['account'] = account - child_stream_params['owner'] = owner + # next_url = 'https://api.linkedin.com/rest/{}?{}'.format(self.path, querystring) + url_list = [] + if self.tap_stream_id in NEW_PATH_STREAMS: + querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in endpoint_params.items()]) + account_list = config['accounts'].replace(" ", "").split(",") + for account in account_list: + url = 'https://api.linkedin.com/rest/adAccounts/{}/{}?{}'.format(account, self.path, querystring) + url_list.append(url) + else: + url = 'https://api.linkedin.com/rest/{}?{}'.format(self.path, querystring) + url_list.append(url) + + + for next_url in url_list: + while next_url: #pylint: disable=too-many-nested-blocks + LOGGER.info('URL for %s: %s', self.tap_stream_id, next_url) + + # Get data, API request + data = client.get( + url=next_url, + endpoint=self.tap_stream_id, + headers=self.headers) + # time_extracted: datetime when the data was extracted from the API + time_extracted = utils.now() + + # Transform data with transform_json from transform.py + # This function converts unix datetimes, de-nests audit fields, + # tranforms URNs to IDs, tranforms/abstracts variably named fields, + # converts camelCase to snake_case for fieldname keys. + # For the Linkedin Ads API, 'elements' is always the root data_key for records. + # The data_key identifies the collection of records below the element + transformed_data = [] # initialize the record list + if self.data_key in data: + transformed_data = transform_json(data, self.tap_stream_id)[self.data_key] + if not transformed_data or transformed_data is None: + LOGGER.info('No transformed_data') + break # No data results + + pre_singer_transformed_data = copy.deepcopy(transformed_data) + if self.tap_stream_id in selected_streams: + # Process records and gets the max_bookmark_value and record_count for the set of records + max_bookmark_value, record_count = self.process_records( + catalog=catalog, + records=transformed_data, + time_extracted=time_extracted, + bookmark_field=bookmark_field, + max_bookmark_value=max_bookmark_value, + last_datetime=last_datetime, + parent_id=parent_id) + LOGGER.info('%s, records processed: %s', self.tap_stream_id, record_count) + total_records = total_records + record_count + + # Loop thru parent batch records for each children objects + for child_stream_name in children: + if child_stream_name in selected_streams: + # For each parent record + child_obj = STREAMS[child_stream_name]() + + for record in pre_singer_transformed_data: + + parent_id = record.get(child_obj.foreign_key) + + child_stream_params = child_obj.params + # Add children filter params based on parent IDs + if self.tap_stream_id == 'accounts': + account = 'urn:li:sponsoredAccount:{}'.format(parent_id) + owner_id = record.get('reference_organization_id', None) + owner = 'urn:li:organization:{}'.format(owner_id) + if child_stream_name == 'video_ads' and owner_id is not None: + child_stream_params['account'] = account + child_stream_params['owner'] = owner + else: + LOGGER.warning("Skipping video_ads call for %s account as reference_organization_id is not found.", account) + continue + elif self.tap_stream_id == 'campaigns': + campaign = 'urn:li:sponsoredCampaign:{}'.format(parent_id) + if child_stream_name == 'creatives': + # The value of the campaigns in the query params should be passed in the encoded format. + # Ref - https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-creatives?view=li-lms-2023-01&tabs=http#sample-request-3 + child_stream_params['campaigns'] = 'List(urn%3Ali%3AsponsoredCampaign%3A{})'.format(parent_id) + elif child_stream_name in ('ad_analytics_by_campaign', 'ad_analytics_by_creative'): + child_stream_params['campaigns[0]'] = campaign + + # Update params for the child stream + child_obj.params = child_stream_params + LOGGER.info('Syncing: %s, parent_stream: %s, parent_id: %s', + child_stream_name, + self.tap_stream_id, + parent_id) + + # Call sync method for the child stream + if child_stream_name in {'ad_analytics_by_campaign', 'ad_analytics_by_creative'}: + child_total_records, child_batch_bookmark_value = child_obj.sync_ad_analytics( + client=client, + catalog=catalog, + last_datetime=child_obj.get_bookmark(state, start_date), + date_window_size=date_window_size, + parent_id=parent_id) else: - LOGGER.warning("Skipping video_ads call for %s account as reference_organization_id is not found.", account) - continue - elif self.tap_stream_id == 'campaigns': - campaign = 'urn:li:sponsoredCampaign:{}'.format(parent_id) - if child_stream_name == 'creatives': - # The value of the campaigns in the query params should be passed in the encoded format. - # Ref - https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-creatives?view=li-lms-2023-01&tabs=http#sample-request-3 - child_stream_params['campaigns'] = 'List(urn%3Ali%3AsponsoredCampaign%3A{})'.format(parent_id) - elif child_stream_name in ('ad_analytics_by_campaign', 'ad_analytics_by_creative'): - child_stream_params['campaigns[0]'] = campaign - - # Update params for the child stream - child_obj.params = child_stream_params - LOGGER.info('Syncing: %s, parent_stream: %s, parent_id: %s', - child_stream_name, - self.tap_stream_id, - parent_id) - - # Call sync method for the child stream - if child_stream_name in {'ad_analytics_by_campaign', 'ad_analytics_by_creative'}: - child_total_records, child_batch_bookmark_value = child_obj.sync_ad_analytics( - client=client, - catalog=catalog, - last_datetime=child_obj.get_bookmark(state, start_date), - date_window_size=date_window_size, - parent_id=parent_id) - else: - child_total_records, child_batch_bookmark_value = child_obj.sync_endpoint( - client=client, - catalog=catalog, - state=state, - page_size=page_size, - start_date=start_date, - selected_streams=selected_streams, - date_window_size=date_window_size, - parent_id=parent_id) - - child_batch_bookmark_dttm = strptime_to_utc(child_batch_bookmark_value) - child_max_bookmark = child_max_bookmarks.get(child_stream_name) - child_max_bookmark_dttm = strptime_to_utc(child_max_bookmark) - if child_batch_bookmark_dttm > child_max_bookmark_dttm: - # Update bookmark for child stream. - child_max_bookmarks[child_stream_name] = strftime(child_batch_bookmark_dttm) - - LOGGER.info('Synced: %s, parent_id: %s, total_records: %s', - child_stream_name, - parent_id, - child_total_records) - LOGGER.info('FINISHED Syncing: %s', child_stream_name) - - # Pagination: Get next_url - next_url = get_next_url(data) - - if self.tap_stream_id in selected_streams: - LOGGER.info('%s: Synced page %s, this page: %s. Total records processed: %s', - self.tap_stream_id, - page, - record_count, - total_records) - page = page + 1 + child_total_records, child_batch_bookmark_value = child_obj.sync_endpoint( + client=client, + catalog=catalog, + config=config, + state=state, + page_size=page_size, + start_date=start_date, + selected_streams=selected_streams, + date_window_size=date_window_size, + parent_id=parent_id) + + child_batch_bookmark_dttm = strptime_to_utc(child_batch_bookmark_value) + child_max_bookmark = child_max_bookmarks.get(child_stream_name) + child_max_bookmark_dttm = strptime_to_utc(child_max_bookmark) + if child_batch_bookmark_dttm > child_max_bookmark_dttm: + # Update bookmark for child stream. + child_max_bookmarks[child_stream_name] = strftime(child_batch_bookmark_dttm) + + LOGGER.info('Synced: %s, parent_id: %s, total_records: %s', + child_stream_name, + parent_id, + child_total_records) + LOGGER.info('FINISHED Syncing: %s', child_stream_name) + + # Pagination: Get next_url + next_url = get_next_url(data) + + if self.tap_stream_id in selected_streams: + LOGGER.info('%s: Synced page %s, this page: %s. Total records processed: %s', + self.tap_stream_id, + page, + record_count, + total_records) + page = page + 1 # Write child stream's bookmarks for key, val in list(child_max_bookmarks.items()): @@ -605,14 +620,14 @@ class CampaignGroups(LinkedInAds): replication_method = "INCREMENTAL" replication_keys = ["last_modified_time"] key_properties = ["id"] - account_filter = "search_account_values_param" path = "adCampaignGroups" data_key = "elements" params = { "q": "search", - "sort.field": "ID", - "sort.order": "ASCENDING" + "sort": "(field:ID,order:ASCENDING)", + "search": "(status:(values:List(DRAFT,ACTIVE,PAUSED,ARCHIVED,CANCELED,PENDING_DELETION,REMOVED)))" } + headers = {'X-Restli-Protocol-Version': "2.0.0"} class Campaigns(LinkedInAds): """ @@ -622,15 +637,15 @@ class Campaigns(LinkedInAds): replication_method = "INCREMENTAL" replication_keys = ["last_modified_time"] key_properties = ["id"] - account_filter = "search_account_values_param" path = "adCampaigns" data_key = "elements" children = ["ad_analytics_by_campaign", "creatives", "ad_analytics_by_creative"] params = { "q": "search", - "sort.field": "ID", - "sort.order": "ASCENDING" + "sort": "(field:ID,order:ASCENDING)", + "search": "(status:(values:List(DRAFT,ACTIVE,PAUSED,ARCHIVED,CANCELED,PENDING_DELETION,REMOVED,COMPLETED)))" } + headers = {'X-Restli-Protocol-Version': "2.0.0"} class Creatives(LinkedInAds): """ From aa1ec85106d716c78e0d5e0f2a1a8dbadd0c9633 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Wed, 25 Oct 2023 13:08:17 +0000 Subject: [PATCH 04/24] Added relevant search query fields for analytics streams. --- tap_linkedin_ads/streams.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index b15b530..58244c4 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -19,6 +19,8 @@ 'endAt', 'creative', 'creativeId', + 'pivot', + 'pivotValue' } NEW_PATH_STREAMS = ["campaign_groups", "campaigns", "creatives"] @@ -495,7 +497,7 @@ def sync_ad_analytics(self, client, catalog, last_datetime, date_window_size, pa # (even if this means the values are all `0`) and a day with null # values. We found that requesting these fields gives you the days with # non-null values - first_chunk = [['dateRange', 'pivot', 'pivotValue']] + first_chunk = [['dateRange', 'pivotValues']] chunks = first_chunk + list(split_into_chunks(valid_selected_fields, MAX_CHUNK_LENGTH)) @@ -503,7 +505,7 @@ def sync_ad_analytics(self, client, catalog, last_datetime, date_window_size, pa # so that we can create the composite primary key for the record and # to merge the multiple responses based on this primary key for chunk in chunks: - for field in ['dateRange', 'pivot', 'pivotValue']: + for field in ['dateRange', 'pivotValues']: if field not in chunk: chunk.append(field) From 4c0c8bdd1b5003ed437558896b4f9d3d63011ddd Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Wed, 1 Nov 2023 11:33:41 +0000 Subject: [PATCH 05/24] Changes: 1) Added new schema for video_ads stream. 2) Modified enpoint and query params for video_ads stream. 3) Removed transform logic to flatten video_ads schema for change_audit_stamps fields. --- tap_linkedin_ads/schemas/video_ads.json | 90 +++++++------------------ tap_linkedin_ads/streams.py | 25 +++---- tap_linkedin_ads/transform.py | 14 ---- 3 files changed, 36 insertions(+), 93 deletions(-) diff --git a/tap_linkedin_ads/schemas/video_ads.json b/tap_linkedin_ads/schemas/video_ads.json index adb192c..de8ca8b 100644 --- a/tap_linkedin_ads/schemas/video_ads.json +++ b/tap_linkedin_ads/schemas/video_ads.json @@ -5,110 +5,66 @@ ], "additionalProperties": false, "properties": { - "account": { - "type": [ - "null", - "string" - ] - }, "account_id": { "type": [ "null", "integer" ] }, - "change_audit_stamps": { + "ad_context": { "type": [ "null", "object" ], "additionalProperties": false, "properties": { - "created": { + "dsc_status": { + "type": [ + "null", + "string" + ] + }, + "dsc_name": { + "type": [ + "null", + "string" + ] + }, + "dsc_ad_type": { "type": [ "null", - "object" - ], - "additionalProperties": false, - "properties": { - "time": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } + "string" + ] }, - "last_modified": { + "dsc_ad_account": { "type": [ "null", - "object" - ], - "additionalProperties": false, - "properties": { - "time": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } + "string" + ] } } }, - "created_time": { + "created_at": { "type": [ "null", "string" ], "format": "date-time" }, - "last_modified_time": { + "last_modified_at": { "type": [ "null", "string" ], "format": "date-time" }, - "content_reference": { - "type": [ - "null", - "string" - ] - }, - "content_reference_ucg_post_id": { - "type": [ - "null", - "integer" - ] - }, - "content_reference_share_id": { - "type": [ - "null", - "integer" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "owner": { + "id": { "type": [ "null", "string" ] }, - "owner_organization_id": { - "type": [ - "null", - "integer" - ] - }, - "type": { + "author": { "type": [ "null", "string" diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index 58244c4..ddb819c 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -7,6 +7,7 @@ from singer import Transformer, should_sync_field, UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING from singer.utils import strptime_to_utc, strftime from tap_linkedin_ads.transform import transform_json, snake_case_to_camel_case +from tap_linkedin_ads.client import BASE_URL LOGGER = singer.get_logger() @@ -312,16 +313,18 @@ def sync_endpoint(self, } querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in endpoint_params.items()]) - # next_url = 'https://api.linkedin.com/rest/{}?{}'.format(self.path, querystring) url_list = [] if self.tap_stream_id in NEW_PATH_STREAMS: querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in endpoint_params.items()]) account_list = config['accounts'].replace(" ", "").split(",") for account in account_list: - url = 'https://api.linkedin.com/rest/adAccounts/{}/{}?{}'.format(account, self.path, querystring) + url = '{}/adAccounts/{}/{}?{}'.format(BASE_URL, account, self.path, querystring) url_list.append(url) else: - url = 'https://api.linkedin.com/rest/{}?{}'.format(self.path, querystring) + if self.path == 'posts': + url = '{}/{}?{}&dscAdAccount=urn%3Ali%3AsponsoredAccount%3A{}'.format(BASE_URL, self.path, querystring, parent_id) + else: + url = '{}/{}?{}'.format(BASE_URL, self.path, querystring) url_list.append(url) @@ -379,11 +382,7 @@ def sync_endpoint(self, if self.tap_stream_id == 'accounts': account = 'urn:li:sponsoredAccount:{}'.format(parent_id) owner_id = record.get('reference_organization_id', None) - owner = 'urn:li:organization:{}'.format(owner_id) - if child_stream_name == 'video_ads' and owner_id is not None: - child_stream_params['account'] = account - child_stream_params['owner'] = owner - else: + if child_stream_name != 'video_ads' or owner_id is None: LOGGER.warning("Skipping video_ads call for %s account as reference_organization_id is not found.", account) continue elif self.tap_stream_id == 'campaigns': @@ -588,16 +587,18 @@ class VideoAds(LinkedInAds): https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/advertising-targeting/create-and-manage-video#finders """ tap_stream_id = "video_ads" - replication_keys = ["last_modified_time"] + replication_keys = ["last_modified_at"] replication_method = "INCREMENTAL" - key_properties = ["content_reference"] + key_properties = ["id"] foreign_key = "id" - path = "adDirectSponsoredContents" + path = "posts" data_key = "elements" parent = "accounts" params = { - "q": "account" + "q": "dscAdAccount", + "dscAdTypes": "List(VIDEO)" } + headers = {'X-Restli-Protocol-Version': "2.0.0"} class AccountUsers(LinkedInAds): """ diff --git a/tap_linkedin_ads/transform.py b/tap_linkedin_ads/transform.py index 02c4e20..b8db09e 100644 --- a/tap_linkedin_ads/transform.py +++ b/tap_linkedin_ads/transform.py @@ -255,19 +255,6 @@ def transform_creatives(data_dict): return new_dict -# Copy audit fields to root level -def transform_audit_fields(data_dict): - if 'change_audit_stamps' in data_dict: - if 'last_modified' in data_dict['change_audit_stamps']: - if 'time' in data_dict['change_audit_stamps']['last_modified']: - data_dict['last_modified_time'] = data_dict['change_audit_stamps']\ - ['last_modified']['time'] - if 'created' in data_dict['change_audit_stamps']: - if 'time' in data_dict['change_audit_stamps']['created']: - data_dict['created_time'] = data_dict['change_audit_stamps']['created']['time'] - return data_dict - - # Create ID field for each URN def transform_urn(data_dict): data_dict_copy = data_dict.copy() @@ -309,7 +296,6 @@ def transform_data(data_dict, stream_name): elif stream_name == 'creatives': this_dict = transform_creatives(this_dict) this_dict = transform_urn(this_dict) - this_dict = transform_audit_fields(this_dict) new_dict['elements'][i] = this_dict i = i + 1 From 393dfb38d03b74aebfd8faad0d0e423e280c57f7 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Thu, 2 Nov 2023 10:11:25 +0000 Subject: [PATCH 06/24] Changes: 1) Modified schema for video_ads stream. 2) Added transform logic to add ad_context fields to root level. --- tap_linkedin_ads/schemas/video_ads.json | 24 ++++++++++++++++++++++++ tap_linkedin_ads/transform.py | 13 +++++++++++++ 2 files changed, 37 insertions(+) diff --git a/tap_linkedin_ads/schemas/video_ads.json b/tap_linkedin_ads/schemas/video_ads.json index de8ca8b..158ba95 100644 --- a/tap_linkedin_ads/schemas/video_ads.json +++ b/tap_linkedin_ads/schemas/video_ads.json @@ -11,6 +11,30 @@ "integer" ] }, + "status": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "account": { + "type": [ + "null", + "string" + ] + }, "ad_context": { "type": [ "null", diff --git a/tap_linkedin_ads/transform.py b/tap_linkedin_ads/transform.py index b8db09e..2db7d3b 100644 --- a/tap_linkedin_ads/transform.py +++ b/tap_linkedin_ads/transform.py @@ -254,6 +254,18 @@ def transform_creatives(data_dict): return new_dict +# Copy ad context fields to root level +def transform_ad_context_fields(data_dict): + if 'ad_context' in data_dict: + if 'dsc_status' in data_dict['ad_context']: + data_dict['status'] = data_dict['ad_context']['dsc_status'] + if 'dsc_name' in data_dict['ad_context']: + data_dict['name'] = data_dict['ad_context']['dsc_name'] + if 'dsc_ad_type' in data_dict['ad_context']: + data_dict['type'] = data_dict['ad_context']['dsc_ad_type'] + if 'dsc_ad_account' in data_dict['ad_context']: + data_dict['account'] = data_dict['ad_context']['dsc_ad_account'] + return data_dict # Create ID field for each URN def transform_urn(data_dict): @@ -295,6 +307,7 @@ def transform_data(data_dict, stream_name): this_dict = transform_campaigns(this_dict) elif stream_name == 'creatives': this_dict = transform_creatives(this_dict) + this_dict = transform_ad_context_fields(this_dict) this_dict = transform_urn(this_dict) new_dict['elements'][i] = this_dict From 13e94900c13fe9bf4e93ad43c34b7a570fceaec8 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Mon, 6 Nov 2023 12:01:54 +0000 Subject: [PATCH 07/24] Modified logic to fetch pivot value from the list. --- tap_linkedin_ads/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index ddb819c..e4e9af4 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -133,7 +133,7 @@ def merge_responses(data): # Loop through each record of the page for element in page: temp_start = element['dateRange']['start'] - temp_pivotValue = element['pivotValue'] + temp_pivotValue = element['pivotValues'][0] string_start = '{}-{}-{}'.format(temp_start['year'], temp_start['month'], temp_start['day']) primary_key = (temp_pivotValue, string_start) if primary_key in full_records: From ea6e61d7193482776774e470a746349addbab5e2 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Wed, 8 Nov 2023 04:30:11 +0000 Subject: [PATCH 08/24] Fixed pylint issues. --- tap_linkedin_ads/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index e4e9af4..df12975 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -328,8 +328,8 @@ def sync_endpoint(self, url_list.append(url) - for next_url in url_list: - while next_url: #pylint: disable=too-many-nested-blocks + for next_url in url_list: #pylint: disable=too-many-nested-blocks + while next_url: LOGGER.info('URL for %s: %s', self.tap_stream_id, next_url) # Get data, API request From e19aa0e6800ed8113df031610f3d219bb628fc2a Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Wed, 8 Nov 2023 04:57:51 +0000 Subject: [PATCH 09/24] Fixed pylint for duplicate code. --- tap_linkedin_ads/sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tap_linkedin_ads/sync.py b/tap_linkedin_ads/sync.py index 92dd280..f219901 100644 --- a/tap_linkedin_ads/sync.py +++ b/tap_linkedin_ads/sync.py @@ -123,6 +123,7 @@ def sync(client, config, catalog, state): if stream_name in selected_streams: stream_obj.write_schema(catalog) + #pylint: disable=duplicate-code total_records, max_bookmark_value = stream_obj.sync_endpoint( client=client, catalog=catalog, config=config, From 57276b1c5231333b279976cf674436a69e62a937 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Wed, 8 Nov 2023 10:25:23 +0000 Subject: [PATCH 10/24] Fixed streams, sync & transform unit tests. --- tests/unittests/test_streams.py | 60 ++++++++++++++++--------------- tests/unittests/test_sync.py | 3 +- tests/unittests/test_transform.py | 47 ++++++++++++------------ 3 files changed, 59 insertions(+), 51 deletions(-) diff --git a/tests/unittests/test_streams.py b/tests/unittests/test_streams.py index 8a69193..e2ab5a8 100644 --- a/tests/unittests/test_streams.py +++ b/tests/unittests/test_streams.py @@ -190,32 +190,32 @@ def test_merge_responses_no_overlap(self): """ expected_output = { ('urn:li:sponsoredCampaign:123456789', '2020-10-1') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}}, - 'a': 1, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'a': 1, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-2') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}}, - 'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-3') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}}, - 'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-4') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}}, - 'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-5') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}}, - 'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-6') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}}, - 'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, } data = [ [{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}}, - 'a': 1, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'a': 1, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}}, - 'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}}, - 'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},], + 'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}], [{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}}, - 'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}}, - 'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}}, - 'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},], + 'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}], ] actual_output = merge_responses(data) @@ -228,34 +228,34 @@ def test_merge_responses_with_overlap(self): """ data = [ [{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}}, - 'a': 1, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'a': 1, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}}, - 'b': 7, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'b': 7, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}}, - 'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}}, - 'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},], + 'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}], [{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}}, - 'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}}, - 'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}}, - 'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},], + 'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}], ] expected_output = { ('urn:li:sponsoredCampaign:123456789', '2020-10-1') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}}, - 'a': 1, 'b': 7, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'a': 1, 'b': 7, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-2') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}}, - 'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-3') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}}, - 'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-4') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}}, - 'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-5') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}}, - 'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, ('urn:li:sponsoredCampaign:123456789', '2020-10-6') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}}, - 'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'}, + 'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}, } actual_output = merge_responses(data) @@ -327,11 +327,12 @@ def test_process_records(self, name, stream_obj, records, replication_key, expec ]) @mock.patch("tap_linkedin_ads.streams.LinkedInAds.sync_ad_analytics", return_value=(1, "2019-07-31T15:07:00.000000Z")) @mock.patch("tap_linkedin_ads.streams.LinkedInAds.get_bookmark", return_value = "2019-07-31T15:07:00.000000Z") + @mock.patch("tap_linkedin_ads.client.LinkedinClient.get") @mock.patch("tap_linkedin_ads.client.LinkedinClient.request") @mock.patch("tap_linkedin_ads.streams.LinkedInAds.process_records") @mock.patch("tap_linkedin_ads.streams.LinkedInAds.write_schema") def test_sync_endpoint(self, name, selected_streams, stream_obj, mock_response, expected_write_schema_count, mock_record_count, - mock_write_schema,mock_process_records,mock_client,mock_get_bookmark, mock_sync_ad_analytics): + mock_write_schema,mock_process_records,mock_client,mock_get,mock_get_bookmark, mock_sync_ad_analytics): """ Test sync_endpoint function for parent and child streams. """ @@ -341,9 +342,11 @@ def test_sync_endpoint(self, name, selected_streams, stream_obj, mock_response, page_size = 100 date_window_size = 7 + mock_get.side_effect = [{"elements": [{"1": "a"}]}] mock_client.side_effect = mock_response + config = {"accounts": "123"} mock_process_records.return_value = "2019-07-31T15:07:00.000000Z",1 - actual_total_record, actual_max_bookmark = stream_obj.sync_endpoint(client, CATALOG, state, page_size, start_date, selected_streams, date_window_size) + actual_total_record, actual_max_bookmark = stream_obj.sync_endpoint(client, CATALOG, config, state, page_size, start_date, selected_streams, date_window_size) # Verify total no of records self.assertEqual(actual_total_record, mock_record_count) @@ -368,10 +371,11 @@ def test_sync_endpoint_for_reference_organization_id_is_None(self, mock_write_sc page_size = 100 date_window_size = 7 selected_streams = ['accounts', 'video_ads'] + config = {"accounts": "123"} mock_client.side_effect = [{'paging': {'start': 0, 'count': 100, 'links': [], 'total': 1},'elements': [{'changeAuditStamps': {'created': {'time': 1564585620000}, 'lastModified': {'time': 1564585620000}}, 'id': 1}]}] mock_process_records.return_value = "2019-07-31T15:07:00.000000Z",1 - ACCOUNT_OBJ.sync_endpoint(client, CATALOG, state, page_size, start_date, selected_streams, date_window_size) + ACCOUNT_OBJ.sync_endpoint(client, CATALOG, config, state, page_size, start_date, selected_streams, date_window_size) mock_warning.assert_called_with('Skipping video_ads call for %s account as reference_organization_id is not found.', 'urn:li:sponsoredAccount:1') diff --git a/tests/unittests/test_sync.py b/tests/unittests/test_sync.py index 7cc0137..74f367f 100644 --- a/tests/unittests/test_sync.py +++ b/tests/unittests/test_sync.py @@ -157,7 +157,8 @@ def test_sync(self, name, config, expected_date_window, mock_sync_endpoint): sync(client, config, CATALOG, state) mock_sync_endpoint.assert_called_with(client=client, - catalog=CATALOG, + catalog=CATALOG, + config=config, state=state, page_size=100, start_date="2019-06-01T00:00:00Z", diff --git a/tests/unittests/test_transform.py b/tests/unittests/test_transform.py index 9729a95..26ae459 100644 --- a/tests/unittests/test_transform.py +++ b/tests/unittests/test_transform.py @@ -4,7 +4,7 @@ from parameterized import parameterized from tap_linkedin_ads.transform import (convert, snake_case_to_camel_case, convert_array, convert_json, transform_accounts, transform_analytics, transform_json, - transform_campaigns, transform_creatives, transform_audit_fields, + transform_campaigns, transform_creatives, transform_ad_context_fields, transform_urn, transform_data, string_to_decimal) @@ -337,34 +337,38 @@ def test_transform_creatives(self, test_dict_1, expected_dict): self.assertEqual(transformed_dict, expected_dict) -class TestTransformAuditFields(unittest.TestCase): +class TestTransformAdContextFields(unittest.TestCase): """ - Test `transform_audit_fields` function. + Test `transform_ad_context_fields` function. """ test_dict_1 = {"reference": "urn:li:organization:20111635"} test_dict_2 = { - "change_audit_stamps": { - "created": {"time": 1563562455000}, - "last_modified": {"time": 1626169039381} + "ad_context": { + "dsc_status": "ACTIVE", + "dsc_name": "Stitch Tableau", + "dsc_ad_type": "VIDEO", + "dsc_ad_account": "urn:li:sponsoredAccount:503498742" } } added_fields_2 = { - "created_time": 1563562455000, - "last_modified_time": 1626169039381, + "status": "ACTIVE", + "name": "Stitch Tableau", + "type": "VIDEO", + "account": "urn:li:sponsoredAccount:503498742" } @parameterized.expand([ (test_dict_1, {**test_dict_1}), (test_dict_2, {**test_dict_2, **added_fields_2}), ]) - def test_transform_audit_fields(self, test_dict, expected_dict): + def test_transform_ad_context_fields(self, test_dict, expected_dict): """ Test that time fields are added to first level. """ - transformed_dict = transform_audit_fields(test_dict) + transformed_dict = transform_ad_context_fields(test_dict) # Verify returned dict is expected self.assertEqual(transformed_dict, expected_dict) @@ -398,8 +402,7 @@ def test_transform_urn(self, test_dict, expected_dict): self.assertEqual(transformed_dict, expected_dict) -@mock.patch("tap_linkedin_ads.transform.transform_urn") -@mock.patch("tap_linkedin_ads.transform.transform_audit_fields") +@mock.patch("tap_linkedin_ads.transform.transform_ad_context_fields") class TestTransformData(unittest.TestCase): """ Test `transform_data` function that it calls other transform function respective to stream_name @@ -408,11 +411,11 @@ class TestTransformData(unittest.TestCase): test_dict = {"type": "BUSINESS", "id": 503491473} @mock.patch("tap_linkedin_ads.transform.transform_accounts") - def test_accounts_stream(self, mock_transform_accounts, mock_audit_fields, mock_transform_urn): + def test_accounts_stream(self, mock_transform_accounts, mock_ad_context_fields): """ Test for `accounts` stream `transform_accounts` is called. """ - mock_audit_fields.return_value = self.test_dict + mock_ad_context_fields.return_value = self.test_dict transformed_dict = transform_data({"elements": [self.test_dict]*3}, "accounts") # Verify transform function called for each element @@ -420,11 +423,11 @@ def test_accounts_stream(self, mock_transform_accounts, mock_audit_fields, mock_ self.assertEqual(transformed_dict, {"elements": [self.test_dict]*3}) @mock.patch("tap_linkedin_ads.transform.transform_campaigns") - def test_campaigns_stream(self, mock_transform_campaigns, mock_audit_fields, mock_transform_urn): + def test_campaigns_stream(self, mock_transform_campaigns, mock_ad_context_fields): """ Test for `campaigns` stream `transform_campaigns` is called. """ - mock_audit_fields.return_value = self.test_dict + mock_ad_context_fields.return_value = self.test_dict transformed_dict = transform_data({"elements": [self.test_dict]*4}, "campaigns") # Verify transform function called for each element @@ -432,11 +435,11 @@ def test_campaigns_stream(self, mock_transform_campaigns, mock_audit_fields, moc self.assertEqual(transformed_dict, {"elements": [self.test_dict]*4}) @mock.patch("tap_linkedin_ads.transform.transform_analytics") - def test_analytics_stream(self, mock_transform_analytics, mock_audit_fields, mock_transform_urn): + def test_analytics_stream(self, mock_transform_analytics, mock_ad_context_fields): """ Test for any analytics stream `transform_analytics` is called. """ - mock_audit_fields.return_value = self.test_dict + mock_ad_context_fields.return_value = self.test_dict transformed_dict = transform_data({"elements": [self.test_dict]*4}, "ad_analytics_by_creatives") # Verify transform function called for each element @@ -444,11 +447,11 @@ def test_analytics_stream(self, mock_transform_analytics, mock_audit_fields, moc self.assertEqual(transformed_dict, {"elements": [self.test_dict]*4}) @mock.patch("tap_linkedin_ads.transform.transform_creatives") - def test_creatives_stream(self, mock_transform_creatives, mock_audit_fields, mock_transform_urn): + def test_creatives_stream(self, mock_transform_creatives, mock_ad_context_fields): """ Test for `creatives` stream `transform_creatives` is called. """ - mock_audit_fields.return_value = self.test_dict + mock_ad_context_fields.return_value = self.test_dict transformed_dict = transform_data({"elements": [self.test_dict]*4}, "creatives") # Verify transform function called for each element @@ -459,11 +462,11 @@ def test_creatives_stream(self, mock_transform_creatives, mock_audit_fields, moc @mock.patch("tap_linkedin_ads.transform.transform_campaigns") @mock.patch("tap_linkedin_ads.transform.transform_accounts") @mock.patch("tap_linkedin_ads.transform.transform_creatives") - def test_other_streams(self, transform_creatives, transform_accounts, transform_campaigns, transform_analytics, mock_audit_fields, mock_transform_urn): + def test_other_streams(self, transform_creatives, transform_accounts, transform_campaigns, transform_analytics, mock_ad_context_fields): """ Test for any other streams transformed dictionary is returned. """ - mock_audit_fields.return_value = self.test_dict + mock_ad_context_fields.return_value = self.test_dict transformed_dict = transform_data({"elements": [self.test_dict]*4}, "video_ads") # Verify any transform function specific to a stream was not called From cf7081928953872e7f97ba9f3ae1c7313fe93bbd Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Wed, 8 Nov 2023 13:36:24 +0000 Subject: [PATCH 11/24] Added back the logic to tranform audit fields. --- tap_linkedin_ads/transform.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tap_linkedin_ads/transform.py b/tap_linkedin_ads/transform.py index 2db7d3b..c77321d 100644 --- a/tap_linkedin_ads/transform.py +++ b/tap_linkedin_ads/transform.py @@ -254,6 +254,18 @@ def transform_creatives(data_dict): return new_dict +# Copy audit fields to root level +def transform_audit_fields(data_dict): + if 'change_audit_stamps' in data_dict: + if 'last_modified' in data_dict['change_audit_stamps']: + if 'time' in data_dict['change_audit_stamps']['last_modified']: + data_dict['last_modified_time'] = data_dict['change_audit_stamps']\ + ['last_modified']['time'] + if 'created' in data_dict['change_audit_stamps']: + if 'time' in data_dict['change_audit_stamps']['created']: + data_dict['created_time'] = data_dict['change_audit_stamps']['created']['time'] + return data_dict + # Copy ad context fields to root level def transform_ad_context_fields(data_dict): if 'ad_context' in data_dict: @@ -309,6 +321,7 @@ def transform_data(data_dict, stream_name): this_dict = transform_creatives(this_dict) this_dict = transform_ad_context_fields(this_dict) this_dict = transform_urn(this_dict) + this_dict = transform_audit_fields(this_dict) new_dict['elements'][i] = this_dict i = i + 1 From f57f63d440f856ab0c9c0f1c312ae6d327f804a3 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 14 Nov 2023 13:52:32 +0000 Subject: [PATCH 12/24] Removed logic to delete Beta fields from Ad analytics streams. --- tap_linkedin_ads/schema.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tap_linkedin_ads/schema.py b/tap_linkedin_ads/schema.py index 8fba591..9fa4c20 100644 --- a/tap_linkedin_ads/schema.py +++ b/tap_linkedin_ads/schema.py @@ -6,14 +6,6 @@ # Reference: # https://github.com/singer-io/getting-started/blob/master/docs/DISCOVERY_MODE.md#Metadata -# The following fields of ads_analytics (...by_campaign and ...by_creative) were previously in beta and are not available on -# API version 202302. Requesting them results in a 403. -# https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-02&tabs=http#accuracy -FIELDS_UNACCEPTED_BY_API = { - "average_daily_reach_metrics", - "average_previous_seven_day_reach_metrics", - "average_previous_thirty_day_reach_metrics", -} def get_abs_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) @@ -27,10 +19,6 @@ def get_schemas(): with open(schema_path, encoding='utf-8') as file: schema = json.load(file) - if stream_name in ('ad_analytics_by_campaign', 'ad_analytics_by_creative'): - for field in FIELDS_UNACCEPTED_BY_API: - metadata.delete(schema, 'properties', field) - schemas[stream_name] = schema mdata = metadata.new() From 4397501b68ebc1b1393152aa529e1c2142e8f0a6 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Thu, 16 Nov 2023 13:06:33 +0000 Subject: [PATCH 13/24] Changelog, readme & setup.py changes. --- CHANGELOG.md | 9 +++++++++ README.md | 18 +++++++++--------- setup.py | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3677b76..19b7b34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 3.0.0 + * Bump to API version `202309` + * Removed `FIELDS_UNACCEPTED_BY_API` beta fields & updated integration tests. + * Updated schema, replication key & primary key for "video_ads" stream + * Updated path for streams "campaign_groups", "campaigns", "creatives" & "video_ads" + * Unit & integration tests updated for version bump + * [#64](https://github.com/singer-io/tap-linkedin-ads/pull/64) + * [#65](https://github.com/singer-io/tap-linkedin-ads/pull/65) + ## 2.1.0 * Bump to API version `202302` * Move and update `FIELDS_UNACCEPTED_BY_API` diff --git a/README.md b/README.md index f1fcd15..0358e60 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). This tap: -- Pulls raw data from the [LinkedIn Marketing Ads July 2022](https://docs.microsoft.com/en-us/linkedin/marketing/) +- Pulls raw data from the [LinkedIn Marketing Ads October 2023](https://docs.microsoft.com/en-us/linkedin/marketing/) - Extracts the following resources: - [Ad Accounts](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-accounts#search-for-accounts) - [Video Ads](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/advertising-targeting/create-and-manage-video#finders) @@ -31,14 +31,14 @@ This tap: - Transformations: Fields camelCase to snake_case. URNs to ids. Unix epoch millisecond integers to date-times. Audit date-times created_at and last_modified_at de-nested. String to decimal for total_budget field. - Children: video_ads -[**video_ads**](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/advertising-targeting/create-and-manage-video#finders) -- Endpoint: https://api.linkedin.com/rest/adDirectSponsoredContents -- Primary key field: content_reference +[**video_ads**](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/community-management/shares/posts-api?view=li-lms-2023-10&tabs=http#find-posts-by-account-1) +- Endpoint: https://api.linkedin.com/rest/posts +- Primary key field: id - Foreign keys: account_id (accounts), owner_organization_id (organizations) - Replication strategy: Incremental (query all, filter results) - Filter: account (from parent account) and owner (from parent account) (see NOTE below) - - Bookmark: last_modified_time (date-time) -- Transformations: Fields camelCase to snake_case. URNs to ids. Unix epoch millisecond integers to date-times. Audit date-times created_at and last_modified_at de-nested. + - Bookmark: last_modified_at (date-time) +- Transformations: Fields camelCase to snake_case. URNs to ids. Unix epoch millisecond integers to date-times. Ad context fields dsc_status, dsc_name, dsc_ad_type & dsc_ad_account de-nested. - Parent: account **NOTE**: The parent Account **MUST** reference and **Organization** (not a Person) - [Campaign Manager User Roles for Video Ads](https://www.linkedin.com/help/lms/answer/90733/campaign-manager-user-roles-for-video-ads?lang=en) @@ -53,7 +53,7 @@ This tap: - Transformations: Fields camelCase to snake_case. URNs to ids. Unix epoch millisecond integers to date-times. Audit date-times created_at and last_modified_at de-nested. [**campaign_groups**](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaign-groups#search-for-campaign-groups) -- Endpoint: https://api.linkedin.com/rest/adCampaignGroups +- Endpoint: https://api.linkedin.com/rest/adAccounts/{account-id}/adCampaignGroups - Primary key field: id - Foreign keys: account_id (accounts) - Replication strategy: Incremental (query all, filter results) @@ -63,7 +63,7 @@ This tap: - Transformations: Fields camelCase to snake_case. URNs to ids. Unix epoch millisecond integers to date-times. Audit date-times created_at and last_modified_at de-nested. [**campaigns**](https://docs.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaigns#search-for-campaigns) -- Endpoint: https://api.linkedin.com/rest/adCampaigns +- Endpoint: https://api.linkedin.com/rest/adAccounts/{account-id}/adCampaigns - Primary key field: id - Foreign keys: account_id (accounts) - Replication strategy: Incremental (query all, filter results) @@ -74,7 +74,7 @@ This tap: - Children: creatives, ad_analytics_by_campaign, ad_analytics_by_creative [**creatives**](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-creatives?view=li-lms-2023-01&tabs=http#search-for-creatives) -- Endpoint: https://api.linkedin.com/rest/creatives +- Endpoint: https://api.linkedin.com/rest/adAccounts/{account-id}/creatives - Primary key field: id - Foreign keys: campaign_id (campaigns) - Replication strategy: Incremental (query all, filter results) diff --git a/setup.py b/setup.py index fb56d58..78d736c 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup(name='tap-linkedin-ads', - version='2.1.0', + version='3.0.0', description='Singer.io tap for extracting data from the LinkedIn Marketing Ads API API 2.0', author='jeff.huth@bytecode.io', classifiers=['Programming Language :: Python :: 3 :: Only'], From 6bd4e7af224fb42f5fd5d54780e4538250682090 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Thu, 23 Nov 2023 09:48:26 +0000 Subject: [PATCH 14/24] Changes: 1) Added get_config() method in LinkedInClient class to fetch config json. 2) Updated usage of get_config() method in discover & sync for tap. --- tap_linkedin_ads/__init__.py | 8 +++----- tap_linkedin_ads/client.py | 3 +++ tap_linkedin_ads/sync.py | 3 ++- tap_linkedin_ads/transform.py | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tap_linkedin_ads/__init__.py b/tap_linkedin_ads/__init__.py index 40be526..8da87df 100644 --- a/tap_linkedin_ads/__init__.py +++ b/tap_linkedin_ads/__init__.py @@ -19,9 +19,9 @@ ] -def do_discover(client, config): +def do_discover(client): LOGGER.info('Starting discover') - client.check_accounts(config) + client.check_accounts(client.get_config()) catalog = _discover() json.dump(catalog.to_dict(), sys.stdout, indent=2) LOGGER.info('Finished discover') @@ -30,7 +30,6 @@ def do_discover(client, config): @singer.utils.handle_top_exception(LOGGER) def main(): parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) - config = parsed_args.config with LinkedinClient(parsed_args.config.get('client_id', None), parsed_args.config.get('client_secret', None), @@ -45,10 +44,9 @@ def main(): if parsed_args.state: state = parsed_args.state if parsed_args.discover: - do_discover(client, config) + do_discover(client) elif parsed_args.catalog: _sync(client=client, - config=config, catalog=parsed_args.catalog, state=state) diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index a198311..eff32a9 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -175,6 +175,9 @@ def set_mock_expires_for_test(self, mock_expire): self.__expires = mock_expire return self.__expires + def get_config(self): + with open(self.__config_path) as file: + return json.load(file) def write_access_token_to_config(self): """ diff --git a/tap_linkedin_ads/sync.py b/tap_linkedin_ads/sync.py index f219901..61a2000 100644 --- a/tap_linkedin_ads/sync.py +++ b/tap_linkedin_ads/sync.py @@ -66,10 +66,11 @@ def get_page_size(config): except Exception: raise Exception("The entered page size ({}) is invalid".format(page_size)) -def sync(client, config, catalog, state): +def sync(client, catalog, state): """ sync selected streams. """ + config = client.get_config() start_date = config['start_date'] page_size = get_page_size(config) diff --git a/tap_linkedin_ads/transform.py b/tap_linkedin_ads/transform.py index c77321d..dbe8c86 100644 --- a/tap_linkedin_ads/transform.py +++ b/tap_linkedin_ads/transform.py @@ -254,6 +254,7 @@ def transform_creatives(data_dict): return new_dict + # Copy audit fields to root level def transform_audit_fields(data_dict): if 'change_audit_stamps' in data_dict: From afb2d2f657a7d981bdbb28a1e6802975bb1e54cc Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Thu, 23 Nov 2023 09:56:33 +0000 Subject: [PATCH 15/24] Added usage for get_config() method in sync_endpoint() method. --- tap_linkedin_ads/streams.py | 3 +-- tap_linkedin_ads/sync.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index df12975..d998f70 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -265,7 +265,6 @@ def process_records(self, def sync_endpoint(self, client, catalog, - config, state, page_size, start_date, @@ -316,6 +315,7 @@ def sync_endpoint(self, url_list = [] if self.tap_stream_id in NEW_PATH_STREAMS: querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in endpoint_params.items()]) + config = client.get_config() account_list = config['accounts'].replace(" ", "").split(",") for account in account_list: url = '{}/adAccounts/{}/{}?{}'.format(BASE_URL, account, self.path, querystring) @@ -413,7 +413,6 @@ def sync_endpoint(self, child_total_records, child_batch_bookmark_value = child_obj.sync_endpoint( client=client, catalog=catalog, - config=config, state=state, page_size=page_size, start_date=start_date, diff --git a/tap_linkedin_ads/sync.py b/tap_linkedin_ads/sync.py index 61a2000..b5bf329 100644 --- a/tap_linkedin_ads/sync.py +++ b/tap_linkedin_ads/sync.py @@ -127,7 +127,6 @@ def sync(client, catalog, state): #pylint: disable=duplicate-code total_records, max_bookmark_value = stream_obj.sync_endpoint( client=client, catalog=catalog, - config=config, state=state, page_size=page_size, start_date=start_date, From 59c526666deae4cf767e97be90b51f4f439048fd Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 28 Nov 2023 05:57:29 +0000 Subject: [PATCH 16/24] Added config instance variable to LinkedInClient class and updated it's usage. --- tap_linkedin_ads/__init__.py | 2 +- tap_linkedin_ads/client.py | 2 ++ tap_linkedin_ads/streams.py | 2 +- tap_linkedin_ads/sync.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tap_linkedin_ads/__init__.py b/tap_linkedin_ads/__init__.py index 8da87df..2d0ba27 100644 --- a/tap_linkedin_ads/__init__.py +++ b/tap_linkedin_ads/__init__.py @@ -21,7 +21,7 @@ def do_discover(client): LOGGER.info('Starting discover') - client.check_accounts(client.get_config()) + client.check_accounts(client.config) catalog = _discover() json.dump(catalog.to_dict(), sys.stdout, indent=2) LOGGER.info('Finished discover') diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index eff32a9..23b55ea 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -134,6 +134,7 @@ def __init__(self, # pylint: disable=too-many-arguments self.__client_secret = client_secret self.__refresh_token = refresh_token self.__config_path = config_path + self.config = self.get_config() self.__user_agent = user_agent self.__access_token = access_token self.__expires = None @@ -192,6 +193,7 @@ def write_access_token_to_config(self): config = json.load(file) # Set new access_token config['access_token'] = self.__access_token + self.config = config with open(self.__config_path, 'w') as file: json.dump(config, file, indent=2) diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py index d998f70..9d26c67 100644 --- a/tap_linkedin_ads/streams.py +++ b/tap_linkedin_ads/streams.py @@ -315,7 +315,7 @@ def sync_endpoint(self, url_list = [] if self.tap_stream_id in NEW_PATH_STREAMS: querystring = '&'.join(['%s=%s' % (key, value) for (key, value) in endpoint_params.items()]) - config = client.get_config() + config = client.config account_list = config['accounts'].replace(" ", "").split(",") for account in account_list: url = '{}/adAccounts/{}/{}?{}'.format(BASE_URL, account, self.path, querystring) diff --git a/tap_linkedin_ads/sync.py b/tap_linkedin_ads/sync.py index b5bf329..fedf5b0 100644 --- a/tap_linkedin_ads/sync.py +++ b/tap_linkedin_ads/sync.py @@ -70,7 +70,7 @@ def sync(client, catalog, state): """ sync selected streams. """ - config = client.get_config() + config = client.config start_date = config['start_date'] page_size = get_page_size(config) From a00f213f5735285cd56d14ca1eb89fc232fea0fa Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 28 Nov 2023 10:47:36 +0000 Subject: [PATCH 17/24] Fixed test_sync unit test as per new config paramter changes. --- tests/unittests/test_sync.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_sync.py b/tests/unittests/test_sync.py index 74f367f..eb75ae7 100644 --- a/tests/unittests/test_sync.py +++ b/tests/unittests/test_sync.py @@ -147,18 +147,19 @@ class TestSync(unittest.TestCase): ['test_sync_without_datewindow', {'start_date': '2019-06-01T00:00:00Z', 'accounts': '12345'}, 30], ['test_sync_with_datewindow', {'start_date': '2019-06-01T00:00:00Z', 'date_window_size': 7, 'accounts': '1245'}, 7] ]) + @mock.patch('tap_linkedin_ads.client.LinkedinClient.get_config') @mock.patch('tap_linkedin_ads.streams.LinkedInAds.sync_endpoint', return_value=(1, '2020-06-01T00:00:00Z')) - def test_sync(self, name, config, expected_date_window, mock_sync_endpoint): + def test_sync(self, name, config, expected_date_window, mock_sync_endpoint, mock_get_config): """ Test sync function """ client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client.config = config state = {} - sync(client, config, CATALOG, state) + sync(client, CATALOG, state) mock_sync_endpoint.assert_called_with(client=client, catalog=CATALOG, - config=config, state=state, page_size=100, start_date="2019-06-01T00:00:00Z", From f3efe122c6bf8c4b7cadb7e2fae22693a7d18e0a Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 28 Nov 2023 11:31:26 +0000 Subject: [PATCH 18/24] Removed get_config method and added config instance variable in LinkedinClient class and updated its usage. --- tap_linkedin_ads/__init__.py | 3 ++- tap_linkedin_ads/client.py | 9 +++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tap_linkedin_ads/__init__.py b/tap_linkedin_ads/__init__.py index 2d0ba27..ff17063 100644 --- a/tap_linkedin_ads/__init__.py +++ b/tap_linkedin_ads/__init__.py @@ -37,7 +37,8 @@ def main(): parsed_args.config.get('access_token'), parsed_args.config_path, REQUEST_TIMEOUT, - parsed_args.config['user_agent'] + parsed_args.config['user_agent'], + parsed_args.config ) as client: state = {} diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index 23b55ea..538b04d 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -129,12 +129,13 @@ def __init__(self, # pylint: disable=too-many-arguments access_token, config_path, request_timeout=REQUEST_TIMEOUT, - user_agent=None): + user_agent=None, + config={}): self.__client_id = client_id self.__client_secret = client_secret self.__refresh_token = refresh_token self.__config_path = config_path - self.config = self.get_config() + self.config = config self.__user_agent = user_agent self.__access_token = access_token self.__expires = None @@ -176,10 +177,6 @@ def set_mock_expires_for_test(self, mock_expire): self.__expires = mock_expire return self.__expires - def get_config(self): - with open(self.__config_path) as file: - return json.load(file) - def write_access_token_to_config(self): """ Write an updated access token in the config to reuse in the next sync. From d21583e89cb969b7b0d9e16f47aad38c200f3ab3 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 28 Nov 2023 11:40:53 +0000 Subject: [PATCH 19/24] Fixed test_main & test_sync unit tests as per new config changes. --- tests/unittests/test_main.py | 2 -- tests/unittests/test_sync.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/unittests/test_main.py b/tests/unittests/test_main.py index f6cb984..f82aa96 100644 --- a/tests/unittests/test_main.py +++ b/tests/unittests/test_main.py @@ -46,7 +46,6 @@ def test_sync_with_catalog(self, mock_sync, mock_discover, mock_args): # Verify `_sync` is called with expected arguments mock_sync.assert_called_with(client=mock.ANY, - config=self.mock_config, catalog=self.mock_catalog, state={}) @@ -74,6 +73,5 @@ def test_sync_with_state(self, mock_sync, mock_discover, mock_args): # Verify `_sync` is called with expected arguments mock_sync.assert_called_with(client=mock.ANY, - config=self.mock_config, state=mock_state, catalog=self.mock_catalog) diff --git a/tests/unittests/test_sync.py b/tests/unittests/test_sync.py index eb75ae7..4685a09 100644 --- a/tests/unittests/test_sync.py +++ b/tests/unittests/test_sync.py @@ -147,9 +147,8 @@ class TestSync(unittest.TestCase): ['test_sync_without_datewindow', {'start_date': '2019-06-01T00:00:00Z', 'accounts': '12345'}, 30], ['test_sync_with_datewindow', {'start_date': '2019-06-01T00:00:00Z', 'date_window_size': 7, 'accounts': '1245'}, 7] ]) - @mock.patch('tap_linkedin_ads.client.LinkedinClient.get_config') @mock.patch('tap_linkedin_ads.streams.LinkedInAds.sync_endpoint', return_value=(1, '2020-06-01T00:00:00Z')) - def test_sync(self, name, config, expected_date_window, mock_sync_endpoint, mock_get_config): + def test_sync(self, name, config, expected_date_window, mock_sync_endpoint): """ Test sync function """ From 754e11f0cb7abc2a23de4bed4daf2f77872039f8 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 28 Nov 2023 12:00:03 +0000 Subject: [PATCH 20/24] Changed the order of config parameter in LinkedinClient object. --- tap_linkedin_ads/__init__.py | 4 ++-- tap_linkedin_ads/client.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tap_linkedin_ads/__init__.py b/tap_linkedin_ads/__init__.py index ff17063..1ea1787 100644 --- a/tap_linkedin_ads/__init__.py +++ b/tap_linkedin_ads/__init__.py @@ -36,9 +36,9 @@ def main(): parsed_args.config.get('refresh_token', None), parsed_args.config.get('access_token'), parsed_args.config_path, + parsed_args.config, REQUEST_TIMEOUT, - parsed_args.config['user_agent'], - parsed_args.config + parsed_args.config['user_agent'] ) as client: state = {} diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index 538b04d..77abaeb 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -128,9 +128,9 @@ def __init__(self, # pylint: disable=too-many-arguments refresh_token, access_token, config_path, + config={}, request_timeout=REQUEST_TIMEOUT, - user_agent=None, - config={}): + user_agent=None): self.__client_id = client_id self.__client_secret = client_secret self.__refresh_token = refresh_token From cfee98ea34071bf7e50b4167510b6dddccdee10d Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 28 Nov 2023 13:54:11 +0000 Subject: [PATCH 21/24] Changes: 1) Fixed unit tests as per the new config param changes in LinkedinClient class. 2) Made changes in check_accounts() method as per config param changes. 3) Fixed unit tests as per changes in check_accounts() method. --- tap_linkedin_ads/__init__.py | 2 +- tap_linkedin_ads/client.py | 3 ++- tests/unittests/test_client.py | 4 +-- tests/unittests/test_exception_handling.py | 29 +++++++++++----------- tests/unittests/test_streams.py | 20 +++++++-------- tests/unittests/test_sync.py | 3 +-- tests/unittests/test_timeout.py | 11 +++++--- 7 files changed, 37 insertions(+), 35 deletions(-) diff --git a/tap_linkedin_ads/__init__.py b/tap_linkedin_ads/__init__.py index 1ea1787..5ff0a9a 100644 --- a/tap_linkedin_ads/__init__.py +++ b/tap_linkedin_ads/__init__.py @@ -21,7 +21,7 @@ def do_discover(client): LOGGER.info('Starting discover') - client.check_accounts(client.config) + client.check_accounts() catalog = _discover() json.dump(catalog.to_dict(), sys.stdout, indent=2) LOGGER.info('Finished discover') diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index 77abaeb..32e27b3 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -290,7 +290,8 @@ def fetch_and_set_access_token(self): (Server5xxError, requests.exceptions.ConnectionError, requests.exceptions.Timeout), max_tries=5, factor=2) - def check_accounts(self, config): + def check_accounts(self): + config = self.config headers = {} if self.__user_agent: headers['User-Agent'] = self.__user_agent diff --git a/tests/unittests/test_client.py b/tests/unittests/test_client.py index 1a13605..e9402cc 100644 --- a/tests/unittests/test_client.py +++ b/tests/unittests/test_client.py @@ -78,7 +78,7 @@ def test_no_access_token(self, mocked_post, mock_write_token): ''' Ensure that we get an access token if we don't already have one ''' - client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', None) + client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', None, 'config_path') expires = client.get_expires_time_for_test() assert expires is None @@ -87,7 +87,7 @@ def test_no_access_token(self, mocked_post, mock_write_token): mocked_token_check_response = mock.Mock() mocked_token_check_response.json.return_value = { "access_token": "abcdef12345", - "expires_at": old_time + "expires_in": old_time } mocked_token_check_response.status_code = 200 diff --git a/tests/unittests/test_exception_handling.py b/tests/unittests/test_exception_handling.py index a680b47..51fff01 100644 --- a/tests/unittests/test_exception_handling.py +++ b/tests/unittests/test_exception_handling.py @@ -78,9 +78,9 @@ def test_check_accounts_backoff(self, error_code, message, error, mock_requests, "status": error_code, "code": ""} mock_requests.return_value = get_response(error_code, json_resp) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', config) with self.assertRaises(error) as e: - linkedIn_client.check_accounts(config) + linkedIn_client.check_accounts() # Verify that `session.get` was called 5 times self.assertEqual(mock_requests.call_count, 5) @@ -97,9 +97,9 @@ def test_check_accounts_backoff_2(self, mock_response, error, mock_requests, moc """ config = {"accounts": "acc1"} mock_requests.side_effect = mock_response - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', config) with self.assertRaises(error) as e: - linkedIn_client.check_accounts(config) + linkedIn_client.check_accounts() # Verify that `session.get` was called 5 times self.assertEqual(mock_requests.call_count, 5) @@ -111,7 +111,7 @@ def test_requests_timeout_backoff(self, mock_requests, mock_sleep): Test `request` method will backoff 5 times Timeout error. """ mock_requests.side_effect = requests.exceptions.Timeout - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(requests.exceptions.Timeout) as e: linkedIn_client.request("GET") @@ -139,7 +139,7 @@ def test_custom_error_message(self, mocked_access_token, mocked_request, error_c """ mocked_request.return_value = mock_response expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(error) as e: linkedIn_client.request("GET") @@ -162,7 +162,7 @@ def test_response_error_message(self, mocked_access_token, mock_request, error_c "status": error_code} mock_request.return_value = get_response(error_code, json_resp) expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(error) as e: linkedIn_client.request("GET") @@ -178,7 +178,7 @@ def test_401_error_expired_access_token(self, mocked_logger, mocked_access_token "status": 401, "code": "UNAUTHORIZED"} mocked_request.return_value = get_response(401, response_json) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(client.LinkedInUnauthorizedError) as e: linkedIn_client.request("POST") @@ -193,7 +193,7 @@ def test_json_decoder_error(self, mocked_access_token, mocked_request): response.status_code = 400 response._content = "abcd" mocked_request.return_value = response - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(client.LinkedInBadRequestError) as e: linkedIn_client.request("POST") @@ -223,7 +223,7 @@ def test_custom_error_message(self, mock_request, mock_sleep, error_code, mock_r """ mock_request.return_value = mock_response expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(error) as e: linkedIn_client.fetch_and_set_access_token() @@ -248,7 +248,7 @@ def test_resopnse_error_message(self, mock_request, mock_sleep, error_code, erro "status": error_code} mock_request.return_value = get_response(error_code, json_resp) expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(error) as e: linkedIn_client.fetch_and_set_access_token() @@ -264,7 +264,7 @@ def test_401_error_expired_access_token(self, mock_logger, mock_request, mock_sl "status": 401, "code": "UNAUTHORIZED"} mock_request.return_value = get_response(401, response_json) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'config_path', 'access_token') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') with self.assertRaises(client.LinkedInUnauthorizedError) as e: linkedIn_client.fetch_and_set_access_token() @@ -279,7 +279,7 @@ class TestCheckAccounts(unittest.TestCase): """ Test exception handling for `check_accounts` method of client. """ - _client = client.LinkedinClient("","","", "","", "", "USR_AGENT") + _client = client.LinkedinClient("","","", "","", {"accounts": "account1,account2"}) @parameterized.expand([ (400,), @@ -289,12 +289,11 @@ def test_invalid_accouts(self, mock_get, error_code): """ Test for 400, 404 errors custom error message is written. """ - config = {"accounts": "account1,account2"} error_message = "Invalid Linked Ads accounts provided during the configuration:{}".format(["account1","account2"]) mock_get.return_value = get_response(error_code) with self.assertRaises(Exception) as e: - self._client.check_accounts(config) + self._client.check_accounts() # Verify that error message is expected self.assertEqual(str(e.exception), error_message) diff --git a/tests/unittests/test_streams.py b/tests/unittests/test_streams.py index e2ab5a8..361d023 100644 --- a/tests/unittests/test_streams.py +++ b/tests/unittests/test_streams.py @@ -124,7 +124,7 @@ def test_sync_analytics_endpoint(self, name, next_url, expected_call_count, mock Test that sync_analytics_endpoint function works properly for single page as well as multiple pages. """ mock_next_url.side_effect = next_url - client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {"accounts": "123"}) data = list(sync_analytics_endpoint(client, "stream", "path", "query=query")) # Verify that get method of client is called expected times. @@ -306,7 +306,7 @@ def test_process_records(self, name, stream_obj, records, replication_key, expec self.assertEqual(expected_record_count, actual_record_count) @parameterized.expand([ - ['test_only_parent_selcted_stream', ['accounts'], ACCOUNT_OBJ, + ['test_only_parent_selected_stream', ['accounts'], ACCOUNT_OBJ, [{'paging': {'start': 0, 'count': 100, 'links': [], 'total': 1},'elements': [{'changeAuditStamps': {'created': {'time': 1564585620000}, 'lastModified': {'time': 1564585620000}}, 'id': 1}]}], 0, 1 ], @@ -320,10 +320,10 @@ def test_process_records(self, name, stream_obj, records, replication_key, expec {'paging': {'start': 0, 'count': 100, 'links': [], 'total': 0},'elements': []}], 1, 1 ], - ['test_only_parent_selcted_stream', ['campaigns'], CAMPAIGN_OBJ, + ['test_only_parent_selected_stream', ['campaigns'], CAMPAIGN_OBJ, [{'paging': {'start': 0, 'count': 100, 'links': [], 'total': 1},'elements': [{'changeAuditStamps': {'created': {'time': 1564585620000}, 'lastModified': {'time': 1564585620000}}, 'id': 1}]}], 0, 1 - ] + ] ]) @mock.patch("tap_linkedin_ads.streams.LinkedInAds.sync_ad_analytics", return_value=(1, "2019-07-31T15:07:00.000000Z")) @mock.patch("tap_linkedin_ads.streams.LinkedInAds.get_bookmark", return_value = "2019-07-31T15:07:00.000000Z") @@ -336,7 +336,7 @@ def test_sync_endpoint(self, name, selected_streams, stream_obj, mock_response, """ Test sync_endpoint function for parent and child streams. """ - client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {"accounts": "123"}) state={} start_date='2019-06-01T00:00:00Z' page_size = 100 @@ -344,9 +344,8 @@ def test_sync_endpoint(self, name, selected_streams, stream_obj, mock_response, mock_get.side_effect = [{"elements": [{"1": "a"}]}] mock_client.side_effect = mock_response - config = {"accounts": "123"} mock_process_records.return_value = "2019-07-31T15:07:00.000000Z",1 - actual_total_record, actual_max_bookmark = stream_obj.sync_endpoint(client, CATALOG, config, state, page_size, start_date, selected_streams, date_window_size) + actual_total_record, actual_max_bookmark = stream_obj.sync_endpoint(client, CATALOG, state, page_size, start_date, selected_streams, date_window_size) # Verify total no of records self.assertEqual(actual_total_record, mock_record_count) @@ -365,17 +364,16 @@ def test_sync_endpoint_for_reference_organization_id_is_None(self, mock_write_sc """ Verify that tap skips API call for video_ads stream if owner_id in the parent's record is None. """ - client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {"accounts": "123"}) state={'currently_syncing': 'accounts'} start_date='2019-06-01T00:00:00Z' page_size = 100 date_window_size = 7 selected_streams = ['accounts', 'video_ads'] - config = {"accounts": "123"} mock_client.side_effect = [{'paging': {'start': 0, 'count': 100, 'links': [], 'total': 1},'elements': [{'changeAuditStamps': {'created': {'time': 1564585620000}, 'lastModified': {'time': 1564585620000}}, 'id': 1}]}] mock_process_records.return_value = "2019-07-31T15:07:00.000000Z",1 - ACCOUNT_OBJ.sync_endpoint(client, CATALOG, config, state, page_size, start_date, selected_streams, date_window_size) + ACCOUNT_OBJ.sync_endpoint(client, CATALOG, state, page_size, start_date, selected_streams, date_window_size) mock_warning.assert_called_with('Skipping video_ads call for %s account as reference_organization_id is not found.', 'urn:li:sponsoredAccount:1') @@ -395,7 +393,7 @@ def test_sync_ad_analytics(self, name, expected_record_count, expected_max_bookm Test that `sync_ad_analytics` function work properly for zero records as well as multiple records. """ - client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {"accounts": "123"}) bookmark='2022-08-01T00:00:00Z' date_window_size = 7 diff --git a/tests/unittests/test_sync.py b/tests/unittests/test_sync.py index 4685a09..7f01703 100644 --- a/tests/unittests/test_sync.py +++ b/tests/unittests/test_sync.py @@ -152,8 +152,7 @@ def test_sync(self, name, config, expected_date_window, mock_sync_endpoint): """ Test sync function """ - client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') - client.config = config + client = LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', config) state = {} sync(client, CATALOG, state) diff --git a/tests/unittests/test_timeout.py b/tests/unittests/test_timeout.py index 9534477..e310938 100644 --- a/tests/unittests/test_timeout.py +++ b/tests/unittests/test_timeout.py @@ -39,6 +39,7 @@ def test_timeout_values(self, test_value, expected_value): refresh_token=config['refresh_token'], access_token=config['access_token'], config_path='config_path', + config=config, user_agent=config['user_agent'], request_timeout=config.get('request_timeout')) @@ -62,6 +63,7 @@ def test_timeout_value_not_passed_in_config(self): refresh_token=config['refresh_token'], access_token=config['access_token'], config_path='config_path', + config=config, user_agent=config['user_agent'], request_timeout=config.get('request_timeout')) @@ -88,6 +90,7 @@ class TestTimeoutBackoff(unittest.TestCase): refresh_token=config['refresh_token'], access_token=config['access_token'], config_path='config_path', + config=config, user_agent=config['user_agent'], request_timeout=config.get('request_timeout')) @@ -123,7 +126,7 @@ def test_timeout_error__check_accounts(self, mocked_request, mocked_sleep): with self.assertRaises(requests.Timeout): # function call - self.client.check_accounts(self.config) + self.client.check_accounts() # verify that we backoff for 5 times self.assertEquals(mocked_request.call_count, 5) @@ -172,7 +175,8 @@ def test_connection_error__check_access_token(self, mocked_request, mocked_sleep client_secret=config['client_secret'], refresh_token=config['refresh_token'], access_token=config['access_token'], - config_path='config_path', + config_path='config_path', + config=config, user_agent=config['user_agent'], request_timeout=config.get('request_timeout')) as cl: pass @@ -201,12 +205,13 @@ def test_connection_error__check_accounts(self, mocked_request, mocked_sleep): refresh_token=config['refresh_token'], access_token=config['access_token'], config_path='config_path', + config=config, user_agent=config['user_agent'], request_timeout=config.get('request_timeout')) with self.assertRaises(requests.ConnectionError): # function call - cl.check_accounts(config) + cl.check_accounts() # verify that we backoff for 5 times self.assertEquals(mocked_request.call_count, 5) From afa8785c0791c07bc8f90a0a10f9ac85865ebe9f Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Tue, 28 Nov 2023 14:20:37 +0000 Subject: [PATCH 22/24] Changes: 1) Fixed dangerous default value pylint error in LinkedinClient class for config parameter. 2) Fixed unit tests as per above change. --- tap_linkedin_ads/client.py | 2 +- tests/unittests/test_client.py | 10 +++++----- tests/unittests/test_exception_handling.py | 20 ++++++++++---------- tests/unittests/test_timeout.py | 1 + 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/tap_linkedin_ads/client.py b/tap_linkedin_ads/client.py index 32e27b3..7113550 100644 --- a/tap_linkedin_ads/client.py +++ b/tap_linkedin_ads/client.py @@ -128,7 +128,7 @@ def __init__(self, # pylint: disable=too-many-arguments refresh_token, access_token, config_path, - config={}, + config, request_timeout=REQUEST_TIMEOUT, user_agent=None): self.__client_id = client_id diff --git a/tests/unittests/test_client.py b/tests/unittests/test_client.py index e9402cc..02b2f2e 100644 --- a/tests/unittests/test_client.py +++ b/tests/unittests/test_client.py @@ -14,7 +14,7 @@ def test_access_token_empty_expires(self, mocked_post, mock_write_token): ''' Ensure that we retrieve and set expires for client with no self.__expires ''' - client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) future_time = int(datetime.utcnow().timestamp()) + 88400 mocked_response = mock.Mock() @@ -37,7 +37,7 @@ def test_access_token_expires_valid(self, mocked_post, mock_write_token): ''' Ensure that we check and return on valid self.__expires ''' - client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) future_time = int(datetime.utcnow().timestamp()) + 88400 mocked_response = mock.MagicMock() @@ -58,7 +58,7 @@ def test_access_token_expires_invalid(self, mocked_post, mock_write_token): ''' Ensure that we check self.__expires and retrieve new access token if it has expired ''' - client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) old_time = int(datetime.utcnow().timestamp()) - 100 mocked_response = mock.MagicMock() @@ -78,7 +78,7 @@ def test_no_access_token(self, mocked_post, mock_write_token): ''' Ensure that we get an access token if we don't already have one ''' - client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', None, 'config_path') + client = _client.LinkedinClient('client_id', 'client_secret', 'refresh_token', None, 'config_path', {}) expires = client.get_expires_time_for_test() assert expires is None @@ -108,7 +108,7 @@ def test_no_refresh_token(self, mocked_post, mock_write_token): Ensure that we use the existing access token if we don't have a refresh token ''' expected_access_token = 'access_token' - client = _client.LinkedinClient(None, None, None, 'access_token', 'config_path') + client = _client.LinkedinClient(None, None, None, 'access_token', 'config_path', {}) client.fetch_and_set_access_token() actual = client.access_token diff --git a/tests/unittests/test_exception_handling.py b/tests/unittests/test_exception_handling.py index 51fff01..1a3ff69 100644 --- a/tests/unittests/test_exception_handling.py +++ b/tests/unittests/test_exception_handling.py @@ -37,7 +37,7 @@ def test_fetch_and_set_token_backoff(self, error_code, message, error, mock_requ "code": ""} mock_requests.return_value = get_response(error_code, json_resp) with self.assertRaises(error) as e: - with client.LinkedinClient("","","refresh_token", "", "") as _client: + with client.LinkedinClient("","","refresh_token", "", "", {}) as _client: pass # Verify that `session.post` was called 5 times @@ -56,7 +56,7 @@ def test_fetch_and_set_token_backoff_2(self, mock_response, error, mock_requests """ mock_requests.side_effect = mock_response with self.assertRaises(error) as e: - with client.LinkedinClient("","","refresh_token", "", "") as _client: + with client.LinkedinClient("","","refresh_token", "", "", {}) as _client: pass # Verify that `session.post` was called 5 times @@ -111,7 +111,7 @@ def test_requests_timeout_backoff(self, mock_requests, mock_sleep): Test `request` method will backoff 5 times Timeout error. """ mock_requests.side_effect = requests.exceptions.Timeout - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(requests.exceptions.Timeout) as e: linkedIn_client.request("GET") @@ -139,7 +139,7 @@ def test_custom_error_message(self, mocked_access_token, mocked_request, error_c """ mocked_request.return_value = mock_response expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(error) as e: linkedIn_client.request("GET") @@ -162,7 +162,7 @@ def test_response_error_message(self, mocked_access_token, mock_request, error_c "status": error_code} mock_request.return_value = get_response(error_code, json_resp) expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(error) as e: linkedIn_client.request("GET") @@ -178,7 +178,7 @@ def test_401_error_expired_access_token(self, mocked_logger, mocked_access_token "status": 401, "code": "UNAUTHORIZED"} mocked_request.return_value = get_response(401, response_json) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(client.LinkedInUnauthorizedError) as e: linkedIn_client.request("POST") @@ -193,7 +193,7 @@ def test_json_decoder_error(self, mocked_access_token, mocked_request): response.status_code = 400 response._content = "abcd" mocked_request.return_value = response - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(client.LinkedInBadRequestError) as e: linkedIn_client.request("POST") @@ -223,7 +223,7 @@ def test_custom_error_message(self, mock_request, mock_sleep, error_code, mock_r """ mock_request.return_value = mock_response expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(error) as e: linkedIn_client.fetch_and_set_access_token() @@ -248,7 +248,7 @@ def test_resopnse_error_message(self, mock_request, mock_sleep, error_code, erro "status": error_code} mock_request.return_value = get_response(error_code, json_resp) expected_message = "HTTP-error-code: {}, Error: {}".format(error_code, message) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(error) as e: linkedIn_client.fetch_and_set_access_token() @@ -264,7 +264,7 @@ def test_401_error_expired_access_token(self, mock_logger, mock_request, mock_sl "status": 401, "code": "UNAUTHORIZED"} mock_request.return_value = get_response(401, response_json) - linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path') + linkedIn_client = client.LinkedinClient('client_id', 'client_secret', 'refresh_token', 'access_token', 'config_path', {}) with self.assertRaises(client.LinkedInUnauthorizedError) as e: linkedIn_client.fetch_and_set_access_token() diff --git a/tests/unittests/test_timeout.py b/tests/unittests/test_timeout.py index e310938..22f2875 100644 --- a/tests/unittests/test_timeout.py +++ b/tests/unittests/test_timeout.py @@ -109,6 +109,7 @@ def test_timeout_error__check_access_token(self, mocked_request, mocked_sleep): refresh_token=self.config['refresh_token'], access_token=self.config['access_token'], config_path='config_path', + config={}, user_agent=self.config['user_agent'], request_timeout=self.config.get('request_timeout')) as cl: pass From 6d2cc567a981ce3ccda218d49013b1d100019630 Mon Sep 17 00:00:00 2001 From: Shantanu Dhiman Date: Wed, 29 Nov 2023 12:01:54 +0530 Subject: [PATCH 23/24] [TDL-24282] Fix integration tests (#65) * Changes: 1. Fixed video_ads metadata. 2. Fixed all fields & automatic fields tests. * Removed analytics streams from start_date, automatic_fields & bookmarks tests. * Fixed Parent child independent test for Ad analytics streams. * Removed unnecessary fields. * Removed beta fields from KNOWN_MISSING_FIELDS in All fields test for ad analytics streams. * Changes: 1) Added comment mentioning why the Ad Analytics streams were removed from testing. 2) Removed unneccary whitespace. * Fixed formatting. --- tests/base.py | 4 +-- tests/test_all_fields.py | 35 ++++++-------------------- tests/test_automatic_fields.py | 3 +-- tests/test_bookmark.py | 3 ++- tests/test_parent_child_independent.py | 3 ++- tests/test_start_date.py | 5 +++- 6 files changed, 19 insertions(+), 34 deletions(-) diff --git a/tests/base.py b/tests/base.py index dbc018a..f660ffb 100644 --- a/tests/base.py +++ b/tests/base.py @@ -101,10 +101,10 @@ def expected_metadata(self): self.REPLICATION_KEYS: {'last_modified_time'} }, 'video_ads': { - self.PRIMARY_KEYS: {'content_reference'}, + self.PRIMARY_KEYS: {'id'}, self.REPLICATION_METHOD: self.INCREMENTAL, self.OBEYS_START_DATE: True, - self.REPLICATION_KEYS: {'last_modified_time'} + self.REPLICATION_KEYS: {'last_modified_at'} }, 'account_users': { self.PRIMARY_KEYS: {'account_id', 'user_person_id'}, diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index 6f86039..91cf65b 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -5,20 +5,14 @@ # As we can't find the below fields in the docs and also # it won't be generated by Linkedin APIs now so expected. KNOWN_MISSING_FIELDS = { - "account_users": { - "campaign_contact" - }, - "creatives": { - "reference_share_id", - }, + "account_users": {"campaign_contact"}, + "creatives": {"reference_share_id"}, "campaigns": { "associated_entity_person_id", "targeting", - "version_tag", - }, - "campaign_groups": { - "allowed_campaign_types" + "version_tag" }, + "campaign_groups": {"allowed_campaign_types"}, "video_ads": { "content_reference_share_id", "content_reference_ucg_post_id", @@ -26,22 +20,8 @@ "accounts": { "total_budget_ends_at", "total_budget", - "reference_person_id", - }, - "ad_analytics_by_creative": { - "average_daily_reach_metrics", - "average_previous_seven_day_reach_metrics", - "average_previous_thirty_day_reach_metrics", - #BUG: TDL-22692 - "approximate_unique_impressions", - }, - "ad_analytics_by_campaign": { - "average_daily_reach_metrics" - "average_previous_seven_day_reach_metrics", - "average_previous_thirty_day_reach_metrics", - #BUG: TDL-22692 - "approximate_unique_impressions", - }, + "reference_person_id" + } } class AllFields(TestLinkedinAdsBase): @@ -59,7 +39,8 @@ def test_run(self): - Verify that more than just the automatic fields are replicated for each stream. """ - expected_streams = self.expected_streams() + # Removed Ad Analytics streams from expected streams as there is insufficient data in test account + expected_streams = self.expected_streams() - {'ad_analytics_by_campaign', 'ad_analytics_by_creative'} # Instantiate connection conn_id = connections.ensure_connection(self) diff --git a/tests/test_automatic_fields.py b/tests/test_automatic_fields.py index 61ae849..eb74092 100644 --- a/tests/test_automatic_fields.py +++ b/tests/test_automatic_fields.py @@ -17,11 +17,10 @@ def test_run(self): • Verify that all replicated records have unique primary key values. """ - streams_to_test = self.expected_streams() # Skip `ad_analytics_by_campaign` and `ad_analytics_by_creative` from the test because we pass only selected fields # in the API param of these streams and that's why in this test we get 0 records. # So, if we select at least one available field then API returns a record otherwise it returns 0 records. - streams_to_test = streams_to_test - {'ad_analytics_by_campaign', 'ad_analytics_by_creative'} + streams_to_test = self.expected_streams() - {'ad_analytics_by_campaign', 'ad_analytics_by_creative'} conn_id = connections.ensure_connection(self) diff --git a/tests/test_bookmark.py b/tests/test_bookmark.py index 225d9f2..3cdf1af 100644 --- a/tests/test_bookmark.py +++ b/tests/test_bookmark.py @@ -39,7 +39,8 @@ def test_run(self): different values for the replication key """ - streams_to_test = self.expected_streams() + # Removed Ad Analytics streams from expected streams as there is insufficient data in test account + streams_to_test = self.expected_streams() - {'ad_analytics_by_campaign', 'ad_analytics_by_creative'} expected_replication_keys = self.expected_replication_keys() expected_replication_methods = self.expected_replication_method() diff --git a/tests/test_parent_child_independent.py b/tests/test_parent_child_independent.py index 7ab3b61..dadfe03 100644 --- a/tests/test_parent_child_independent.py +++ b/tests/test_parent_child_independent.py @@ -12,7 +12,8 @@ def test_run(self): • Verify that if only child streams are selected then only child streams are replicated. """ - child_streams = {"video_ads", "creatives", "ad_analytics_by_campaign", "ad_analytics_by_creative"} + # Removed Ad Analytics streams from child streams as there is insufficient data in test account + child_streams = {"video_ads", "creatives"} # Instantiate connection conn_id = connections.ensure_connection(self) diff --git a/tests/test_start_date.py b/tests/test_start_date.py index 80513e2..5c741d1 100644 --- a/tests/test_start_date.py +++ b/tests/test_start_date.py @@ -18,10 +18,13 @@ def name(): def test_run(self): + # Removed Ad Analytics streams from expected streams as there is insufficient data in test account + expected_streams = self.expected_streams() - {'ad_analytics_by_campaign', 'ad_analytics_by_creative'} + streams_to_test = {"account_users"} self.run_start_date(streams_to_test, "2021-08-07T00:00:00Z") - self.run_start_date(self.expected_streams() - streams_to_test, "2019-08-01T00:00:00Z") + self.run_start_date(expected_streams - streams_to_test, "2019-08-01T00:00:00Z") def run_start_date(self, expected_streams, start_date_2): """ From 80bcf5e73f7dc25be907db00e192a84d56948b19 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Wed, 29 Nov 2023 06:34:16 +0000 Subject: [PATCH 24/24] Updated scope information for video_ads stream in new API version upgrade in README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0358e60..ca2821e 100644 --- a/README.md +++ b/README.md @@ -118,11 +118,13 @@ The API user account should be assigned one of the following roles: - **VIEWER** (Recommended) The API user account should be assigned the following **permissions** for the API endpoints: -- accounts, account_users, video_ads, campaign_groups, campaigns, creatives: +- accounts, account_users, campaign_groups, campaigns, creatives: - r_ads: read ads (Recommended) - rw_ads: read-write ads - ad_analytics_by_campaign, ad_analytics_by_creative: - r_ads_reporting: read ads reporting +- video_ads: + - r_organization_social: read video ads **NOTE**: Legacy permissions (r_ad_campaigns) have been migrated to the new permissions (r_ads and r_ads_reporting) based on this [permissions mapping](https://docs.microsoft.com/en-us/linkedin/shared/references/migrations/marketing-permissions-migration?context=linkedin/marketing/context).