From f4c4992ab6215916866922dfc56cf99c29fab18a Mon Sep 17 00:00:00 2001 From: Ivan Ogasawara Date: Tue, 22 Jun 2021 20:51:50 -0400 Subject: [PATCH] Use pagination just for assigment --- tap_harvest_forecast/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tap_harvest_forecast/__init__.py b/tap_harvest_forecast/__init__.py index 0f550af..ffc5e1c 100644 --- a/tap_harvest_forecast/__init__.py +++ b/tap_harvest_forecast/__init__.py @@ -176,11 +176,19 @@ def sync_endpoint(catalog_entry, schema, mdata, date_fields = None): end = utils.strptime_to_utc(get_end(catalog_entry.tap_stream_id)) delta = datetime.timedelta(days=180) + if catalog_entry.tap_stream_id == 'assignments': + date_range = window(start, end, delta) + else: + date_range = [(start, None)] + # for slice of 180 days in total date range from start date to arbitrary end date, x years into the future? ids = [] with Transformer() as transformer: - for dateStart, dateEnd in window(start, end, delta): - params = {"start_date": dateStart.strftime(DATE_FORMAT), "end_date": dateEnd.strftime(DATE_FORMAT)} + for dateStart, dateEnd in date_range: + params = {"start_date": dateStart.strftime(DATE_FORMAT)} + if dateEnd: + params["end_date"] = dateEnd.strftime(DATE_FORMAT) + data = request(url, params)[catalog_entry.tap_stream_id] for row in data: rec = transformer.transform(row, schema, mdata)