gis fetcher fixes

Author: OriHoch

datacity_ckan_dgp/ckan/__init__.py

@@ -20,6 +20,8 @@ def api_request(method, instance_name, action_name, auth=True, **kwargs):
     headers = {'Authorization': api_key} if auth and api_key else {}
     if "://data.gov.il" in url:
         headers['user-agent'] = 'datagov-external-client'
+    if 'verify' not in kwargs:
+        kwargs['verify'] = os.getenv("CKAN_VERIFY_SSL") != "no"
     res = getattr(requests, method.lower())(url, headers=headers, **kwargs)
     try:
         return res.json()

datacity_ckan_dgp/operators/gis_fetcher.py

@@ -28,14 +28,17 @@ def gis_query_geojson_iterate_all(gis_url):
             'resultRecordCount': result_record_count
         }
         res = requests.get(url, params=params)
-        res.raise_for_status()
-        data = res.json()
-        assert data['type'] == 'FeatureCollection'
-        if not data['features']:
-            break
-        for feature in data['features']:
-            yield feature
-            result_offset += 1
+        try:
+            assert res.status_code == 200
+            data = res.json()
+            assert data['type'] == 'FeatureCollection'
+            if not data['features']:
+                break
+            for feature in data['features']:
+                yield feature
+                result_offset += 1
+        except Exception as e:
+            raise Exception(f'failed to fetch geojson from {url} with params {params}\n{res.content}') from e
 
 
 def iterate_gis_jsonlines(tmpdir):