Merge pull request #12 from CyberCRI/fix-wikipedia-updater-compare-error

Catch KeyError in case of unexpected response from Wikipedia API

Author: jmsevin

welearn_datastack/modules/wikipedia_updater.py

@@ -81,6 +81,9 @@ def compare_with_current_version(document: WeLearnDocument) -> bool:
     }
 
     resp = session.get(url=url, params=params)
-    data = resp.json()["compare"]
+    try:
+        data = resp.json()["compare"]
+    except KeyError as e:
+        raise KeyError("Unexpected response from Wikipedia API") from e
 
     return data["diffsize"] > 0.05 * data["fromsize"]  # 5% threshold

welearn_datastack/nodes_workflow/WikipediaUpdater/wikipedia_updater.py

@@ -64,7 +64,7 @@ def main() -> None:
                         title=Step.URL_RETRIEVED.value,
                     )
                 )
-        except ValueError as e:
+        except (ValueError, KeyError) as e:
             logger.error("Error while comparing document '%s': %s", wld.title, e)
             continue