bug fix and simplify

Author: jmargutt

routes/classify.py

@@ -45,15 +45,19 @@ async def classify_text(
         source=request.headers["source-name"],
         source_settings={k: v for k, v in request.headers.items()},
     )
-    logger.info(f"Classification schema source: {cs.source}")
-    logger.info(f"Classification schema settings: {cs.settings}")
 
     try:
         cs.load_from_cosmos()
     except CosmosResourceNotFoundError:
-        logger.info("Classification schema not found in CosmosDB, loading from source.")
-        create_classification_schema({k: v for k, v in request.headers.items()})
-    cs.load_from_cosmos()
+        logger.info(
+            "Classification schema not found in CosmosDB, loading from source and saving to CosmosDB."
+        )
+        cs = ClassificationSchema(
+            source=request.headers["source-name"],
+            source_settings=request.headers,
+        )
+        cs.load_from_source()
+        cs.save_to_cosmos()
 
     text = payload[request.headers["text-field"]]
     response = {}

utils/classification_schema.py

@@ -195,89 +195,6 @@ def load_from_source(self):
                             parent=choice[conditional_column3],
                         )
                     )
-
-            # load from attached cvs
-            # URL = f"https://kobo.ifrc.org/api/v2/assets/{self.settings['source-origin']}/files"
-            # file_url = requests.get(URL, headers=headers).json()["results"][0]["content"]
-            # file = requests.get(file_url, headers=headers, allow_redirects=True)
-            # open("schema.csv", "wb").write(file.content)
-            # try:
-            #     df_schema = pd.read_csv("schema.csv")
-            # except pd.errors.ParserError:
-            #     try:
-            #         df_schema = pd.read_csv("schema.csv", delimiter=";")
-            #     except pd.errors.ParserError:
-            #         raise HTTPException(
-            #             status_code=400,
-            #             detail="Could not parse the schema file. Please check the file format.",
-            #         )
-            # df_schema_lvl1 = df_schema.dropna(
-            #     subset=[self.settings["source-level1"] + "_name"]
-            # ).drop_duplicates(subset=[self.settings["source-level1"] + "_name"])
-            # for ix, level1_record in df_schema_lvl1.iterrows():
-            #     cs_records.append(
-            #         ClassificationSchemaRecord(
-            #             name=level1_record[self.settings["source-level1"] + "_name"],
-            #             label=level1_record[self.settings["source-level1"] + "_label"],
-            #             level=1,
-            #         )
-            #     )
-            # if self.settings["source-level2"]:
-            #     df_schema_lvl2 = df_schema.dropna(
-            #         subset=[
-            #             self.settings["source-level1"] + "_name",
-            #             self.settings["source-level2"] + "_name",
-            #         ]
-            #     ).drop_duplicates(
-            #         subset=[
-            #             self.settings["source-level1"] + "_name",
-            #             self.settings["source-level2"] + "_name",
-            #         ]
-            #     )
-            #     for ix, level2_record in df_schema_lvl2.iterrows():
-            #         cs_records.append(
-            #             ClassificationSchemaRecord(
-            #                 name=level2_record[
-            #                     self.settings["source-level2"] + "_name"
-            #                 ],
-            #                 label=level2_record[
-            #                     self.settings["source-level2"] + "_label"
-            #                 ],
-            #                 level=2,
-            #                 parent=level2_record[
-            #                     self.settings["source-level1"] + "_name"
-            #                 ],
-            #             )
-            #         )
-            # if self.settings["source-level3"]:
-            #     df_schema_lvl3 = df_schema.dropna(
-            #         subset=[
-            #             self.settings["source-level1"] + "_name",
-            #             self.settings["source-level2"] + "_name",
-            #             self.settings["source-level3"] + "_name",
-            #         ]
-            #     ).drop_duplicates(
-            #         subset=[
-            #             self.settings["source-level1"] + "_name",
-            #             self.settings["source-level2"] + "_name",
-            #             self.settings["source-level3"] + "_name",
-            #         ]
-            #     )
-            #     for ix, level3_record in df_schema_lvl3.iterrows():
-            #         cs_records.append(
-            #             ClassificationSchemaRecord(
-            #                 name=level3_record[
-            #                     self.settings["source-level3"] + "_name"
-            #                 ],
-            #                 label=level3_record[
-            #                     self.settings["source-level3"] + "_label"
-            #                 ],
-            #                 level=3,
-            #                 parent=level3_record[
-            #                     self.settings["source-level2"] + "_name"
-            #                 ],
-            #             )
-            #         )
         else:
             raise NotImplementedError(
                 f"Classification schema source {self.source} is not supported"