Merge pull request #450 from dataforgoodfr/fix-country-keywords-update

fix country filtering on keywords

Author: gmguarino

quotaclimat/data_processing/mediatree/detect_keywords.py

@@ -199,7 +199,7 @@ def get_themes_keywords_duration(plaintext: str, subtitle_duration: List[str], s
     plaitext_without_stopwords = remove_stopwords(plaintext=plaintext, stopwords=stop_words, country=country)
     logging.debug(f"display datetime start {start}")
 
-    logging.debug(f"Keeping only {country.language} keywords...")
+    logging.info(f"Keeping only {country.language} keywords...")
     try:
         for theme, keywords_dict in THEME_KEYWORDS.items():
             logging.debug(f"searching {theme} for {keywords_dict}")

quotaclimat/data_processing/mediatree/update_pg_keywords.py

@@ -71,6 +71,8 @@ def update_keywords(session: Session, batch_size: int = 50000, start_date : str
                         stop_word_keyword_only = False, \
                         biodiversity_only = False,
                         country=FRANCE) -> list:
+    
+    logging.info(f"Updating keywords for Country : {country.name}")
 
     filter_days_stop_word = int(os.environ.get("FILTER_DAYS_STOP_WORD", 30))
     logging.info(f"FILTER_DAYS_STOP_WORD is used to get only last {filter_days_stop_word} days of new stop words - to improve update speed")
@@ -135,7 +137,7 @@ def update_keywords(session: Session, batch_size: int = 50000, start_date : str
                     ,number_of_biodiversite_causes_no_hrfp \
                     ,number_of_biodiversite_consequences_no_hrfp \
                     ,number_of_biodiversite_solutions_no_hrfp \
-                    ,country_name = get_themes_keywords_duration(plaintext, srt, start, stop_words=stop_words)
+                    ,country_name = get_themes_keywords_duration(plaintext, srt, start, stop_words=stop_words, country=country)
 
 
                     if(number_of_keywords != new_number_of_keywords or