update (refering Wikidata Random item)

Author: dignityc

config.yaml

@@ -1,5 +1,6 @@
 database:
-  name: 'wikidata_claims_refs_parsed.db'
+  name: '/hpc/scratch/prj/inf_wqp/wikidata_claims_refs_parsed.db'
+  result_db_for_API: '/hpc/scratch/prj/inf_wqp/reference_checked.db'
 
 parsing:
   reset_database: False #This is a developer mode to clean-up DB to test soemthing

eventHandler.py

@@ -9,6 +9,7 @@
 import datetime
 import time
 import uuid
+import yaml
 
 def save_to_sqlite(result_df, db_path, table_name):
     result_df = result_df.astype(str)
@@ -34,7 +35,7 @@ def save_to_sqlite(result_df, db_path, table_name):
     finally:
         conn.close()
 
-def initialize_database(db_path='reference_checked.db'):
+def initialize_database(db_path):
     conn = sqlite3.connect(db_path)
     cursor = conn.cursor()
     cursor.execute("""
@@ -77,6 +78,7 @@ def initialize_database(db_path='reference_checked.db'):
     CREATE TABLE IF NOT EXISTS aggregated_results (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         triple TEXT,
+        property_id TEXT,
         url TEXT,
         Results TEXT,
         qid TEXT,
@@ -114,11 +116,21 @@ def initialize_database(db_path='reference_checked.db'):
 def get_random_qids(num_qids=5, max_retries=3, delay=5):
     sparql = SPARQLWrapper("https://query.wikidata.org/sparql")
     sparql.setQuery("""
-    SELECT ?item
-    WHERE {
-      ?item wdt:P31 wd:Q5 .  # Instance of human
-    }
-    LIMIT 1000  # Fetch more items than needed
+        SELECT ?item {
+        SERVICE bd:sample {
+            ?item wikibase:sitelinks [].
+            bd:serviceParam bd:sample.limit "100".
+        }
+            MINUS {?item wdt:P31/wdt:P279* wd:Q4167836.}
+            MINUS {?item wdt:P31/wdt:P279* wd:Q4167410.}
+            MINUS {?item wdt:P31 wd:Q13406463.}
+            MINUS {?item wdt:P31/wdt:P279* wd:Q11266439.}
+            MINUS {?item wdt:P31 wd:Q17633526.}
+            MINUS {?item wdt:P31 wd:Q13442814.}
+            MINUS {?item wdt:P3083 [].}
+            MINUS {?item wdt:P1566 [].}
+            MINUS {?item wdt:P442 [].}
+        }
     """)
     sparql.setReturnFormat(JSON)
 
@@ -237,8 +249,15 @@ def prove_process(db_path, batch_qids, algo_version):
         if 'conn' in locals():
             conn.close()
 
-def main(db_path, batch_qids, algo_version, Test_mode):
-    reset_database = False  # Developer mode to test, it initialize db for getting clean db
+def load_config(config_path: str):
+    with open(config_path, 'r') as file:
+        return yaml.safe_load(file)
+
+
+def main(batch_qids, algo_version):
+    reset_database = True  # Developer mode to test, it initialize db for getting clean db
+    config = load_config('config.yaml')
+    db_path = config['database']['result_db_for_API']
     if reset_database and os.path.exists(db_path):
         os.remove(db_path)
         print(f"Database file {db_path} has been deleted.")
@@ -254,12 +273,8 @@ def main(db_path, batch_qids, algo_version, Test_mode):
             time.sleep(30)  
 
 
-
-
 if __name__ == "__main__":
-    db_path = 'reference_checked.db'
-    batch_qids = 3
-    algo_version = '1.0.2'
-    Test_mode = True #using different temp .db to test code.
-    main(db_path, batch_qids, algo_version, Test_mode)
+    batch_qids = 5
+    algo_version = '1.0.3'
+    main(batch_qids, algo_version)
     # nohup python3 eventHandler.py > output.log 2>&1 &

functions.py

@@ -1,8 +1,14 @@
 import sqlite3
 import pandas as pd
 from datetime import datetime
+import yaml
+
 #Params.
-db_path = 'reference_checked.db'
+def load_config(config_path: str):
+    with open(config_path, 'r') as file:
+        return yaml.safe_load(file)
+config = load_config('config.yaml')
+db_path = config['database']['result_db_for_API']
 
 #Table summary
 def get_all_tables_and_schemas(db_path):
@@ -81,10 +87,10 @@ def comprehensive_results(target_id):
         first_item = response[0]
         if isinstance(first_item, dict):
             if 'error' in first_item:
-                return {'health_value': 'processing error', 
-                        'NOT ENOUGH INFO': 'processing error',
-                        'SUPPORTS': 'processing error',
-                        'REFUTES': 'processing error'
+                return {'health_value': 'Not processed yet', 
+                        'NOT ENOUGH INFO': 'Not processed yet',
+                        'SUPPORTS': 'Not processed yet',
+                        'REFUTES': 'Not processed yet'
                         }
             elif 'status' in first_item and first_item['status'] == 'error':
                 return {'health_value': 'processing error', 

html_fetching.py

@@ -308,15 +308,17 @@ def fetch_and_update_html(self):
                 chrome_options.add_argument("--disable-dev-shm-usage")
                 chrome_options.add_argument("--disable-plugins")
                 chrome_options.add_argument("--disable-pdf-viewer")
-                service = Service('/usr/bin/chromedriver')
-                driver = webdriver.Chrome(service=service, options=chrome_options)
+                chrome_options.add_argument("--disable-extensions")
                 chrome_options.add_experimental_option("prefs", {
-                    "download.default_directory": "/CodeArchive/payloads/",
+                    "download.default_directory": "/dev/null",
                     "download.prompt_for_download": False,
                     "download.directory_upgrade": True,
                     "plugins.always_open_pdf_externally": False,
-                    "safebrowsing.enabled": True
+                    "plugins.plugins_list": [{"enabled": False, "name": "Chrome PDF Viewer"}],
+                    "download_restrictions": 3
                 })
+                service = Service('/usr/bin/chromedriver')
+                driver = webdriver.Chrome(service=service, options=chrome_options)
                 driver.set_page_load_timeout(10) 
                 for i, (url,) in enumerate(urls_to_fetch):
                     if i > 0 and i % batch_size == 0:
@@ -813,5 +815,5 @@ def main(qids: List[str]):
 
 
 if __name__ == "__main__":
-    qids =['Q4616']
+    qids =['Q42']
     main(qids)

reference_checking.py

@@ -15,11 +15,11 @@
 import torch, gc
 
 class ReferenceChecker:
-    def __init__(self, db_name: str = 'wikidata_claims_refs_parsed.db', config_path: str = 'config.yaml'):
-        self.db_name = db_name
+    def __init__(self, config_path: str = 'config.yaml'):
+        self.config = self.load_config(config_path)
+        self.db_name = self.config['database']['name']
         self.conn = None
         self.cursor = None
-        self.config = self.load_config(config_path)
         self.verb_module = VerbModule()
         nltk.download('punkt', quiet=True)
 
@@ -376,8 +376,9 @@ def TableMaking(self, verbalised_claims_df_final, result):
             aResult = pd.concat([aResult, pd.DataFrame(row["evidence_TE_labels_all_TOP_N"], columns=['TextEntailment'])], axis=1)
             aResult = pd.concat([aResult, pd.DataFrame(np.max(row["evidence_TE_prob_all_TOP_N"], axis=1), columns=['Entailment_score'])], axis=1)
             aResult = aResult.reindex(columns=['sentence', 'TextEntailment', 'Entailment_score','Relevance_score'])
-            aBox = pd.DataFrame({'triple': [row["triple"]], 'url': row['url'],'Results': [aResult]})
+            aBox = pd.DataFrame({'triple': [row["triple"]], 'property_id' : row['property_id'], 'url': row['url'],'Results': [aResult]})
             all_result = pd.concat([all_result,aBox], axis=0)
+            
 
         def dataframe_to_html(all_result):
             html = '<html><head><style>table {border-collapse: collapse; width: 100%;} th, td {border: 1px solid black; padding: 8px; text-align: left;} th {background-color: #f2f2f2;}</style></head><body>'
@@ -439,7 +440,6 @@ def main(qids: List[str]):
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-
-    qids =['Q4934']
+    qids =['Q42']
     original_results, aggregated_results, reformedHTML_results = main(qids)
 

wikidata_reader.py

@@ -360,5 +360,5 @@ def main(qids: List[str], reset: bool = False):
 
 if __name__ == "__main__":
     nltk.download('punkt', quiet=True)
-    qids =['Q4616']
+    qids =['Q42']
     main(qids)