Predictor fixes (#70)

* Add textual dependency and update predictor configurations

* Implement SVO extraction and enhance predictor configurations

* Remove outdated TPP data file and add README files for Personachat and Human Conversation datasets.  Start of re-organization to support localization of ConvAssist

* Add argument picking to VSCode launch configuration

* Remove unused startwords property from GeneralWordPredictor

* Add startwords property to SmoothedNgramPredictor for resource path management

* Remove SVOPredictor class and associated methods from smoothed_ngram_predictor

* Update default values for startsents and startwords to filenames in Predictor class

* Improve HNSWLIB index initialization and add progress tracking with tqdm

* Replace SVOPredictor with SmoothedNgramPredictor in predictor registry

* Updated Poetry Lock file with latest dependency updates.

* Add utility for converting parquet to CSV and enhance logging in PredictorActivator

Author: michaelbeale-IL

.vscode/launch.json

@@ -22,7 +22,8 @@
             "justMyCode": false,
             "env": {
                 "PYTHONPATH": "${workspaceFolder}"
-            }
+            },
+            "args": "${command:pickArgs}"
         },
         {
             "name": "ConvAssist",
@@ -41,7 +42,9 @@
             "name": "Python Debugger: Python File",
             "type": "debugpy",
             "request": "launch",
-            "program": "${file}"
+            "program": "${file}",
+            "justMyCode": false,
+
         },
         {
             "name": "Python: Debug Tests",

…rty_resources/human-conversation/LICENSE …alizations/en/human-conversation/LICENSE3rd_party_resources/human-conversation/LICENSE renamed to 3rd_party_resources/localizations/en/human-conversation/LICENSE 3rd_party_resources/human-conversation/LICENSE renamed to 3rd_party_resources/localizations/en/human-conversation/LICENSE

@@ -1,4 +1,5 @@
 import argparse
+import os
 from typing import List
 
 from tqdm import tqdm
@@ -51,7 +52,6 @@ def configure():
     #flag to clean the database
 
     parser.add_argument(
-        "-c",
         "--clean",
         action="store_true",
         help="Whether to clean the database"
@@ -89,14 +89,20 @@ def main(argv=None):
         if response.lower() != 'y':
             print("Exiting...")
             return
+        else:
+            print("Cleaning database...")
+            os.remove(args.database)
 
     phrases = []
 
     with open(args.input_file) as f:
         for line in f:
             phrases.append(line.strip())
 
+
     with NGramUtil(args.database, args.cardinality, args.lowercase, args.normalize) as ngramutil:
+        ngramutil.create_update_ngram_tables()
+        
         threads = []
         for i in range(args.cardinality):
             p = Thread(target=insertngrambycardinality, args=(ngramutil, phrases, i + 1))

…y_resources/human-conversation/README.md …izations/en/human-conversation/README.md3rd_party_resources/human-conversation/README.md renamed to 3rd_party_resources/localizations/en/human-conversation/README.md 3rd_party_resources/human-conversation/README.md renamed to 3rd_party_resources/localizations/en/human-conversation/README.md

@@ -0,0 +1,5 @@
+import pandas as pd
+
+df = pd.read_parquet('train-00000-of-00001-aaf72b9960b78228.parquet')
+df.to_csv('data.csv', index=False)
+

3rd_party_resources/personachat/LICENSE …ces/localizations/en/personachat/LICENSE3rd_party_resources/personachat/LICENSE renamed to 3rd_party_resources/localizations/en/personachat/LICENSE 3rd_party_resources/personachat/LICENSE renamed to 3rd_party_resources/localizations/en/personachat/LICENSE

@@ -61,7 +61,7 @@ def initialize(
         self.set_predictors()
 
         self.predictor_activator = PredictorActivator(
-            self.config, self.predictor_registry, self.context_tracker, self.logger
+            self.name, self.config, self.predictor_registry, self.context_tracker #, self.logger
         )
         self.predictor_activator.combination_policy = "meritocracy"
 

3rd_party_resources/personachat/README.md …s/localizations/en/personachat/README.md3rd_party_resources/personachat/README.md renamed to 3rd_party_resources/localizations/en/personachat/README.md 3rd_party_resources/personachat/README.md renamed to 3rd_party_resources/localizations/en/personachat/README.md

@@ -58,8 +58,8 @@ def __init__(
         self._sentence_transformer_model: str = ""  # Path
         self._sentences_db: str = ""  # Path
         self._spellingdatabase: str = ""  # Path
-        self._startsents: str = ""  # Path
-        self._startwords: str = ""  # Path
+        self._startsents: str = "start_sentences.txt"  # Filename
+        self._startwords: str = "start_words.txt"  # Filename
         self._static_resources_path: str = ""
         self._stopwords: str = ""  # Path
         self._test_generalsentenceprediction: bool = False