Swapped out MongoDB in favor of SQLite

.gitignore

@@ -3,4 +3,8 @@
 __pycache__
 
 # Exclude pycharm metadata
- .idea/
+ .idea/
+
+model/
+jarvis_virtualenv/
+*.log

data/.gitignore

@@ -0,0 +1 @@
+*.db

requirements.txt

@@ -21,13 +21,13 @@ lxml==4.6.3
 mock==3.0.5
 more-itertools==7.0.0
 nltk==3.4.5
-numpy==1.16.4
 patch==1.16
 platformdirs==2.4.0
 playsound==1.2.2
 psutil==5.6.6
 PTable==0.9.2
 PyAudio==0.2.11
+pybind11==2.9.1
 pycairo==1.20.1
 PyGObject==3.42.0
 pymongo==3.10.1
@@ -49,3 +49,4 @@ wikipedia==1.4.0
 wolframalpha==3.0.1
 word2number==1.1
 xmltodict==0.12.0
+sounddevice==0.4.4

run_jarvis.sh

@@ -1,15 +1,6 @@
 #!/usr/bin/env bash
-# --------------------------------
-# Start MongoDB service
-# --------------------------------
-sudo systemctl start mongodb
 
 # --------------------------------
 # Start Jarvis service with virtualenv
 # --------------------------------
 ./jarvis_virtualenv/bin/python ./src/jarvis/start.py
-
-# --------------------------------
-# Stop MongoDB service
-# --------------------------------
-sudo systemctl stop mongodb

run_tests.sh

@@ -5,20 +5,10 @@
 export PYTHONPATH="${PYTHONPATH}:./src/jarvis"
 source jarvis_virtualenv/bin/activate
 
-# --------------------------------
-# Start MongoDB service
-# --------------------------------
-sudo systemctl start mongodb
-
 # --------------------------------
 # Run unittests
 # --------------------------------
 python -m unittest discover -s ./src -p "*tests.py"
 exit_code=($?)
 
-# --------------------------------
-# Stop MongoDB service
-# --------------------------------
-sudo systemctl stop mongodb
-
 exit $exit_code

setup.sh

@@ -25,18 +25,16 @@ fi
 #-----------------------------------
 # System dependencies installation
 #-----------------------------------
-sudo apt-get update && /
-sudo apt-get install build-essential && /
-sudo apt-get install python3-dev && /
-sudo apt-get install python3-setuptools && /
-sudo apt-get install python3-pip && /
-sudo apt-get install python3-venv && /
-sudo apt-get install portaudio19-dev python3-pyaudio python3-pyaudio && /
-sudo apt-get install libasound2-plugins libsox-fmt-all libsox-dev libxml2-dev libxslt-dev sox ffmpeg && /
-sudo apt-get install espeak && /
-sudo apt-get install libcairo2-dev libgirepository1.0-dev gir1.2-gtk-3.0  && /
-sudo apt install mongodb && /
-sudo apt-get install gnupg
+sudo apt-get update && \
+sudo apt-get install build-essential \
+    python3-dev python3-setuptools python3-pip python3-venv \
+    portaudio19-dev python3-pyaudio python3-pyaudio \
+    python3-numpy \
+    libasound2-plugins libsox-fmt-all libsox-dev libxml2-dev libxslt-dev sox ffmpeg \
+    libgstreamer1.0-dev libgstreamer1.0-0 \
+    espeak \
+    libcairo2-dev libgirepository1.0-dev gir1.2-gtk-3.0  \
+    gnupg
 
 # Reload local package database
 sudo apt-get update
@@ -73,7 +71,7 @@ echo "${green} ${activated_python_version} activated!${reset}"
 # Install python requirements
 pip3 install --upgrade cython
 pip3 install wheel
-python setup.py bdist_wheel
+python3 setup.py bdist_wheel
 pip3 install -r $JARVIS_DIR/requirements.txt
 pip3 install -U scikit-learn
 

src/jarvis/jarvis/__init__.py

@@ -24,10 +24,12 @@
 
 from jarvis import settings
 from jarvis.settings import ROOT_LOG_CONF
-from jarvis.utils.mongoDB import db
-from jarvis.utils.startup import configure_MongoDB
+from jarvis.utils.startup import configure_defaults
 from jarvis.enumerations import InputMode
 import jarvis.engines as engines
+from jarvis.utils.settings_database import settingsDB
+from jarvis.utils.skills_registry import skills_registry
+from jarvis.skills.registry import skill_objects, CONTROL_SKILLS, ENABLED_BASIC_SKILLS
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Create a Console & Rotating file logger
@@ -41,16 +43,20 @@
     f.close()
 
 # ----------------------------------------------------------------------------------------------------------------------
-# Configuare MongoDB, load skills and settings
+# Configuare defaults, load skills and settings
 # ----------------------------------------------------------------------------------------------------------------------
-configure_MongoDB(db, settings)
+configure_defaults(settings)
+skills_registry.skill_objects = skill_objects
+skills_registry.basic_skills = ENABLED_BASIC_SKILLS
+skills_registry.control_skills = CONTROL_SKILLS
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Get assistant settings
 # ----------------------------------------------------------------------------------------------------------------------
-input_mode = db.get_documents(collection='general_settings')[0]['input_mode']
-response_in_speech = db.get_documents(collection='general_settings')[0]['response_in_speech']
-assistant_name = db.get_documents(collection='general_settings')[0]['assistant_name']
+generalSettings = settingsDB.getGeneralSettings()
+input_mode = generalSettings.input_mode
+response_in_speech = generalSettings.response_in_speech
+assistant_name = generalSettings.assistant_name
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Create assistant input and output engine instances

src/jarvis/jarvis/core/console.py

@@ -26,9 +26,9 @@
 import logging
 
 from jarvis import settings
-from jarvis.utils.mongoDB import db
+from jarvis.utils.settings_database import settingsDB
 from jarvis.utils.console import jarvis_logo, start_text, OutputStyler, headerize
-from jarvis.enumerations import MongoCollections, InputMode
+from jarvis.enumerations import InputMode
 
 
 class ConsoleManager:
@@ -90,15 +90,13 @@ def console_output(self, text='', debug_log=None, info_log=None, warn_log=None,
             # ----------------------------------------------------------------------------------------------------------
             # General info sector
             # ----------------------------------------------------------------------------------------------------------
-            settings_documents = db.get_documents(collection=MongoCollections.GENERAL_SETTINGS.value)
-            if settings_documents:
-                settings_ = settings_documents[0]
-                print(OutputStyler.HEADER + headerize('GENERAL INFO') + OutputStyler.ENDC)
-                enabled = OutputStyler.GREEN + 'ENABLED' + OutputStyler.ENDC if settings_['response_in_speech'] else OutputStyler.WARNING + 'NOT ENABLED' + OutputStyler.ENDC
-                print(OutputStyler.BOLD + 'RESPONSE IN SPEECH: ' + enabled)
-                print(OutputStyler.BOLD + 'INPUT MODE: ' + OutputStyler.GREEN + '{0}'.format(settings_['input_mode'].upper() + OutputStyler.ENDC) + OutputStyler.ENDC)
-                if settings_['input_mode'] == InputMode.VOICE.value:
-                    print(OutputStyler.BOLD + 'NOTE: ' + OutputStyler.GREEN + "Include " + "'{0}'".format(settings_['assistant_name'].upper()) + " in you command to enable assistant" + OutputStyler.ENDC + OutputStyler.ENDC)
+            settings_ = settingsDB.getGeneralSettings()
+            print(OutputStyler.HEADER + headerize('GENERAL INFO') + OutputStyler.ENDC)
+            enabled = OutputStyler.GREEN + 'ENABLED' + OutputStyler.ENDC if settings_.response_in_speech else OutputStyler.WARNING + 'NOT ENABLED' + OutputStyler.ENDC
+            print(OutputStyler.BOLD + 'RESPONSE IN SPEECH: ' + enabled)
+            print(OutputStyler.BOLD + 'INPUT MODE: ' + OutputStyler.GREEN + '{0}'.format(settings_.input_mode.upper() + OutputStyler.ENDC) + OutputStyler.ENDC)
+            if settings_.input_mode == InputMode.VOICE.value:
+                print(OutputStyler.BOLD + 'NOTE: ' + OutputStyler.GREEN + "Include " + "'{0}'".format(settings_.assistant_name.upper()) + " in you command to enable assistant" + OutputStyler.ENDC + OutputStyler.ENDC)
 
             # ----------------------------------------------------------------------------------------------------------
             # System info sector

src/jarvis/jarvis/core/processor.py

@@ -26,10 +26,10 @@
 from sklearn.metrics.pairwise import cosine_similarity
 
 from jarvis.skills.analyzer import SkillAnalyzer
-from jarvis.skills.registry import skill_objects
+from jarvis.utils.skills_registry import skills_registry
 from jarvis.core.nlp import ResponseCreator
 from jarvis.skills.collection.activation import ActivationSkills
-from jarvis.utils.mongoDB import db
+from jarvis.utils.history_database import historyDB, History
 from jarvis.skills.collection.wolframalpha import WolframSkills
 
 
@@ -53,7 +53,7 @@ def run(self):
         - STEP 2: Matches the input with a skill
         - STEP 3: Create a response
         - STEP 4: Execute matched skill
-        - STEP 5: Insert user transcript and response in history collection (in MongoDB)
+        - STEP 5: Insert user transcript and response in history collection
 
         """
 
@@ -100,13 +100,10 @@ def run(self):
         # --------------------------------------------------------------------------------------------------------------
         # Add new record to history
         # --------------------------------------------------------------------------------------------------------------
-
-        record = {'user_transcript': transcript,
-                  'response': response if response else '--',
-                  'executed_skill': skill_to_execute if skill_to_execute else '--'
-                  }
-
-        db.insert_many_documents('history', [record])
+        historyDB.addHistory(History(
+            user_transcript=transcript,
+            response=response,
+            executed_skill='--' if not skill_to_execute else skill_to_execute['skill']['name']))
 
     def _execute_skill(self, skill):
         if skill:
@@ -115,7 +112,7 @@ def _execute_skill(self, skill):
             try:
                 ActivationSkills.enable_assistant()
                 skill_func_name = skill.get('skill').get('func')
-                skill_func = skill_objects[skill_func_name]
+                skill_func = skills_registry.skill_objects[skill_func_name]
                 skill_func(**skill)
             except Exception as e:
                 self.console_manager.console_output(error_log="Failed to execute skill {0} with message: {1}"

src/jarvis/jarvis/enumerations.py

@@ -26,9 +26,3 @@
 class InputMode(Enum):
     VOICE = 'voice'
     TEXT = 'text'
-
-
-class MongoCollections(Enum):
-    GENERAL_SETTINGS = 'general_settings'
-    CONTROL_SKILLS = 'control_skills'
-    ENABLED_BASIC_SKILLS = 'enabled_basic_skills'

src/jarvis/jarvis/settings.py

@@ -65,6 +65,12 @@
 
 """
 
+DATABASE_SETTINGS = {
+    'base_dir': './data/',
+    'history_filename': 'history.db',
+    'settings_filename': 'settings.db',
+    'skills_filename': 'skills.db'
+}
 
 DEFAULT_GENERAL_SETTINGS = {
     'assistant_name': 'Jarvis',

src/jarvis/jarvis/skills/analyzer.py

@@ -21,8 +21,7 @@
 # SOFTWARE.
 import logging
 from jarvis.utils.mapping import math_symbols_mapping
-from jarvis.utils.mongoDB import db
-
+from jarvis.utils.skills_registry import skills_registry
 
 class SkillAnalyzer:
     def __init__(self, weight_measure, similarity_measure, args, sensitivity):
@@ -35,9 +34,7 @@ def __init__(self, weight_measure, similarity_measure, args, sensitivity):
 
     @property
     def skills(self):
-        return db.get_documents(collection='control_skills')\
-               + db.get_documents(collection='enabled_basic_skills')\
-               + db.get_documents(collection='learned_skills')
+        return skills_registry.basic_skills + skills_registry.control_skills + skills_registry.learned_skills
 
     @property
     def tags(self):

src/jarvis/jarvis/skills/collection/activation.py

@@ -26,8 +26,8 @@
 
 from jarvis.skills.skill import AssistantSkill
 from jarvis.utils.startup import play_activation_sound
-from jarvis.utils.mongoDB import db
-from jarvis.enumerations import InputMode, MongoCollections
+from jarvis.utils.settings_database import settingsDB
+from jarvis.enumerations import InputMode
 
 
 class ActivationSkills(AssistantSkill):
@@ -38,8 +38,8 @@ def enable_assistant(cls, **kwargs):
         Plays activation sound and creates the assistant response according to the day hour.
         """
 
-        input_mode = db.get_documents(collection=MongoCollections.GENERAL_SETTINGS.value)[0]['input_mode']
-        if input_mode == InputMode.VOICE.value:
+        generalSettings = settingsDB.getGeneralSettings()
+        if generalSettings.input_mode == InputMode.VOICE.value:
             play_activation_sound()
 
     @classmethod

src/jarvis/jarvis/skills/collection/configuration.py

@@ -24,14 +24,15 @@
 
 from jarvis.skills.skill import AssistantSkill
 from jarvis import settings
-from jarvis.utils.mongoDB import db
-from jarvis.enumerations import InputMode, MongoCollections
+from jarvis.utils.settings_database import settingsDB
+from jarvis.enumerations import InputMode
 from jarvis.utils import console
 from jarvis.utils import input
 
-input_mode = db.get_documents(collection='general_settings')[0]['input_mode']
-response_in_speech = db.get_documents(collection='general_settings')[0]['response_in_speech']
-assistant_name = db.get_documents(collection='general_settings')[0]['assistant_name']
+generalSettings = settingsDB.getGeneralSettings()
+input_mode = generalSettings.input_mode
+response_in_speech = generalSettings.response_in_speech
+assistant_name = generalSettings.assistant_name
 
 
 class ConfigurationSkills(AssistantSkill):
@@ -70,7 +71,10 @@ def configure_assistant(cls, **kwargs):
         cls.response('Do you want to save new settings? ', refresh_console=False)
         save = input.check_input_to_continue()
         if save:
-            db.update_collection(collection=MongoCollections.GENERAL_SETTINGS.value, documents=[new_settings])
+            generalSettings.input_mode = new_input_mode
+            generalSettings.assistant_name = new_assistant_name
+            generalSettings.response_in_speech = new_response_in_speech
+            settingsDB.updateGeneralSettings(generalSettings)
 
             import jarvis
             importlib.reload(jarvis)

src/jarvis/jarvis/skills/collection/history.py

@@ -23,7 +23,7 @@
 import re
 
 from jarvis.skills.skill import AssistantSkill
-from jarvis.utils.mongoDB import db
+from jarvis.utils.history_database import historyDB
 
 header = """
 -----------------------------------------------------------------------------------------------
@@ -52,19 +52,18 @@ def show_history_log(cls, voice_transcript, skill):
 
         limit = cls._extract_history_limit(voice_transcript, skill)
         limit = limit if limit else cls.default_limit
-        documents = db.get_documents(collection='history', limit=limit)
-        response = cls._create_response(documents)
+        records = historyDB.getHistory(limit=limit)
+        response = cls._create_response(records)
         cls.console(response)
 
     @classmethod
-    def _create_response(cls, documents):
+    def _create_response(cls, records):
         response = ''
         try:
-            for document in documents:
-                response += response_base.format(document.get('user_transcript', '--'),
-                                                 document.get('response', '--'),
-                                                 document.get('executed_skill').get('skill').get('name') if
-                                                 document.get('executed_skill') else '--'
+            for record in records:
+                response += response_base.format(record.user_transcript,
+                                                 record.response,
+                                                 record.executed_skill
                                                  )
         except Exception as e:
             cls.console(error_log=e)

src/jarvis/jarvis/skills/collection/info.py

@@ -21,7 +21,7 @@
 # SOFTWARE.
 
 from jarvis.skills.skill import AssistantSkill
-from jarvis.utils.mongoDB import db
+from jarvis.utils.skills_registry import skills_registry
 from jarvis.utils.console import headerize
 
 basic_skills_format = """
@@ -94,7 +94,8 @@ def _create_skill_response(cls, response):
         # --------------------------------------------------------------------------------------------------------------
         # For existing skills (basic skills)
         # --------------------------------------------------------------------------------------------------------------
-        basic_skills = db.get_documents(collection='enabled_basic_skills')
+        # basic_skills = db.get_documents(collection='enabled_basic_skills')
+        basic_skills = skills_registry.basic_skills
         response = response + basic_skills_format
         for skill_id, skill in enumerate(basic_skills, start=1):
             response = response + basic_skills_body_format.format(skill_id,
@@ -106,7 +107,7 @@ def _create_skill_response(cls, response):
         # --------------------------------------------------------------------------------------------------------------
         # For learned skills (created from 'learn' skill)
         # --------------------------------------------------------------------------------------------------------------
-        skills = db.get_documents(collection='learned_skills')
+        skills = skills_registry.learned_skills
         response = response + learned_skills_format
         for skill_id, skill in enumerate(skills, start=1):
             response = response + learned_skills_body_format.format(skill_id,