[Hack session] Create protected s3 endpoint for secured assets (#311)

* Create s3 data script to fetch protected resources

* Change safety.txt load path

* Add fallback for s3 read with boto3 download using access tokens

* Fix bug in s3 download_file option

* Download protected resources in download script

* Add protected resources path to gitingore

* Update safety words unit test

* Add boto3 to requirements

* Format schema files and add OTHER and ALL to location and selector

* Fix NSP initialization in safety unit test

* Fix safety words loading bug

* Update unit test to check safety

* Fix merge conflict

* Remove safety check from datasets

* Correct gitignore paths

* Use pkg_resources to fetch resource location

Author: Rebecca-Qian

.gitignore

@@ -16,6 +16,7 @@ __pycache__
 *.sql
 !memory_schema.sql
 *~
+base_agent/documents/internal/*
 bin/*
 writeups/*.pdf
 writeups/*.aux

base_agent/droidlet_nsp_model_wrapper.py

@@ -69,7 +69,10 @@ def read_datasets(self, opts):
         """
         # Extract the set of safety words from safety file
         self.safety_words = set()
-        safety_words_path = opts.ground_truth_data_dir + "safety.txt"
+        safety_words_path = "{}/{}".format(
+            pkg_resources.resource_filename("base_agent.documents", "internal"),
+            "safety.txt",
+        )
         if os.path.isfile(safety_words_path):
             """Read a set of safety words to prevent abuse."""
             with open(safety_words_path) as f:

craftassist/agent/craftassist_agent.py

@@ -325,10 +325,11 @@ def add_self_memory_node(self):
     logger.addHandler(sh)
     logging.info("LOG LEVEL: {}".format(logger.level))
 
-    # Check that models and datasets are up to date
+    # Check that models and datasets are up to date and download latest resources.
+    # Also fetches additional resources for internal users.
     if not opts.dev:
         rc = subprocess.call([opts.verify_hash_script_path, "craftassist"])
-
+    
     set_start_method("spawn", force=True)
 
     sa = CraftAssistAgent(opts)

craftassist/test/test_safety.py

@@ -8,7 +8,7 @@
 from base_craftassist_test_case import BaseCraftassistTestCase
 from fake_agent import MockOpt
 
-GROUND_TRUTH_DATA_DIR = os.path.join(os.path.dirname(__file__), "../agent/datasets/ground_truth/")
+TTAD_BERT_DATA_DIR = os.path.join(os.path.dirname(__file__), "../agent/datasets/annotated_data/")
 
 """This class tests safety checks using a preset list of blacklisted words.
 """
@@ -17,10 +17,11 @@
 class SafetyTest(BaseCraftassistTestCase):
     def setUp(self):
         opts = MockOpt()
-        opts.ground_truth_data_dir = GROUND_TRUTH_DATA_DIR
+        opts.nsp_data_dir = TTAD_BERT_DATA_DIR
         opts.no_ground_truth = False
         super().setUp(agent_opts=opts)
 
+
     def test_unsafe_word(self):
         is_safe = self.agent.dialogue_manager.semantic_parsing_model_wrapper.is_safe("bad Clinton")
         self.assertFalse(is_safe)

requirements.txt

@@ -1,5 +1,6 @@
 awscli>=1.18.0
 black>=18.9b0
+boto3>=1.17.32
 botocore>=1.15.0
 cython>=0.29.14
 decorator>=4.3.0

tools/data_scripts/default_checksums/datasets.txt

@@ -1 +1 @@
-95ad845207f5d67383535a1c97ef1e94e1f687c9
+95ad845207f5d67383535a1c97ef1e94e1f687c9

tools/data_scripts/fetch_internal_resources.py

@@ -0,0 +1,40 @@
+import boto3
+import subprocess
+import os
+import sys
+
+
+def fetch_safety_words():
+    """
+    Fetch secure s3 resources for internal users and production systems, eg. safety keyword list.
+
+    Currently needs to be called with sys arg passing in output directory.
+    """
+    try:
+        s3 = boto3.client('s3')
+        response = s3.head_bucket(Bucket='droidlet-internal')
+        if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
+            print("Authenticated user, fetching safety words list.")
+            path_to_root = os.path.realpath(sys.argv[1])
+            return subprocess.run(
+                "aws s3 cp s3://droidlet-internal/safety.txt {}".format(path_to_root), shell=True
+            )
+    except Exception as e:
+        print(e)
+        pass
+    # If awscli is not setup, eg. in one time use containers, read access tokens from environment   
+    try:
+        s3 = boto3.client('s3',
+            aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
+            aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY")
+        )
+        return s3.download_file(
+            'droidlet-internal',
+            'safety.txt',
+            os.path.realpath(sys.argv[1])
+        )
+    except Exception as e:
+        print(e)
+
+if __name__ == "__main__":
+    fetch_safety_words()

tools/data_scripts/try_download.sh

@@ -11,6 +11,9 @@ function pyabspath() {
 ROOTDIR=$(pyabspath $(dirname "$0")/../../)
 echo "Rootdir ${ROOTDIR}"
 
+# Optionally fetch secure resources for internal users and prod systems
+python ${ROOTDIR}/tools/data_scripts/fetch_internal_resources.py ${ROOTDIR}/base_agent/documents/internal/
+
 . ${ROOTDIR}/tools/data_scripts/checksum_fn.sh --source-only # import checksum function
 
 if [ -z $1 ]