Merge pull request #299 from stanford-oval/wip/dev_3

Merge wip/dev_3

Author: Mehrad0711

genienlp/data_utils/almond_utils.py

@@ -106,7 +106,7 @@
     (ord(u"\U0002b820"), ord(u"\U0002ceaf")),
 ]
 
-CJK_ADDONS = [ord(u"\u3001")]
+CJK_ADDONS = [ord(u"\u3001"), ord('，'), ord('。'), ord('！'), ord('？')]
 
 
 def is_cjk_char(cp):

genienlp/models/base.py

@@ -570,9 +570,7 @@ def validate_e2e_dialogues(
                 dataset.update_state(state_update, dialogue_state)
 
                 #### save latest state
-                state_to_record = copy.deepcopy(dialogue_state)
-                state_to_record = {dataset.domain2api_name(k): v for k, v in state_to_record.items()}
-                e2e_dialogue_preds[dial_id]["turns"][str(turn_id)]["state"] = state_to_record
+                e2e_dialogue_preds[dial_id]["turns"][str(turn_id)]["state"] = dataset.state2span(dialogue_state)
                 ####
 
             elif train_target == 'api':
@@ -591,7 +589,7 @@ def validate_e2e_dialogues(
 
                 elif do_api_call == 'no':
                     # do nothing
-                    new_knowledge_text = 'null'
+                    pass
                 else:
                     logger.error(
                         f'API call should be either yes or no but got {do_api_call}. Seems model has not learnt the task yet. For now we assume it\'s a no'
@@ -804,9 +802,7 @@ def interact_e2e_dialogues(self, task, eval_dir=None, output_predictions_only=Fa
                     new_state_text = dataset.state2span(dialogue_state)
 
                     #### save latest state
-                    state_to_record = copy.deepcopy(dialogue_state)
-                    state_to_record = {dataset.domain2api_name(k): v for k, v in state_to_record.items()}
-                    e2e_dialogue_preds[dial_id]["turns"][str(turn_id)]["state"] = state_to_record
+                    e2e_dialogue_preds[dial_id]["turns"][str(turn_id)]["state"] = new_state_text
                     ####
 
                 elif train_target == 'api':

genienlp/util.py

@@ -55,6 +55,7 @@
 
 ENTITY_MATCH_REGEX = re.compile('^([A-Z].*)_[0-9]+$')
 QUOTED_MATCH_REGEX = re.compile(' " (.*?) " ')
+NUMBER_MATCH_REGEX = re.compile('^([0-9]+\.?[0-9]?)$')
 
 
 def find_span_type(program, begin_index, end_index):
@@ -107,8 +108,11 @@ def requote_program(program):
 
         elif not in_string:
             entity_match = ENTITY_MATCH_REGEX.match(token)
+            number_match = NUMBER_MATCH_REGEX.match(token)
             if entity_match is not None:
                 requoted.append(entity_match[1])
+            elif number_match is not None:
+                requoted.append('NUMBER')
             elif token != 'location:':
                 requoted.append(token)
 
@@ -363,7 +367,11 @@ def make_data_loader(
 
     all_features_filtered = []
     # remove examples longer than model input length
+
     for ex in all_features:
+        # Uncomment for debugging to print the long examples
+        # if dataset.batch_size_fn([ex]) > model_input_max_length:
+        #     print(ex)
         if batch_size_fn([ex]) < model_input_max_length:
             all_features_filtered.append(ex)
 

setup.py

@@ -72,6 +72,6 @@
         'dill~=0.3',
         'xgboost~=1.3',
         # for e2e dialogues:
-        'dialogues @ git+https://github.com/stanford-oval/dialogues@6695f29557f347238ef01b9a64127b7d4e4345cc',
+        'dialogues==0.0.3',
     ],
 )

tests/expected_results/bitod/e2e_dialogue_preds.json

@@ -2,27 +2,23 @@
   "ec909032-d5fd-4dee-8dd2-53a895a98824": {
     "turns": {
       "1": {
-        "state": {},
+        "state": "null",
         "api": "null",
         "actions": "Two herry null herry ( Hotels suchen ) herry herry herry Hey, kannst du mir bei der Hotelbuchung helfen? <endofknowledge> herry <endofknowledge> herry <endofknowledge> herry <endofknowledge> herry <endofknowledge> herry DA: herry <endofknowledge> herry <endofknowledge> herry <endofknowledge> herry <endofknowledge> herry DA: sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet sweet s",
         "response": [
           "Tüpfel ( Hotels suchen ) Anfrage Ort, Anfrage Bewertung Tüpfel Tüpfel Hey, können Sie mir helfen bei der Buchung? <endofknowledge> ? <endofknowledge> <endofknowledge> title <endofknowledge> <endofknowledge> <endofknowledge> <endofknowledge> title <endofknowledge> title <endofknowledge> title <endofknowledge> title <endofknowledge> title <endofknowledge> title <endofknowledge> title <endofknowledge> title <endofknowledge> title title title title title title title title title title title title title title title title title title title title title title title title ti"
         ]
       },
       "2": {
-        "state": {
-          "Hotels suchen": {}
-        },
+        "state": "( Hotels suchen )",
         "api": "null",
         "actions": "Sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat sweat ",
         "response": [
           "Tüpfellig ( Hotelsuche ) Anfrage Sterne Tüpfellig ( Hotelsuche ) Anfrage Ort, Anfrage Bewertung Tüpfellig Kein bestimmtes Ziel, aber ich möchte ein Hotel, das mindestens 4 Sterne ist. <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> . <endofknowledge> ."
         ]
       },
       "3": {
-        "state": {
-          "Hotels suchen": {}
-        },
+        "state": "( Hotels suchen )",
         "api": "null",
         "actions": "Sweat overnight null overnight ( hotels search ) location level_to \" keine Sorge \" , rating at_least \" 4 \" , stars level_to \" 5 \" sweat sweat sweat overnight ( hotels search ) location anfordern , quest rating AGENT_ Sweat ( hotels search ) request stars sweat stars sweat 5 stars. Response: sweat sweat <endofknowledge> sweat <endofknowledge> sweat <endofknowledge> sweat sweat swe",
         "response": [