fix text preprocessing for inference model (#40)

* fix text preprocessing for inference model

* add filter_window_sizes override for non-cnn models

Author: xwli-chelsea

src/detext/train/train_helper.py

@@ -43,7 +43,7 @@ def get_query(hparams):
             hparams.CLS, hparams.SEP, hparams.PAD,
             hparams.max_len,
             hparams.min_len,
-            cnn_filter_window_size=max(hparams.filter_window_sizes)
+            cnn_filter_window_size=max(hparams.filter_window_sizes) if hparams.ftr_ext == 'cnn' else 0
         )
     return query, query_placeholder
 
@@ -75,7 +75,7 @@ def get_doc_fields(hparams):
                 hparams.CLS, hparams.SEP, hparams.PAD,
                 hparams.max_len,
                 hparams.min_len,
-                cnn_filter_window_size=max(hparams.filter_window_sizes)
+                cnn_filter_window_size=max(hparams.filter_window_sizes) if hparams.ftr_ext == 'cnn' else 0
             )
             one_doc_field = tf.expand_dims(one_doc_field, axis=0)
             doc_fields.append(one_doc_field)
@@ -118,7 +118,7 @@ def get_usr_fields(hparams):
                 hparams.CLS, hparams.SEP, hparams.PAD,
                 hparams.max_len,
                 hparams.min_len,
-                cnn_filter_window_size=max(hparams.filter_window_sizes)
+                cnn_filter_window_size=max(hparams.filter_window_sizes) if hparams.ftr_ext == 'cnn' else 0
             )
             usr_fields.append(one_usr_field)
     return usr_fields, usr_text_placeholders

src/detext/utils/misc_utils.py

@@ -99,6 +99,10 @@ def extend_hparams(hparams):
     tok2regex_pattern = {'plain': None, 'punct': r'(\pP)'}
     hparams.regex_replace_pattern = tok2regex_pattern[hparams.tokenization]
 
+    # if not using cnn models, then disable cnn parameters
+    if hparams.ftr_ext != 'cnn':
+        hparams.filter_window_sizes = [0]
+
     assert hparams.pmetric is not None, "Please set your primary evaluation metric using --pmetric option"
     assert hparams.pmetric != 'confusion_matrix', 'confusion_matrix cannot be used as primary evaluation metric.'