Merge pull request #93 from Living-with-machines/dependabot/pip/pyyaml-5.4

v1.2.3

Author: kasra-hosseini

.github/workflows/dm_ci.yml

@@ -0,0 +1,22 @@
+name: Continuous integration
+
+on: [push, pull_request]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Install dependencies
+      run: |
+        $CONDA/bin/pip install -r requirements.txt
+    - name: Test with pytest
+      run: |
+        $CONDA/bin/conda install pytest
+        $CONDA/bin/pip install .
+        $CONDA/bin/pytest ./DeezyMatch/tests

DeezyMatch/rnn_networks.py

@@ -27,7 +27,7 @@
 import time
 #from tqdm import tqdm, tnrange
 from tqdm.autonotebook import tqdm
-from tqdm import tnrange
+from tqdm.notebook import tnrange
 
 from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
 

DeezyMatch/tests/test_import.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+import pytest
+
+def test_import():
+    import DeezyMatch

DeezyMatch/tests/test_pipeline.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+import pytest
+
+def test_train():
+    from DeezyMatch import train as dm_train
+    # train a new model
+    dm_train(input_file_path="./inputs/input_dfm_pytest_001.yaml",
+             dataset_path="./dataset/dataset-string-similarity_test.txt",
+             model_name="test001")
+
+def test_finetune():
+    from DeezyMatch import finetune as dm_finetune
+    # fine-tune a pretrained model stored at pretrained_model_path and pretrained_vocab_path
+    dm_finetune(input_file_path="./inputs/input_dfm_pytest_001.yaml",
+                dataset_path="./dataset/dataset-string-similarity_test.txt",
+                model_name="finetuned_test001",
+                pretrained_model_path="./models/test001/test001.model",
+                pretrained_vocab_path="./models/test001/test001.vocab")
+
+def test_inference():
+    from DeezyMatch import inference as dm_inference
+
+    # model inference using a model stored at pretrained_model_path and pretrained_vocab_path
+    dm_inference(input_file_path="./inputs/input_dfm_pytest_001.yaml",
+                 dataset_path="./dataset/dataset-string-similarity_test.txt",
+                 pretrained_model_path="./models/finetuned_test001/finetuned_test001.model",
+                 pretrained_vocab_path="./models/finetuned_test001/finetuned_test001.vocab")
+    
+def test_generate_query_vecs():
+    from DeezyMatch import inference as dm_inference
+    
+    # generate vectors for queries (specified in dataset_path)
+    # using a model stored at pretrained_model_path and pretrained_vocab_path
+    dm_inference(input_file_path="./inputs/input_dfm_pytest_001.yaml",
+                 dataset_path="./dataset/dataset-string-similarity_test.txt",
+                 pretrained_model_path="./models/finetuned_test001/finetuned_test001.model",
+                 pretrained_vocab_path="./models/finetuned_test001/finetuned_test001.vocab",
+                 inference_mode="vect",
+                 scenario="queries/test")
+
+def test_generate_candidate_vecs():
+    from DeezyMatch import inference as dm_inference
+    
+    # generate vectors for candidates (specified in dataset_path)
+    # using a model stored at pretrained_model_path and pretrained_vocab_path
+    dm_inference(input_file_path="./inputs/input_dfm_pytest_001.yaml",
+                 dataset_path="./dataset/dataset-string-similarity_test.txt",
+                 pretrained_model_path="./models/finetuned_test001/finetuned_test001.model",
+                 pretrained_vocab_path="./models/finetuned_test001/finetuned_test001.vocab",
+                 inference_mode="vect",
+                 scenario="candidates/test")
+
+def test_assemble_queries():
+    from DeezyMatch import combine_vecs
+
+    # combine vectors stored in queries/test and save them in combined/queries_test
+    combine_vecs(rnn_passes=['fwd', 'bwd'], 
+                 input_scenario='queries/test', 
+                 output_scenario='combined/queries_test', 
+                 print_every=10)
+
+def test_assemble_candidates():
+    from DeezyMatch import combine_vecs
+
+    # combine vectors stored in candidates/test and save them in combined/candidates_test
+    combine_vecs(rnn_passes=['fwd', 'bwd'],
+                 input_scenario='candidates/test',
+                 output_scenario='combined/candidates_test',
+                 print_every=10)
+
+def test_candidate_ranker():
+    from DeezyMatch import candidate_ranker
+    
+    # Select candidates based on L2-norm distance (aka faiss distance):
+    # find candidates from candidate_scenario
+    # for queries specified in query_scenario
+    candidates_pd = \
+        candidate_ranker(query_scenario="./combined/queries_test",
+                         candidate_scenario="./combined/candidates_test",
+                         ranking_metric="faiss",
+                         selection_threshold=5.,
+                         num_candidates=2,
+                         search_size=10,
+                         output_path="ranker_results/test_candidates_deezymatch",
+                         pretrained_model_path="./models/finetuned_test001/finetuned_test001.model",
+                         pretrained_vocab_path="./models/finetuned_test001/finetuned_test001.vocab",
+                         number_test_rows=5)

README.md

@@ -17,6 +17,9 @@
     <a href="https://mybinder.org/v2/gh/Living-with-machines/DeezyMatch/HEAD?filepath=examples">
         <img alt="Binder" src="https://mybinder.org/badge_logo.svg">
     </a>
+    <a href="https://github.com/Living-with-machines/DeezyMatch/workflows/Continuous%20integration/badge.svg">
+        <img alt="Continuous integration badge" src="https://github.com/Living-with-machines/DeezyMatch/workflows/Continuous%20integration/badge.svg">
+    </a>
     <br/>
 </p>
 

inputs/input_dfm_pytest_001.yaml

@@ -0,0 +1,101 @@
+general:
+  use_gpu: True    # only if available
+  # specify CUDA device, these are 0-indexed, e.g., 
+  #   cuda:0, cuda:1 or others. "cuda" is the default CUDA device
+  gpu_device: "cuda"
+  # Parent dir to save trained models 
+  models_dir: "./models"
+
+preprocessing:
+  # normalizing text to the ASCII encoding standard
+  uni2ascii: True
+  lowercase: True
+  # removing both the leading and the trailing empty characters
+  strip: True
+  only_latin_letters: False
+  # Strings in the inputs will be prefix + string + suffix
+  prefix_suffix: ["|", "|"]
+  # Accepted proportion of characters in a string that are not present in our vocabulary, i.e.,
+  # String is accepted if:
+  # (number of its characters found in the vocabulary)/(total number its characters) >= missing_char_threshold
+  missing_char_threshold: 0.5
+  # read a list of characters and add to the vocabulary
+  read_list_chars: "./dataset/characters_v001.vocab"
+
+# --- RNN/GRU/LSTM architecture/misc info
+gru_lstm:
+  main_architecture: "gru"    # rnn, gru, lstm
+  mode:    # Tokenization mode
+    # choices: "char", "ngram", "word"
+    # for example: tokenize: ["char", "ngram", "word"] or ["char", "word"] 
+    tokenize: ["char"]
+    # ONLY if "ngram" is selected in tokenize, the following args will be used:
+    min_gram: 2
+    max_gram: 3
+  bidirectional: True    # if True, becomes a bidirectional RNN/GRU/LSTM
+  # num_layers
+  # number of recurrent layers. e.g., setting num_layers=2 means stacking two 
+  #   RNN/GRU/LSTMs together to form a stacked RNN/GRU/LSTM,
+  #   with the second RNN/GRU/LSTM taking in outputs of the first RNN/GRU/LSTM and computing the final results.
+  num_layers: 2
+  # number of dimensions of the first fully connected network
+  fc1_out_dim: 120    
+  # pooling_mode:
+  # hstates_layers_simple, hstates_layers, hstates
+  # hstates_subtract, hstates_l2_distance, hstates_cosine
+  # average, max
+  # attention
+  pooling_mode: 'hstates_layers_simple'    
+  # rnn_dropout:
+  # if non-zero, introduces a Dropout layer on the outputs of each RNN/LSTM/GRU layer except the last layer,
+  #   with dropout probability equal to rnn_dropout.
+  rnn_dropout: 0.01
+  # fully-connected layers dropout depends on the number of fc layers (currently there are two)
+  fc_dropout: [0.01, 0.01]  
+  # attention layer dropout depends on the number of attention layers (currently there are two)
+  att_dropout: [0.01, 0.01] 
+  # Add bias to all learnable parameters
+  bias: True
+
+  rnn_hidden_dim: 60
+  max_seq_len: 120
+  embedding_dim: 60
+  output_dim: 2
+
+  learning_rate: 0.001    # 3e-4
+  optimizer: adam
+  epochs: 3
+  batch_size: 32
+  # shuffle when creating DataLoader
+  dl_shuffle: True 
+  random_seed: 123
+  # Early stopping:
+  # Number of epochs with no improvement after which training will be stopped and
+  #   the model with the least validation loss will be saved
+  # If 0 or negative, early stopping will be deactivated
+  early_stopping_patience: -1
+
+  # if -1 or 1, perform the validation step in every epoch; 
+  # if 0, no validation will be done
+  # otherwise, specify the interval (integer)
+  validation: 1
+  # split dataset
+  train_proportion: 0.5
+  val_proportion: 0.3
+  test_proportion: 0.2
+
+  # False or path to a directory to create tensor-board
+  #create_tensor_board: "./tb_gru_test" 
+  create_tensor_board: False
+
+  # Layers to freeze during fine-tuning
+  layers_to_freeze: ["emb", "rnn_1", "attn"]
+
+inference:
+ # Output predictions and save the results in output_preds_file
+ output_preds: True
+ # either a path or "default"
+ #   "default" saves the prediction output inside the model directory
+ output_preds_file: "default"
+ # change it to true to have Mean Average Precision as an eval metric. Note that this would have an impact on computational time
+ eval_map_metric: False

requirements.txt

@@ -1,12 +1,12 @@
 torch==1.5.0
 torchvision==0.6.0
 ipywidgets==7.5.1
-PyYAML==5.3.1
+PyYAML==5.4
 scikit-learn==0.23.1
-pandas==1.0.3
+pandas==1.0.5
 faiss-cpu==1.6.3
 tqdm==4.46.0
 tensorboard==2.2.2
 matplotlib==3.2.1
 jupyter-client==6.1.5
-jupyter-core==4.6.3
+jupyter-core==4.6.3

setup.py

@@ -2,7 +2,7 @@
 
 setuptools.setup(
     name="DeezyMatch",
-    version="1.2.2",
+    version="1.2.3",
     description="A Flexible Deep Learning Approach to Fuzzy String Matching",
     author=u"The LwM Development Team",
     #author_email="",
@@ -21,9 +21,9 @@
         "torch==1.5.0",
         "torchvision==0.6.0",
         "ipywidgets==7.5.1",
-        "PyYAML==5.3.1",
+        "PyYAML==5.4",
         "scikit-learn==0.23.1",
-        "pandas==1.0.3",
+        "pandas==1.0.5",
         "faiss-cpu==1.6.3",
         "tqdm==4.46.0",
         "tensorboard==2.2.2",