MVP_cognitive_search_distilbert.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.7 64-bit ('base': conda)",
   "metadata": {
    "interpreter": {
     "hash": "dca0ade3e726a953b501b15e8e990130d2b7799f14cfd9f4271676035ebe5511"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# MVP Cognitive Search Application using \"distilbert\" model"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### Importing Libraries"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from ast import literal_eval\n",
    "\n",
    "from cdqa.utils.converters import pdf_converter\n",
    "from cdqa.pipeline import QAPipeline\n",
    "from cdqa.utils.download import download_model"
   ]
  },
  {
   "source": [
    "### Downloading Pre-trained model: BERT (Stanford question and answer data set)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\nDownloading trained model...\ndistilbert_qa.joblib already downloaded\n"
     ]
    }
   ],
   "source": [
    "download_model(model = 'distilbert-squad_1.1', dir='./models')"
   ]
  },
  {
   "source": [
    "### Converting text from PDF into Pandas dataframe"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                       title  \\\n",
       "0    09_080661e_Golpanol_ALS   \n",
       "1  PRD_30042690_Golpanol_ALS   \n",
       "2  08_150301e_Glucopon_GD_70   \n",
       "3    30594438_Glucopon_GD_70   \n",
       "\n",
       "                                          paragraphs  \n",
       "0  [Technical Information, June 2016Supersedes is...  \n",
       "1  [Product specification, Golpanol  ALS, Test pr...  \n",
       "2  [Technical Information , March 2015, 08_150301...  \n",
       "3  [Product specification, Glucopon  GD 70, Test ...  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>title</th>\n      <th>paragraphs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>09_080661e_Golpanol_ALS</td>\n      <td>[Technical Information, June 2016Supersedes is...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>PRD_30042690_Golpanol_ALS</td>\n      <td>[Product specification, Golpanol  ALS, Test pr...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>08_150301e_Glucopon_GD_70</td>\n      <td>[Technical Information , March 2015, 08_150301...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>30594438_Glucopon_GD_70</td>\n      <td>[Product specification, Glucopon  GD 70, Test ...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "df = pdf_converter(directory_path='./PDF_docs/')\n",
    "df.head(4)"
   ]
  },
  {
   "source": [
    "### Using pre-trained language model: distilbert"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "QAPipeline(reader=BertQA(adam_epsilon=1e-08,\n",
       "                         bert_model='distilbert-base-uncased',\n",
       "                         do_lower_case=True, fp16=False,\n",
       "                         gradient_accumulation_steps=1, learning_rate=5e-05,\n",
       "                         local_rank=-1, loss_scale=0, max_answer_length=30,\n",
       "                         n_best_size=20, no_cuda=False,\n",
       "                         null_score_diff_threshold=0.0, num_train_epochs=3.0,\n",
       "                         output_dir=None, predict_batch_size=8, seed=42,\n",
       "                         server_ip='', ser..._size=8,\n",
       "                         verbose_logging=False, version_2_with_negative=False,\n",
       "                         warmup_proportion=0.1, warmup_steps=0),\n",
       "           retrieve_by_doc=False,\n",
       "           retriever=BM25Retriever(b=0.75, floor=None, k1=2.0, lowercase=True,\n",
       "                                   max_df=1.0, min_df=2, ngram_range=(1, 2),\n",
       "                                   preprocessor=None, stop_words='english',\n",
       "                                   token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "                                   tokenizer=None, top_n=20, verbose=False,\n",
       "                                   vocabulary=None))"
      ]
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "cdqa_pipeline = QAPipeline(reader='./models/distilbert_qa.joblib', max_df = 1.0)\n",
    "cdqa_pipeline.fit_retriever(df=df)"
   ]
  },
  {
   "source": [
    "### Questions from technical information document:"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Question : What is the shelf life of Golpanol ALS?\nAnswer: 2 years\n"
     ]
    }
   ],
   "source": [
    "question_1 = 'What is the shelf life of Golpanol ALS?'\n",
    "prediction = cdqa_pipeline.predict(question_1)\n",
    "print('Question : {}'.format(question_1))\n",
    "print('Answer: {}'.format(prediction[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Question : What is the appearance of Golpanol ALS?\nAnswer: clear, colorless or yellowish liquid\n"
     ]
    }
   ],
   "source": [
    "question_2 = 'What is the appearance of Golpanol ALS?'\n",
    "prediction = cdqa_pipeline.predict(question_2)\n",
    "print('Question : {}'.format(question_2))\n",
    "print('Answer: {}'.format(prediction[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Question : What is Golpanol ALS pH value?\nAnswer: Unit Value\n"
     ]
    }
   ],
   "source": [
    "question_3 = 'What is Golpanol ALS pH value?'\n",
    "prediction = cdqa_pipeline.predict(question_3)\n",
    "print('Question : {}'.format(question_3))\n",
    "print('Answer: {}'.format(prediction[0]))"
   ]
  },
  {
   "source": [
    "### Questions from product specification document:"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Question : What is the PRD number of Golpanol ALS?\nAnswer: 30042690\n"
     ]
    }
   ],
   "source": [
    "question_4 = 'What is the PRD number of Golpanol ALS?'\n",
    "prediction = cdqa_pipeline.predict(question_4)\n",
    "print('Question : {}'.format(question_4))\n",
    "print('Answer: {}'.format(prediction[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Question : What is the density value of Golpanol ALS?\nAnswer: 1.19 – 1.23pH value\n"
     ]
    }
   ],
   "source": [
    "question_5 = 'What is the density value of Golpanol ALS?'\n",
    "prediction = cdqa_pipeline.predict(question_5)\n",
    "print('Question : {}'.format(question_5))\n",
    "print('Answer: {}'.format(prediction[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Question : What is the chemical nature of Golpanol ALS?\nAnswer: Sodium allyl sulfonate\n"
     ]
    }
   ],
   "source": [
    "question_6 = 'What is the chemical nature of Golpanol ALS?'\n",
    "prediction = cdqa_pipeline.predict(question_6)\n",
    "print('Question : {}'.format(question_6))\n",
    "print('Answer: {}'.format(prediction[0]))"
   ]
  }
 ]
}