03_Tokenisation_Stopwords_Delimiters 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "febee6f4-adf3-4127-837f-814b6b9c7c59",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f91cde17-6ecd-41eb-aaf1-9ad097b5eadc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\dell\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords') "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e39cd85-134f-41b1-a0b0-241b3fb536d9",
   "metadata": {},
   "source": [
    "## Python function to tokenize text using stopwords as delimiters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5d06f776-ca0a-4722-bb48-0f6e3b06d225",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load English stop words\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def tokenize_with_stopwords(text):\n",
    "    \"\"\"Tokenizes a document using stop words as delimiters.\"\"\"\n",
    "\n",
    "    # Compile a regular expression pattern to match stop words followed by optional whitespace\n",
    "    pattern = re.compile(r\"\\b(?:{})\\b\\s*\".format(\"|\".join(stop_words)), re.IGNORECASE)\n",
    "\n",
    "    # Split the text using the pattern, preserving consecutive non-stop words\n",
    "    tokens = pattern.split(text)\n",
    "    \n",
    "\n",
    "    # Remove any empty tokens\n",
    "    tokens = [token.strip() for token in tokens if token.strip()]\n",
    "\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5b22c42-3bb1-41f9-b358-becb34b9a49d",
   "metadata": {},
   "source": [
    "### Performing tokenize on a sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4988fb12-450f-4202-937d-e03ebc852f7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample document', 'tokenization using stopwords', 'delimiters.']\n"
     ]
    }
   ],
   "source": [
    "document = \"This is a sample document for tokenization using stopwords as delimiters.\"\n",
    "tokens = tokenize_with_stopwords(document)\n",
    "print(tokens)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "321b9c0f-98c6-4786-bff1-f0639b988f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_text_file_with_stopwords(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        text = file.read()\n",
    "        tokens = tokenize_with_stopwords(text)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cc0fb68f-d815-4871-96dd-9e3ccd812391",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens: ['good morning! Good morning', 'ye', 'thou!', '’', 'say', 'patients,', 'worse', 'hypocrites,', 'hypocrites,', 'cruel', 'phony hypocrites,', 'worst.”']\n"
     ]
    }
   ],
   "source": [
    "file_path = 'demo.txt'\n",
    "\n",
    "tokens = tokenize_text_file_with_stopwords(file_path)\n",
    "print(\"Tokens:\", tokens)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}