IRProject.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E3tW7KN_jNWO",
        "outputId": "21bc43e5-8467-4f46-fb1d-9181639238ac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (2.27.1)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (4.11.2)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests) (3.4)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4) (2.4.1)\n"
          ]
        }
      ],
      "source": [
        "!pip install requests beautifulsoup4\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install nltk"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x9-8FIMqxOKs",
        "outputId": "fac4493e-e650-482e-be1d-9236cd2021f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.8.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk) (8.1.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk) (1.2.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk) (2022.10.31)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk) (4.65.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6xq-Lfa6xKFG",
        "outputId": "c022015b-05af-476d-bd3c-7577feb521c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install scikit-learn rank-bm25"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f8ZQQv2rWtK6",
        "outputId": "a22317a0-8250-4c7c-baa0-490d94b8c7b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
            "Requirement already satisfied: rank-bm25 in /usr/local/lib/python3.10/dist-packages (0.2.2)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.22.4)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.10.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (3.1.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install Flask beautifulsoup4"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QG9uJOy4-MLo",
        "outputId": "e3d88d31-0bb3-4432-989e-533434105e29"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: Flask in /usr/local/lib/python3.10/dist-packages (2.2.4)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (4.11.2)\n",
            "Requirement already satisfied: Werkzeug>=2.2.2 in /usr/local/lib/python3.10/dist-packages (from Flask) (2.3.0)\n",
            "Requirement already satisfied: Jinja2>=3.0 in /usr/local/lib/python3.10/dist-packages (from Flask) (3.1.2)\n",
            "Requirement already satisfied: itsdangerous>=2.0 in /usr/local/lib/python3.10/dist-packages (from Flask) (2.1.2)\n",
            "Requirement already satisfied: click>=8.0 in /usr/local/lib/python3.10/dist-packages (from Flask) (8.1.3)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4) (2.4.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from Jinja2>=3.0->Flask) (2.1.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "from urllib.parse import urljoin\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from rank_bm25 import BM25Okapi\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "import nltk\n",
        "import re\n",
        "\n",
        "\n",
        "import os\n",
        "from bs4 import BeautifulSoup\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "hMjU0QhGZs-a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import requests\n",
        "from urllib.parse import urljoin\n",
        "from bs4 import BeautifulSoup\n",
        "from flask import Flask, render_template, request\n",
        "from bs4 import BeautifulSoup\n",
        "import requests\n",
        "import os\n",
        "\n",
        "app = Flask(__name__)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "class WebCrawler:\n",
        "    def __init__(self, seeds):\n",
        "        self.seeds = seeds\n",
        "        self.url_frontier = list(seeds)\n",
        "        self.visited_urls = set()\n",
        "        self.crawled_pages = []\n",
        "\n",
        "    def crawl(self, max_pages):\n",
        "        while self.url_frontier and len(self.crawled_pages) < max_pages:\n",
        "            url = self.url_frontier.pop(0)\n",
        "            if url not in self.visited_urls:\n",
        "                content = self.fetch_page_content(url)\n",
        "                if content is not None:\n",
        "                    self.save_page(url, content)\n",
        "                    new_links = self.extract_links(url, content)\n",
        "                    self.add_new_links(new_links)\n",
        "                    self.visited_urls.add(url)\n",
        "\n",
        "    def fetch_page_content(self, url):\n",
        "        try:\n",
        "            response = requests.get(url)\n",
        "            if response.status_code == 200:\n",
        "                return response.text\n",
        "        except requests.exceptions.RequestException as e:\n",
        "            print(\"Error fetching page:\", e)\n",
        "        return None\n",
        "\n",
        "    def save_page(self, url, content):\n",
        "        self.crawled_pages.append((url, content))\n",
        "        filename = os.path.basename(url) + \".html\"  # Extract filename from the URL\n",
        "        with open(filename, \"w\", encoding=\"utf-8\") as file:\n",
        "            file.write(content)\n",
        "        print(\"Page saved:\", filename)\n",
        "\n",
        "    def extract_links(self, url, content):\n",
        "        links = []\n",
        "        soup = BeautifulSoup(content, 'html.parser')\n",
        "        for link in soup.find_all('a'):\n",
        "            href = link.get('href')\n",
        "            if href:\n",
        "                absolute_url = urljoin(url, href)\n",
        "                links.append(absolute_url)\n",
        "        return links\n",
        "\n",
        "    def add_new_links(self, links):\n",
        "        for link in links:\n",
        "            if link not in self.visited_urls and link not in self.url_frontier:\n",
        "                self.url_frontier.append(link)\n",
        "\n",
        "\n",
        "# Initialize web crawler\n",
        "seeds = [\n",
        "    'https://en.wikipedia.org/wiki/Cricket',\n",
        "    'https://en.wikipedia.org/wiki/Cricket_(insect)',\n",
        "    'https://en.wikipedia.org/wiki/Cricket_(disambiguation)',\n",
        "    'https://en.wikipedia.org/wiki/New_York_City',\n",
        "    'https://en.wikipedia.org/wiki/Box_score_(baseball)',\n",
        "    'https://en.wikipedia.org/wiki/Cricket#cite_note-158',\n",
        "    'https://en.wikipedia.org/wiki/New_York_Clipper',\n",
        "    'https://en.wikipedia.org/wiki/Cricket#cite_note-NPR-159',\n",
        "    'https://en.wikipedia.org/wiki/Cricket#cite_note-Myth-160',\n",
        "    'https://en.wikipedia.org/wiki/Portal:Cricket',\n",
        "]\n",
        "\n",
        "max_pages = 100\n",
        "crawler = WebCrawler(seeds)\n",
        "crawler.crawl(max_pages)\n",
        "\n",
        "\n",
        "# Text Extraction and Preprocessing\n",
        "documents = []\n",
        "folder_path = \"/content/sample_data\"\n",
        "for url, content in crawler.crawled_pages:\n",
        "    soup = BeautifulSoup(content, 'html.parser')\n",
        "    title = soup.find('title')\n",
        "    body = soup.find('body')\n",
        "    if title and body:\n",
        "        document = title.get_text() + \" \" + body.get_text()\n",
        "        documents.append(document)\n",
        "\n",
        "'''for file_name in os.listdir(folder_path):\n",
        "    file_path = os.path.join(folder_path, file_name)\n",
        "    if file_name.endswith('.html'):\n",
        "        with open(file_path, 'r', encoding='utf-8') as file:\n",
        "            html_content = file.read()\n",
        "            soup = BeautifulSoup(html_content, 'html.parser')\n",
        "            title = soup.find('h1').get_text()\n",
        "            body = soup.find('div', {'id': 'bodyContent'}).get_text()\n",
        "            document = Document(title, body)\n",
        "            documents.append(document)'''\n",
        "\n",
        "# Search Engine\n",
        "# Indexing\n",
        "class Indexer:\n",
        "    def __init__(self):\n",
        "        self.vectorizer = None\n",
        "        self.index = None\n",
        "\n",
        "    def build_index(self, documents):\n",
        "        self.vectorizer = CountVectorizer()\n",
        "        self.index = self.vectorizer.fit_transform(documents)\n",
        "\n",
        "indexer = Indexer()\n",
        "if documents:\n",
        "    indexer.build_index(documents)\n",
        "else:\n",
        "    print(\"No documents found for indexing.\")\n",
        "\n",
        "\n",
        "    def preprocess_text(self, text):\n",
        "        tokens = word_tokenize(text)  # Tokenization\n",
        "        tokens = [token.lower() for token in tokens]  # Convert tokens to lowercase\n",
        "        tokens = [token for token in tokens if token.isalpha()]  # Remove non-alphabetic tokens\n",
        "        tokens = [token for token in tokens if token not in self.stop_words]  # Remove stopwords\n",
        "        preprocessed_text = ' '.join(tokens)\n",
        "        return preprocessed_text\n",
        "\n",
        "    def search(self, query, ranking_method, k=10):\n",
        "        preprocessed_query = self.preprocess_text(query)\n",
        "        query_vector = self.vectorizer.transform([preprocessed_query])\n",
        "        if ranking_method == 'cosine_similarity':\n",
        "            scores = cosine_similarity(query_vector, self.index)\n",
        "            results = [(score, doc) for score, doc in zip(scores[0], self.documents)]\n",
        "        elif ranking_method == 'bm25':\n",
        "            corpus = [doc.split() for doc in self.documents]\n",
        "            bm25 = BM25Okapi(corpus)\n",
        "            scores = bm25.get_scores(preprocessed_query.split())\n",
        "            results = [(score, doc) for score, doc in zip(scores, self.documents)]\n",
        "        else:\n",
        "            raise ValueError(\"Invalid ranking method. Choose 'cosine_similarity' or 'bm25'.\")\n",
        "\n",
        "        results.sort(reverse=True)\n",
        "        top_k_results = results[:k]\n",
        "        return top_k_results\n",
        "\n",
        "\n",
        "# Initialize search engine and build index\n",
        "indexer = Indexer()\n",
        "indexer.build_index(documents)\n",
        "crawler = WebCrawler(seeds)\n",
        "\n",
        "@app.route('/')\n",
        "def home():\n",
        "    return render_template('index.html')\n",
        "\n",
        "@app.route('/search', methods=['POST'])\n",
        "def search():\n",
        "    query = request.form['query']\n",
        "    ranking_method = 'cosine_similarity'  # or 'bm25'\n",
        "    results = indexer.search(query, ranking_method, k=10)\n",
        "\n",
        "    # Retrieve document summaries\n",
        "    summaries = []\n",
        "    for _, doc in results:\n",
        "        # Extract a brief summary of the document (e.g., first few sentences)\n",
        "        # Modify this code to extract relevant information from your documents\n",
        "        summary = doc.body[:200] + '...'  # Example summary: First 200 characters of the body\n",
        "        summaries.append(summary)\n",
        "\n",
        "    return render_template('results.html', query=query, results=results, summaries=summaries)\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    app.run(debug=True)\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FU31Qbh9xdeN",
        "outputId": "29734da6-ed4e-4484-d339-01572099ac00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Page saved: Cricket.html\n",
            "Page saved: Cricket_(insect).html\n",
            "Page saved: Cricket_(disambiguation).html\n",
            "Page saved: New_York_City.html\n",
            "Page saved: Box_score_(baseball).html\n",
            "Page saved: Cricket#cite_note-158.html\n",
            "Page saved: New_York_Clipper.html\n",
            "Page saved: Cricket#cite_note-NPR-159.html\n",
            "Page saved: Cricket#cite_note-Myth-160.html\n",
            "Page saved: Portal:Cricket.html\n",
            "Page saved: Cricket#bodyContent.html\n",
            "Page saved: Main_Page.html\n",
            "Page saved: Wikipedia:Contents.html\n",
            "Page saved: Portal:Current_events.html\n",
            "Page saved: Special:Random.html\n",
            "Page saved: Wikipedia:About.html\n",
            "Page saved: Wikipedia:Contact_us.html\n",
            "Page saved: Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en.html\n",
            "Page saved: Help:Contents.html\n",
            "Page saved: Help:Introduction.html\n",
            "Page saved: Wikipedia:Community_portal.html\n",
            "Page saved: Special:RecentChanges.html\n",
            "Page saved: Wikipedia:File_upload_wizard.html\n",
            "Page saved: Special:Search.html\n",
            "Page saved: index.php?title=Special:CreateAccount&returnto=Cricket.html\n",
            "Page saved: index.php?title=Special:UserLogin&returnto=Cricket.html\n",
            "Page saved: Special:MyContributions.html\n",
            "Page saved: Special:MyTalk.html\n",
            "Page saved: Cricket#History.html\n",
            "Page saved: Cricket#Origins.html\n",
            "Page saved: Cricket#Growth_of_amateur_and_professional_cricket_in_England.html\n",
            "Page saved: Cricket#English_cricket_in_the_18th_and_19th_centuries.html\n",
            "Page saved: Cricket#Cricket_becomes_an_international_sport.html\n",
            "Page saved: Cricket#World_cricket_in_the_20th_century.html\n",
            "Page saved: Cricket#The_rise_of_limited_overs_cricket.html\n",
            "Page saved: Cricket#Laws_and_gameplay.html\n",
            "Page saved: Cricket#Playing_area.html\n",
            "Page saved: Cricket#Match_structure_and_closure.html\n",
            "Page saved: Cricket#Innings.html\n",
            "Page saved: Cricket#Overs.html\n",
            "Page saved: Cricket#Clothing_and_equipment.html\n",
            "Page saved: Cricket#Bat_and_ball.html\n",
            "Page saved: Cricket#Player_roles.html\n",
            "Page saved: Cricket#Basic_gameplay:_bowler_to_batter.html\n",
            "Page saved: Cricket#Fielding.html\n",
            "Page saved: Cricket#Bowling_and_dismissal.html\n",
            "Page saved: Cricket#Batting,_runs_and_extras.html\n",
            "Page saved: Cricket#Specialist_roles.html\n",
            "Page saved: Cricket#Umpires_and_scorers.html\n",
            "Page saved: Cricket#Spirit_of_the_Game.html\n",
            "Page saved: Cricket#Women's_cricket.html\n",
            "Page saved: Cricket#Governance.html\n",
            "Page saved: Cricket#Forms_of_cricket.html\n",
            "Page saved: Cricket#Competitions.html\n",
            "Page saved: Cricket#International_competitions.html\n",
            "Page saved: Cricket#National_competitions.html\n",
            "Page saved: Cricket#First-class.html\n",
            "Page saved: Cricket#Limited_overs.html\n",
            "Page saved: Cricket#Other.html\n",
            "Page saved: Cricket#Club_and_school_cricket.html\n",
            "Page saved: Cricket#Culture.html\n",
            "Page saved: Cricket#Influence_on_everyday_life.html\n",
            "Page saved: Cricket#In_the_arts_and_popular_culture.html\n",
            "Page saved: Cricket#Influence_on_other_sports.html\n",
            "Page saved: Cricket#See_also.html\n",
            "Page saved: Cricket#Footnotes.html\n",
            "Page saved: Cricket#Citations.html\n",
            "Page saved: Cricket#Sources.html\n",
            "Page saved: Cricket#Further_reading.html\n",
            "Page saved: Cricket#External_links.html\n",
            "Page saved: Krieket.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: %D9%83%D8%B1%D9%8A%D9%83%D8%AA.html\n",
            "Page saved: Cr%C3%ADquet.html\n",
            "Page saved: %E0%A6%95%E0%A7%8D%E0%A7%B0%E0%A6%BF%E0%A6%95%E0%A7%87%E0%A6%9F.html\n",
            "Page saved: Cr%C3%ADquet.html\n",
            "Page saved: %E0%A4%95%E0%A5%8D%E0%A4%B0%E0%A4%BF%E0%A4%95%E0%A5%87%E0%A4%9F.html\n",
            "Page saved: Kir%C3%ADkete.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: %DA%A9%D8%B1%DB%8C%DA%A9%D8%AA.html\n",
            "Page saved: %E0%A6%95%E0%A7%8D%E0%A6%B0%E0%A6%BF%E0%A6%95%E0%A7%87%E0%A6%9F.html\n",
            "Page saved: P%C3%A1n-ki%C3%BB.html\n",
            "Page saved: %D0%9A%D1%80%D0%B8%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: %D0%9A%D1%80%D1%8B%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: %D0%9A%D1%80%D1%8B%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: %E0%A4%95%E0%A5%8D%E0%A4%B0%E0%A4%BF%E0%A4%95%E0%A5%87%E0%A4%9F.html\n",
            "Page saved: %D0%9A%D1%80%D0%B8%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: Kriked.html\n",
            "Page saved: Criquet.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: Criced.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: %E0%A4%95%E0%A5%8D%E0%A4%B0%E0%A4%BF%E0%A4%95%E0%A5%87%E0%A4%9F.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: %CE%9A%CF%81%CE%AF%CE%BA%CE%B5%CF%84.html\n",
            "Page saved: Cr%C3%ADquet.html\n",
            "Page saved: Kriketo.html\n",
            " * Serving Flask app '__main__'\n",
            " * Debug mode: on\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:werkzeug:\u001b[31m\u001b[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.\u001b[0m\n",
            " * Running on http://127.0.0.1:5000\n",
            "INFO:werkzeug:\u001b[33mPress CTRL+C to quit\u001b[0m\n",
            "INFO:werkzeug: * Restarting with stat\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "xZ4ZUo_ixdgw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from flask import Flask, request, render_template\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from rank_bm25 import BM25Okapi\n",
        "from bs4 import BeautifulSoup\n",
        "import requests\n",
        "import os\n",
        "\n",
        "app = Flask(__name__)\n",
        "\n",
        "class WebCrawler:\n",
        "    def __init__(self, seeds):\n",
        "        self.seeds = seeds\n",
        "        self.url_frontier = list(seeds)\n",
        "        self.visited_urls = set()\n",
        "        self.crawled_pages = []\n",
        "\n",
        "    def crawl(self, max_pages):\n",
        "        while self.url_frontier and len(self.crawled_pages) < max_pages:\n",
        "            url = self.url_frontier.pop(0)\n",
        "            if url not in self.visited_urls:\n",
        "                content = self.fetch_page_content(url)\n",
        "                if content is not None:\n",
        "                    self.save_page(url, content)\n",
        "                    new_links = self.extract_links(url, content)\n",
        "                    self.add_new_links(new_links)\n",
        "                    self.visited_urls.add(url)\n",
        "\n",
        "    def fetch_page_content(self, url):\n",
        "        try:\n",
        "            response = requests.get(url)\n",
        "            if response.status_code == 200:\n",
        "                return response.text\n",
        "        except requests.exceptions.RequestException as e:\n",
        "            print(\"Error fetching page:\", e)\n",
        "        return None\n",
        "\n",
        "    def save_page(self, url, content):\n",
        "        self.crawled_pages.append((url, content))\n",
        "        filename = os.path.basename(url) + \".html\"  # Extract filename from the URL\n",
        "        with open(filename, \"w\", encoding=\"utf-8\") as file:\n",
        "            file.write(content)\n",
        "        print(\"Page saved:\", filename)\n",
        "\n",
        "    def extract_links(self, url, content):\n",
        "        links = []\n",
        "        soup = BeautifulSoup(content, 'html.parser')\n",
        "        for link in soup.find_all('a'):\n",
        "            href = link.get('href')\n",
        "            if href:\n",
        "                absolute_url = urljoin(url, href)\n",
        "                links.append(absolute_url)\n",
        "        return links\n",
        "\n",
        "    def add_new_links(self, links):\n",
        "        for link in links:\n",
        "            if link not in self.visited_urls and link not in self.url_frontier:\n",
        "                self.url_frontier.append(link)\n",
        "\n",
        "# Example usage\n",
        "seeds = ['https://en.wikipedia.org/wiki/Cricket']\n",
        "max_pages = 100\n",
        "crawler = WebCrawler(seeds)\n",
        "crawler.crawl(max_pages)\n",
        "\n",
        "# Text Extraction and Preprocessing\n",
        "documents = []\n",
        "for url, content in crawler.crawled_pages:\n",
        "    soup = BeautifulSoup(content, 'html.parser')\n",
        "    title = soup.find('title')\n",
        "    body = soup.find('body')\n",
        "    if title and body:\n",
        "        document = title.get_text() + \" \" + body.get_text()\n",
        "        documents.append(document)\n",
        "\n",
        "# Indexing\n",
        "class Indexer:\n",
        "    def __init__(self):\n",
        "        self.vectorizer = None\n",
        "        self.index = None\n",
        "        self.document_urls = []\n",
        "\n",
        "    def build_index(self, documents):\n",
        "        self.vectorizer = TfidfVectorizer()\n",
        "        self.index = self.vectorizer.fit_transform(documents)\n",
        "\n",
        "    def get_document_urls(self):\n",
        "        return self.document_urls\n",
        "\n",
        "indexer = Indexer()\n",
        "if documents:\n",
        "    indexer.build_index(documents)\n",
        "    indexer.document_urls = [url for url, _ in crawler.crawled_pages]\n",
        "else:\n",
        "    print(\"No documents found for indexing.\")\n",
        "\n",
        "# Search Engine\n",
        "class SearchEngine:\n",
        "    def __init__(self, indexer):\n",
        "        self.indexer = indexer\n",
        "\n",
        "    def search(self, query, ranking_method):\n",
        "        query_vector = self.indexer.vectorizer.transform([query])\n",
        "\n",
        "        if ranking_method == 'cosine':\n",
        "            scores = cosine_similarity(query_vector, self.indexer.index).flatten()\n",
        "        elif ranking_method == 'bm25':\n",
        "            bm25 = BM25Okapi(self.indexer.index)\n",
        "            scores = bm25.get_scores(query_vector)\n",
        "\n",
        "        top_indices = scores.argsort()[::-1][:10]\n",
        "        top_urls = [self.indexer.document_urls[i] for i in top_indices]\n",
        "        return top_urls\n",
        "\n",
        "search_engine = SearchEngine(indexer)\n",
        "\n",
        "@app.route('/', methods=['GET', 'POST'])\n",
        "def search():\n",
        "    if request.method == 'POST':\n",
        "        query = request.form['query']\n",
        "        ranking_method = request.form['ranking_method']\n",
        "        results = search_engine.search(query, ranking_method)\n",
        "        print(\"Results:\", results)  # Print the results\n",
        "        return render_template('results.html', query=query, results=results)\n",
        "    return render_template('index.html')\n",
        "\n",
        "'''def search():\n",
        "    if request.method == 'POST':\n",
        "        query = request.form['query']\n",
        "        ranking_method = request.form['ranking_method']\n",
        "        results = search_engine.search(query, ranking_method)\n",
        "        return render_template('results.html', query=query, results=results)\n",
        "    return render_template('index.html')'''\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    app.run(debug=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qWxCVJC7uYIo",
        "outputId": "adba2865-d2ae-4f9b-f7a1-276be817864d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Page saved: Cricket.html\n",
            "Page saved: Cricket#bodyContent.html\n",
            "Page saved: Main_Page.html\n",
            "Page saved: Wikipedia:Contents.html\n",
            "Page saved: Portal:Current_events.html\n",
            "Page saved: Special:Random.html\n",
            "Page saved: Wikipedia:About.html\n",
            "Page saved: Wikipedia:Contact_us.html\n",
            "Page saved: Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en.html\n",
            "Page saved: Help:Contents.html\n",
            "Page saved: Help:Introduction.html\n",
            "Page saved: Wikipedia:Community_portal.html\n",
            "Page saved: Special:RecentChanges.html\n",
            "Page saved: Wikipedia:File_upload_wizard.html\n",
            "Page saved: Special:Search.html\n",
            "Page saved: index.php?title=Special:CreateAccount&returnto=Cricket.html\n",
            "Page saved: index.php?title=Special:UserLogin&returnto=Cricket.html\n",
            "Page saved: Special:MyContributions.html\n",
            "Page saved: Special:MyTalk.html\n",
            "Page saved: Cricket#History.html\n",
            "Page saved: Cricket#Origins.html\n",
            "Page saved: Cricket#Growth_of_amateur_and_professional_cricket_in_England.html\n",
            "Page saved: Cricket#English_cricket_in_the_18th_and_19th_centuries.html\n",
            "Page saved: Cricket#Cricket_becomes_an_international_sport.html\n",
            "Page saved: Cricket#World_cricket_in_the_20th_century.html\n",
            "Page saved: Cricket#The_rise_of_limited_overs_cricket.html\n",
            "Page saved: Cricket#Laws_and_gameplay.html\n",
            "Page saved: Cricket#Playing_area.html\n",
            "Page saved: Cricket#Match_structure_and_closure.html\n",
            "Page saved: Cricket#Innings.html\n",
            "Page saved: Cricket#Overs.html\n",
            "Page saved: Cricket#Clothing_and_equipment.html\n",
            "Page saved: Cricket#Bat_and_ball.html\n",
            "Page saved: Cricket#Player_roles.html\n",
            "Page saved: Cricket#Basic_gameplay:_bowler_to_batter.html\n",
            "Page saved: Cricket#Fielding.html\n",
            "Page saved: Cricket#Bowling_and_dismissal.html\n",
            "Page saved: Cricket#Batting,_runs_and_extras.html\n",
            "Page saved: Cricket#Specialist_roles.html\n",
            "Page saved: Cricket#Umpires_and_scorers.html\n",
            "Page saved: Cricket#Spirit_of_the_Game.html\n",
            "Page saved: Cricket#Women's_cricket.html\n",
            "Page saved: Cricket#Governance.html\n",
            "Page saved: Cricket#Forms_of_cricket.html\n",
            "Page saved: Cricket#Competitions.html\n",
            "Page saved: Cricket#International_competitions.html\n",
            "Page saved: Cricket#National_competitions.html\n",
            "Page saved: Cricket#First-class.html\n",
            "Page saved: Cricket#Limited_overs.html\n",
            "Page saved: Cricket#Other.html\n",
            "Page saved: Cricket#Club_and_school_cricket.html\n",
            "Page saved: Cricket#Culture.html\n",
            "Page saved: Cricket#Influence_on_everyday_life.html\n",
            "Page saved: Cricket#In_the_arts_and_popular_culture.html\n",
            "Page saved: Cricket#Influence_on_other_sports.html\n",
            "Page saved: Cricket#See_also.html\n",
            "Page saved: Cricket#Footnotes.html\n",
            "Page saved: Cricket#Citations.html\n",
            "Page saved: Cricket#Sources.html\n",
            "Page saved: Cricket#Further_reading.html\n",
            "Page saved: Cricket#External_links.html\n",
            "Page saved: Krieket.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: %D9%83%D8%B1%D9%8A%D9%83%D8%AA.html\n",
            "Page saved: Cr%C3%ADquet.html\n",
            "Page saved: %E0%A6%95%E0%A7%8D%E0%A7%B0%E0%A6%BF%E0%A6%95%E0%A7%87%E0%A6%9F.html\n",
            "Page saved: Cr%C3%ADquet.html\n",
            "Page saved: %E0%A4%95%E0%A5%8D%E0%A4%B0%E0%A4%BF%E0%A4%95%E0%A5%87%E0%A4%9F.html\n",
            "Page saved: Kir%C3%ADkete.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: %DA%A9%D8%B1%DB%8C%DA%A9%D8%AA.html\n",
            "Page saved: %E0%A6%95%E0%A7%8D%E0%A6%B0%E0%A6%BF%E0%A6%95%E0%A7%87%E0%A6%9F.html\n",
            "Page saved: P%C3%A1n-ki%C3%BB.html\n",
            "Page saved: %D0%9A%D1%80%D0%B8%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: %D0%9A%D1%80%D1%8B%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: %D0%9A%D1%80%D1%8B%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: %E0%A4%95%E0%A5%8D%E0%A4%B0%E0%A4%BF%E0%A4%95%E0%A5%87%E0%A4%9F.html\n",
            "Page saved: %D0%9A%D1%80%D0%B8%D0%BA%D0%B5%D1%82.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: Kriked.html\n",
            "Page saved: Criquet.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: Criced.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: %E0%A4%95%E0%A5%8D%E0%A4%B0%E0%A4%BF%E0%A4%95%E0%A5%87%E0%A4%9F.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: %CE%9A%CF%81%CE%AF%CE%BA%CE%B5%CF%84.html\n",
            "Page saved: Cr%C3%ADquet.html\n",
            "Page saved: Kriketo.html\n",
            "Page saved: Cr%C3%ADquet.html\n",
            "Page saved: Kriket.html\n",
            "Page saved: %DA%A9%D8%B1%DB%8C%DA%A9%D8%AA.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: Cricket.html\n",
            "Page saved: Cruic%C3%A9ad.html\n",
            "Page saved: Criogaid.html\n",
            "Page saved: Cr%C3%ADcket.html\n",
            " * Serving Flask app '__main__'\n",
            " * Debug mode: on\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:werkzeug:\u001b[31m\u001b[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.\u001b[0m\n",
            " * Running on http://127.0.0.1:5000\n",
            "INFO:werkzeug:\u001b[33mPress CTRL+C to quit\u001b[0m\n",
            "INFO:werkzeug: * Restarting with stat\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "UvKUMk8GzRZa"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}