update crawling notebook

Author: zoltanctoth

serverless/3_web_scraping.ipynb

@@ -24,9 +24,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "📚 Setting up the environment...\n",
+      "✅ Libraries imported successfully\n"
+     ]
+    }
+   ],
    "source": [
     "# Import required libraries for web scraping\n",
     "print(\"📚 Setting up the environment...\")\n",
@@ -48,14 +57,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🌐 Attempting to access webpage without headers...\n",
+      "📡 Response status code: 403\n",
+      "❌ Access forbidden! This demonstrates why we need proper headers.\n",
+      "ℹ️  Websites often block requests without proper User-Agent headers\n",
+      "   to prevent automated scraping.\n"
+     ]
+    }
+   ],
    "source": [
     "# First, let's try without headers to demonstrate why they're needed\n",
     "print(\"🌐 Attempting to access webpage without headers...\")\n",
     "\n",
-    "url = \"https://ceu.edu/article/2024-12-03/combining-data-science-society-thanika-haltrich-presidential-scholar-award\"\n",
+    "url = \"https://www.ceu.edu/news/2025-11/cultivating-mindsets-and-practices-environmental-sustainability\"\n",
     "\n",
     "try:\n",
     "    response = requests.get(url)\n",
@@ -80,9 +101,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🌐 Attempting to access webpage with proper headers...\n",
+      "📡 Response status code: 200\n",
+      "✅ Successfully retrieved the webpage! Received 59136 bytes of data.\n"
+     ]
+    }
+   ],
    "source": [
     "# Now let's try with proper headers\n",
     "print(\"🌐 Attempting to access webpage with proper headers...\")\n",
@@ -105,6 +136,13 @@
     "    print(f\"❌ Error: {str(e)}\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Install https://selectorgadget.com/ to extract different parts of the website."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -118,7 +156,27 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🔍 Parsing webpage content...\n",
+      "\n",
+      "📑 Page Title:\n",
+      "----------------------------------------\n",
+      "Cultivating Mindsets and Practices for Environmental Sustainability | Central European University\n",
+      "----------------------------------------\n",
+      "\n",
+      "📝 Article Content:\n",
+      "----------------------------------------\n",
+      "\n",
+      "----------------------------------------\n",
+      "\n",
+      "✅ Content extracted successfully\n"
+     ]
+    }
+   ],
    "source": [
     "# Parse the HTML and extract content\n",
     "print(\"🔍 Parsing webpage content...\")\n",
@@ -137,7 +195,7 @@
     "    # Extract paragraphs\n",
     "    print(\"\\n📝 Article Content:\")\n",
     "    print(\"-\" * 40)\n",
-    "    description_html = webpage.select(\"#block-system-main p\")  # <----------------- !!! Our selector !!!\n",
+    "    description_html = webpage.select(\".col-lg-8\")  # <----------------- !!! Our selector !!!\n",
     "    texts = [text.get_text().strip() for text in description_html]\n",
     "    text = \"\\n\".join(texts)\n",
     "    print(text)\n",
@@ -165,7 +223,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.0"
+   "version": "3.13.5"
   }
  },
  "nbformat": 4,