SERP (MVP)

CHANGES.rst

@@ -1,6 +1,12 @@
 Changes
 =======
 
+Unreleased
+----------
+
+* Now requires ``zyte-common-items >= 0.22.0``.
+
+
 0.8.0 (2024-08-21)
 ------------------
 

docs/conf.py

@@ -48,8 +48,11 @@
     ),
 }
 
+autodoc_pydantic_model_show_config_summary = False
 autodoc_pydantic_model_show_field_summary = False
 autodoc_pydantic_model_show_json = False
+autodoc_pydantic_model_show_validator_members = False
+autodoc_pydantic_model_show_validator_summary = False
 
 # sphinx-reredirects
 redirects = {

docs/index.rst

@@ -18,6 +18,7 @@ zyte-spider-templates documentation
 
    templates/index
    E-commerce <templates/e-commerce>
+   Google search <templates/google-search>
 
 .. toctree::
    :caption: Customization

docs/reference/index.rst

@@ -9,6 +9,8 @@ Spiders
 
 .. autoclass:: zyte_spider_templates.EcommerceSpider
 
+.. autoclass:: zyte_spider_templates.GoogleSearchSpider
+
 
 Pages
 =====
@@ -41,3 +43,6 @@ Parameter mixins
     :exclude-members: model_computed_fields
 
 .. autoenum:: zyte_spider_templates.spiders.ecommerce.EcommerceCrawlStrategy
+
+.. autopydantic_model:: zyte_spider_templates.spiders.serp.SerpMaxPagesParam
+    :exclude-members: model_computed_fields

docs/templates/google-search.rst

@@ -0,0 +1,19 @@
+.. _google-search:
+
+=================================================
+Google search spider template (``google_search``)
+=================================================
+
+Basic use
+=========
+
+.. code-block:: shell
+
+    scrapy crawl google_search -a search_queries="foo bar"
+
+Parameters
+==========
+
+.. autopydantic_model:: zyte_spider_templates.spiders.serp.GoogleSearchSpiderParams
+    :inherited-members: BaseModel
+    :exclude-members: model_computed_fields

docs/templates/index.rst

@@ -29,3 +29,6 @@ Spider template list
 
 :ref:`E-commerce <e-commerce>`
     Get products from an e-commerce website.
+
+:ref:`Google Search <google-search>`
+    Get Google search results.

setup.py

@@ -18,7 +18,7 @@
         "scrapy-poet>=0.21.0",
         "scrapy-spider-metadata>=0.1.2",
         "scrapy-zyte-api[provider]>=0.16.0",
-        "zyte-common-items>=0.13.0",
+        "zyte-common-items>=0.22.0",
     ],
     classifiers=[
         "Development Status :: 3 - Alpha",

tests/__init__.py

@@ -1,7 +1,11 @@
 from typing import Any, Dict, Optional
 
+import pytest
 from scrapy.utils.test import TestSpider
 
+# https://docs.pytest.org/en/stable/how-to/writing_plugins.html#assertion-rewriting
+pytest.register_assert_rewrite("tests.utils")
+
 
 # scrapy.utils.test.get_crawler alternative that does not freeze settings.
 def get_crawler(*, settings: Optional[Dict[str, Any]] = None):

tests/test_base.py

@@ -5,4 +5,4 @@
 
 def test_deprecation():
     with pytest.deprecated_call(match="^BaseSpiderParams is deprecated.*"):
-        BaseSpiderParams(url="https://example.com")
+        BaseSpiderParams(url="https://example.com")  # type: ignore[call-arg]

tests/test_ecommerce.py

@@ -1,6 +1,4 @@
-import json
 import logging
-import re
 from unittest.mock import MagicMock, call, patch
 
 import pytest
@@ -11,7 +9,6 @@
 from scrapy_spider_metadata import get_spider_metadata
 from zyte_common_items import ProbabilityRequest, Product, ProductNavigation, Request
 
-from zyte_spider_templates import BaseSpiderParams
 from zyte_spider_templates._geolocations import (
     GEOLOCATION_OPTIONS,
     GEOLOCATION_OPTIONS_WITH_CODE,
@@ -24,6 +21,7 @@
 
 from . import get_crawler
 from .test_utils import URL_TO_DOMAIN
+from .utils import assertEqualSpiderMetadata
 
 
 def test_parameters():
@@ -362,21 +360,6 @@ def test_arguments():
         assert spider.allowed_domains == ["example.com"]
 
 
-def assertEqualJson(actual, expected):
-    """Compare the JSON representation of 2 Python objects.
-
-    This allows to take into account things like the order of key-value pairs
-    in dictionaries, which would not be taken into account when comparing
-    dictionaries directly.
-
-    It also generates a better diff in pytest output when enums are involved,
-    e.g. geolocation values.
-    """
-    actual_json = json.dumps(actual, indent=2)
-    expected_json = json.dumps(expected, indent=2)
-    assert actual_json == expected_json
-
-
 def test_metadata():
     actual_metadata = get_spider_metadata(EcommerceSpider, normalize=True)
     expected_metadata = {
@@ -480,7 +463,7 @@ def test_metadata():
                             "title": "Pagination Only",
                         },
                     },
-                    "title": "Crawl strategy",
+                    "title": "Crawl Strategy",
                     "enum": [
                         "automatic",
                         "full",
@@ -550,60 +533,14 @@ def test_metadata():
             "type": "object",
         },
     }
-    assertEqualJson(actual_metadata, expected_metadata)
+    assertEqualSpiderMetadata(actual_metadata, expected_metadata)
 
     geolocation = actual_metadata["param_schema"]["properties"]["geolocation"]
     assert geolocation["enum"][0] == "AF"
     assert geolocation["enumMeta"]["UY"] == {"title": "Uruguay (UY)"}
     assert set(geolocation["enum"]) == set(geolocation["enumMeta"])
 
 
-@pytest.mark.parametrize(
-    "valid,url",
-    [
-        (False, ""),
-        (False, "http://"),
-        (False, "http:/example.com"),
-        (False, "ftp://example.com"),
-        (False, "example.com"),
-        (False, "//example.com"),
-        (False, "http://foo:[email protected]"),
-        (False, " http://example.com"),
-        (False, "http://example.com "),
-        (False, "http://examp le.com"),
-        (False, "https://example.com:232323"),
-        (True, "http://example.com"),
-        (True, "http://bücher.example"),
-        (True, "http://xn--bcher-kva.example"),
-        (True, "https://i❤.ws"),
-        (True, "https://example.com"),
-        (True, "https://example.com/"),
-        (True, "https://example.com:2323"),
-        (True, "https://example.com:2323/"),
-        (True, "https://example.com:2323/foo"),
-        (True, "https://example.com/f"),
-        (True, "https://example.com/foo"),
-        (True, "https://example.com/foo/"),
-        (True, "https://example.com/foo/bar"),
-        (True, "https://example.com/foo/bar/"),
-        (True, "https://example.com/foo/bar?baz"),
-        (True, "https://example.com/foo/bar/?baz"),
-        (True, "https://example.com?foo"),
-        (True, "https://example.com?foo=bar"),
-        (True, "https://example.com/?foo=bar&baz"),
-        (True, "https://example.com/?foo=bar&baz#"),
-        (True, "https://example.com/?foo=bar&baz#frag"),
-        (True, "https://example.com#"),
-        (True, "https://example.com/#"),
-        (True, "https://example.com/&"),
-        (True, "https://example.com/&#"),
-    ],
-)
-def test_validation_url(url, valid):
-    url_re = BaseSpiderParams.model_fields["url"].metadata[0].pattern
-    assert bool(re.match(url_re, url)) == valid
-
-
 def test_get_parse_product_request():
     base_kwargs = {
         "url": "https://example.com",
@@ -818,7 +755,7 @@ def test_urls_file():
     crawler = get_crawler()
     url = "https://example.com"
 
-    with patch("zyte_spider_templates.spiders.ecommerce.requests.get") as mock_get:
+    with patch("zyte_spider_templates.params.requests.get") as mock_get:
         response = requests.Response()
         response._content = (
             b"https://a.example\n \nhttps://b.example\nhttps://c.example\n\n"

tests/test_middlewares.py

@@ -36,13 +36,7 @@ def results_gen():
     crawl_logs = middleware.crawl_logs(response, results_gen())
     assert crawl_logs == (
         "Crawling Logs for https://example.com (parsed as: None):\n"
-        "Number of Requests per page type:\n"
-        "- product: 0\n"
-        "- nextPage: 0\n"
-        "- subCategories: 0\n"
-        "- productNavigation: 0\n"
-        "- productNavigation-heuristics: 0\n"
-        "- unknown: 0\n"
+        "Nothing to crawl.\n"
         "Structured Logs:\n"
         "{\n"
         '  "time": "2023-10-10 20:09:29",\n'
@@ -53,14 +47,7 @@ def results_gen():
         '    "page_type": null,\n'
         '    "probability": null\n'
         "  },\n"
-        '  "to_crawl": {\n'
-        '    "product": [],\n'
-        '    "nextPage": [],\n'
-        '    "subCategories": [],\n'
-        '    "productNavigation": [],\n'
-        '    "productNavigation-heuristics": [],\n'
-        '    "unknown": []\n'
-        "  }\n"
+        '  "to_crawl": {}\n'
         "}"
     )
 
@@ -131,15 +118,19 @@ def test_crawling_logs_middleware():
             },
         },
     )
-    unknown_request = Request(
-        "https://example.com/other-unknown",
+    custom_request = Request(
+        "https://example.com/custom-page-type",
         meta={
             "crawling_logs": {
-                "name": "Unknown Page",
+                "name": "Custom Page",
                 "page_type": "some other page_type",
+                "foo": "bar",
             },
         },
     )
+    unknown_request = Request(
+        "https://example.com/other-unknown",
+    )
 
     request_fingerprint = get_fingerprinter(crawler)
     fingerprint = request_fingerprint(request)
@@ -150,6 +141,7 @@ def test_crawling_logs_middleware():
     product_navigation_heuristics_request_fp = request_fingerprint(
         product_navigation_heuristics_request
     )
+    custom_request_fp = request_fingerprint(custom_request)
     unknown_request_fp = request_fingerprint(unknown_request)
 
     def results_gen():
@@ -158,6 +150,7 @@ def results_gen():
         yield subcategory_request
         yield product_navigation_request
         yield product_navigation_heuristics_request
+        yield custom_request
         yield unknown_request
 
     crawl_logs = middleware.crawl_logs(response, results_gen())
@@ -169,6 +162,7 @@ def results_gen():
         "- subCategories: 1\n"
         "- productNavigation: 1\n"
         "- productNavigation-heuristics: 1\n"
+        "- some other page_type: 1\n"
         "- unknown: 1\n"
         "Structured Logs:\n"
         "{\n"
@@ -231,10 +225,18 @@ def results_gen():
         f'        "request_fingerprint": "{product_navigation_heuristics_request_fp}"\n'
         "      }\n"
         "    ],\n"
-        '    "unknown": [\n'
+        '    "some other page_type": [\n'
         "      {\n"
-        '        "name": "Unknown Page",\n'
+        '        "name": "Custom Page",\n'
         '        "page_type": "some other page_type",\n'
+        '        "foo": "bar",\n'
+        '        "request_url": "https://example.com/custom-page-type",\n'
+        '        "request_priority": 0,\n'
+        f'        "request_fingerprint": "{custom_request_fp}"\n'
+        "      }\n"
+        "    ],\n"
+        '    "unknown": [\n'
+        "      {\n"
         '        "request_url": "https://example.com/other-unknown",\n'
         '        "request_priority": 0,\n'
         f'        "request_fingerprint": "{unknown_request_fp}"\n'

tests/test_params.py

@@ -0,0 +1,51 @@
+import re
+
+import pytest
+
+from zyte_spider_templates.params import URL_FIELD_KWARGS
+
+
+@pytest.mark.parametrize(
+    "valid,url",
+    [
+        (False, ""),
+        (False, "http://"),
+        (False, "http:/example.com"),
+        (False, "ftp://example.com"),
+        (False, "example.com"),
+        (False, "//example.com"),
+        (False, "http://foo:[email protected]"),
+        (False, " http://example.com"),
+        (False, "http://example.com "),
+        (False, "http://examp le.com"),
+        (False, "https://example.com:232323"),
+        (True, "http://example.com"),
+        (True, "http://bücher.example"),
+        (True, "http://xn--bcher-kva.example"),
+        (True, "https://i❤.ws"),
+        (True, "https://example.com"),
+        (True, "https://example.com/"),
+        (True, "https://example.com:2323"),
+        (True, "https://example.com:2323/"),
+        (True, "https://example.com:2323/foo"),
+        (True, "https://example.com/f"),
+        (True, "https://example.com/foo"),
+        (True, "https://example.com/foo/"),
+        (True, "https://example.com/foo/bar"),
+        (True, "https://example.com/foo/bar/"),
+        (True, "https://example.com/foo/bar?baz"),
+        (True, "https://example.com/foo/bar/?baz"),
+        (True, "https://example.com?foo"),
+        (True, "https://example.com?foo=bar"),
+        (True, "https://example.com/?foo=bar&baz"),
+        (True, "https://example.com/?foo=bar&baz#"),
+        (True, "https://example.com/?foo=bar&baz#frag"),
+        (True, "https://example.com#"),
+        (True, "https://example.com/#"),
+        (True, "https://example.com/&"),
+        (True, "https://example.com/&#"),
+    ],
+)
+def test_url_pattern(url, valid):
+    assert isinstance(URL_FIELD_KWARGS["pattern"], str)
+    assert bool(re.match(URL_FIELD_KWARGS["pattern"], url)) == valid