base.py

import attrs
from web_poet import ItemPage, RequestUrl, WebPage, field
from web_poet.pages import ItemT

from .._dateutils import utcnow_formatted
from ..processors import metadata_processor, string_processor
from .mixins import HasMetadata, MetadataT


class _BasePage(ItemPage[ItemT], HasMetadata[MetadataT]):
    class Processors:
        metadata = [metadata_processor]

    @field
    def metadata(self) -> MetadataT:
        if self.metadata_cls is None:
            raise ValueError(f"{type(self)} doesn'have a metadata class configured.")
        value = self.metadata_cls()
        attributes = dir(value)
        if "dateDownloaded" in attributes:
            value.dateDownloaded = utcnow_formatted()  # type: ignore
        if "probability" in attributes:
            value.probability = 1.0  # type: ignore
        return value

    def no_item_found(self) -> ItemT:
        """Return an item with the current url and probability=0,
        indicating that the passed URL doesn't contain the expected item.

        Use it in your .validate_input implementation.
        """
        if self.metadata_cls is None:
            raise ValueError(f"{type(self)} doesn'have a metadata class configured.")
        metadata = self.metadata_cls()
        metadata_attributes = dir(metadata)
        if "dateDownloaded" in metadata_attributes:
            metadata.dateDownloaded = utcnow_formatted()  # type: ignore
        if "probability" in metadata_attributes:
            metadata.probability = 0.0  # type: ignore
        return self.item_cls(  # type: ignore
            url=self.url,  # type: ignore[attr-defined]
            metadata=metadata,
        )


class _BaseProcessors(_BasePage.Processors):
    url = [string_processor]


@attrs.define
class BasePage(_BasePage):
    """Base class for page object classes that has
    :class:`~web_poet.page_inputs.http.RequestUrl` as a dependency."""

    class Processors(_BaseProcessors):
        pass

    request_url: RequestUrl

    @field
    def url(self) -> str:
        return str(self.request_url)


@attrs.define
class Page(_BasePage, WebPage):
    """Base class for page object classes that has
    :class:`~web_poet.page_inputs.http.HttpResponse` as a dependency."""

    class Processors(_BaseProcessors):
        pass

    @field
    def url(self) -> str:
        return str(self.response.url)