docs: Add @docs_group decorator (#655)

Adds support for the `@docs_group` decorator, which sets the symbol's
group in the documentation.

Adds the decorators as per the list in #650 .

Hides the undecorated symbols under the project root (but keeps those in
classes etc., so class / object methods + properties are still
documented).


![obrazek](https://github.com/user-attachments/assets/f196f75b-015e-4247-a6bf-f062cb93e97a)

Closes #650

---------

Co-authored-by: Vlada Dusek <[email protected]>

Author: barjin

src/crawlee/_autoscaling/autoscaled_pool.py

@@ -10,6 +10,7 @@
 from typing import TYPE_CHECKING, Awaitable, Callable
 
 from crawlee._types import ConcurrencySettings
+from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
 
 if TYPE_CHECKING:
@@ -32,6 +33,7 @@ def __init__(self) -> None:
         self.result: asyncio.Future = asyncio.Future()
 
 
+@docs_group('Classes')
 class AutoscaledPool:
     """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
 

src/crawlee/_autoscaling/snapshotter.py

@@ -10,6 +10,7 @@
 
 from crawlee._autoscaling.types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, MemorySnapshot, Snapshot
 from crawlee._utils.byte_size import ByteSize
+from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
 from crawlee.events._types import Event, EventSystemInfoData
 
@@ -21,6 +22,7 @@
 logger = getLogger(__name__)
 
 
+@docs_group('Classes')
 class Snapshotter:
     """Monitors and logs system resource usage at predefined intervals for performance optimization.
 

src/crawlee/_autoscaling/system_status.py

@@ -9,6 +9,7 @@
 from more_itertools import pairwise
 
 from crawlee._autoscaling.types import LoadRatioInfo, Snapshot, SystemInfo
+from crawlee._utils.docs import docs_group
 from crawlee._utils.math import compute_weighted_avg
 
 if TYPE_CHECKING:
@@ -17,6 +18,7 @@
 logger = getLogger(__name__)
 
 
+@docs_group('Classes')
 class SystemStatus:
     """Provides a simple interface for evaluating system resource usage from snapshots collected by `Snapshotter`.
 

src/crawlee/_request.py

@@ -21,6 +21,7 @@
 
 from crawlee._types import EnqueueStrategy, HttpHeaders, HttpMethod, HttpPayload, JsonSerializable
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.docs import docs_group
 from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
 from crawlee._utils.urls import extract_query_params, validate_http_url
 
@@ -227,6 +228,7 @@ def get_query_param_from_url(self, param: str, *, default: str | None = None) ->
         return values[0]
 
 
+@docs_group('Data structures')
 class Request(BaseRequestData):
     """Represents a request in the Crawlee framework, containing the necessary information for crawling operations.
 

src/crawlee/_types.py

@@ -19,6 +19,8 @@
 from pydantic import ConfigDict, Field, PlainValidator, RootModel
 from typing_extensions import NotRequired, TypeAlias, TypedDict, Unpack
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     import logging
     import re
@@ -97,6 +99,7 @@ def __len__(self) -> int:
         return len(self.root)
 
 
+@docs_group('Data structures')
 class EnqueueStrategy(str, Enum):
     """Strategy for deciding which links should be followed and which ones should be ignored."""
 
@@ -117,6 +120,7 @@ class EnqueueStrategy(str, Enum):
     the same protocol, domain, and port, ensuring a strict scope for the crawl."""
 
 
+@docs_group('Data structures')
 class ConcurrencySettings:
     """Concurrency settings for AutoscaledPool."""
 
@@ -333,6 +337,7 @@ def __call__(
 
 
 @dataclass(frozen=True)
+@docs_group('Data structures')
 class BasicCrawlingContext:
     """Basic crawling context intended to be extended by crawlers."""
 

src/crawlee/_utils/docs.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import Callable, Literal
+
+GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Errors', 'Functions']
+
+
+def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001
+    """Decorator to mark symbols for rendering and grouping in documentation.
+
+    This decorator is used purely for documentation purposes and does not alter the behavior
+    of the decorated callable.
+    """
+
+    def wrapper(func: Callable) -> Callable:
+        return func
+
+    return wrapper

src/crawlee/base_storage_client/_base_dataset_client.py

@@ -3,13 +3,16 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, AsyncContextManager, AsyncIterator
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from httpx import Response
 
     from crawlee._types import JsonSerializable
     from crawlee.base_storage_client._models import DatasetItemsListPage, DatasetMetadata
 
 
+@docs_group('Abstract classes')
 class BaseDatasetClient(ABC):
     """Abstract base class for dataset resource clients.
 

src/crawlee/base_storage_client/_base_dataset_collection_client.py

@@ -3,10 +3,13 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from crawlee.base_storage_client._models import DatasetListPage, DatasetMetadata
 
 
+@docs_group('Abstract classes')
 class BaseDatasetCollectionClient(ABC):
     """Abstract base class for dataset collection clients.
 

src/crawlee/base_storage_client/_base_key_value_store_client.py

@@ -3,6 +3,8 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, AsyncContextManager
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from httpx import Response
 
@@ -13,6 +15,7 @@
     )
 
 
+@docs_group('Abstract classes')
 class BaseKeyValueStoreClient(ABC):
     """Abstract base class for key-value store resource clients.
 

src/crawlee/base_storage_client/_base_key_value_store_collection_client.py

@@ -3,10 +3,13 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from crawlee.base_storage_client._models import KeyValueStoreListPage, KeyValueStoreMetadata
 
 
+@docs_group('Abstract classes')
 class BaseKeyValueStoreCollectionClient(ABC):
     """Abstract base class for key-value store collection clients.
 

src/crawlee/base_storage_client/_base_request_queue_client.py

@@ -3,6 +3,8 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
@@ -17,6 +19,7 @@
     )
 
 
+@docs_group('Abstract classes')
 class BaseRequestQueueClient(ABC):
     """Abstract base class for request queue resource clients.
 

src/crawlee/base_storage_client/_base_request_queue_collection_client.py

@@ -3,10 +3,13 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from crawlee.base_storage_client._models import RequestQueueListPage, RequestQueueMetadata
 
 
+@docs_group('Abstract classes')
 class BaseRequestQueueCollectionClient(ABC):
     """Abstract base class for request queue collection clients.
 

src/crawlee/base_storage_client/_base_storage_client.py

@@ -5,6 +5,8 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from ._base_dataset_client import BaseDatasetClient
     from ._base_dataset_collection_client import BaseDatasetCollectionClient
@@ -14,6 +16,7 @@
     from ._base_request_queue_collection_client import BaseRequestQueueCollectionClient
 
 
+@docs_group('Abstract classes')
 class BaseStorageClient(ABC):
     """Defines an abstract base for storage clients.
 

src/crawlee/base_storage_client/_models.py

@@ -10,6 +10,7 @@
 
 from crawlee._request import Request
 from crawlee._types import HttpMethod
+from crawlee._utils.docs import docs_group
 from crawlee._utils.urls import validate_http_url
 
 KvsValueType = TypeVar('KvsValueType', default=Any)
@@ -27,6 +28,7 @@ class _BaseStorageMetadata(BaseModel):
     modified_at: Annotated[datetime, Field(alias='modifiedAt')]
 
 
+@docs_group('Data structures')
 class DatasetMetadata(_BaseStorageMetadata):
     """Model for a dataset metadata."""
 
@@ -35,6 +37,7 @@ class DatasetMetadata(_BaseStorageMetadata):
     item_count: Annotated[int, Field(alias='itemCount')]
 
 
+@docs_group('Data structures')
 class KeyValueStoreMetadata(_BaseStorageMetadata):
     """Model for a key-value store metadata."""
 
@@ -43,6 +46,7 @@ class KeyValueStoreMetadata(_BaseStorageMetadata):
     user_id: Annotated[str, Field(alias='userId')]
 
 
+@docs_group('Data structures')
 class RequestQueueMetadata(_BaseStorageMetadata):
     """Model for a request queue metadata."""
 
@@ -57,6 +61,7 @@ class RequestQueueMetadata(_BaseStorageMetadata):
     resource_directory: Annotated[str, Field(alias='resourceDirectory')]
 
 
+@docs_group('Data structures')
 class KeyValueStoreRecord(BaseModel, Generic[KvsValueType]):
     """Model for a key-value store record."""
 
@@ -68,6 +73,7 @@ class KeyValueStoreRecord(BaseModel, Generic[KvsValueType]):
     filename: Annotated[str | None, Field(alias='filename', default=None)]
 
 
+@docs_group('Data structures')
 class KeyValueStoreRecordMetadata(BaseModel):
     """Model for a key-value store record metadata."""
 
@@ -77,6 +83,7 @@ class KeyValueStoreRecordMetadata(BaseModel):
     content_type: Annotated[str, Field(alias='contentType')]
 
 
+@docs_group('Data structures')
 class KeyValueStoreKeyInfo(BaseModel):
     """Model for a key-value store key info."""
 
@@ -86,6 +93,7 @@ class KeyValueStoreKeyInfo(BaseModel):
     size: Annotated[int, Field(alias='size')]
 
 
+@docs_group('Data structures')
 class KeyValueStoreListKeysPage(BaseModel):
     """Model for listing keys in the key-value store."""
 
@@ -99,6 +107,7 @@ class KeyValueStoreListKeysPage(BaseModel):
     next_exclusive_start_key: Annotated[str | None, Field(alias='nextExclusiveStartKey', default=None)]
 
 
+@docs_group('Data structures')
 class RequestQueueHeadState(BaseModel):
     """Model for the request queue head state."""
 
@@ -111,6 +120,7 @@ class RequestQueueHeadState(BaseModel):
     had_multiple_clients: Annotated[bool, Field(alias='hadMultipleClients')]
 
 
+@docs_group('Data structures')
 class RequestQueueHead(BaseModel):
     """Model for the request queue head."""
 
@@ -122,6 +132,7 @@ class RequestQueueHead(BaseModel):
     items: Annotated[list[Request], Field(alias='items', default_factory=list)]
 
 
+@docs_group('Data structures')
 class RequestQueueHeadWithLocks(RequestQueueHead):
     """Model for request queue head with locks."""
 
@@ -149,34 +160,39 @@ class _BaseListPage(BaseModel):
     """Indicates if the returned list is in descending order."""
 
 
+@docs_group('Data structures')
 class DatasetListPage(_BaseListPage):
     """Model for a single page of dataset items returned from a collection list method."""
 
     items: Annotated[list[DatasetMetadata], Field(default_factory=list)]
     """The list of dataset items returned on this page."""
 
 
+@docs_group('Data structures')
 class KeyValueStoreListPage(_BaseListPage):
     """Model for a single page of key-value store items returned from a collection list method."""
 
     items: Annotated[list[KeyValueStoreMetadata], Field(default_factory=list)]
     """The list of key-value store items returned on this page."""
 
 
+@docs_group('Data structures')
 class RequestQueueListPage(_BaseListPage):
     """Model for a single page of request queue items returned from a collection list method."""
 
     items: Annotated[list[RequestQueueMetadata], Field(default_factory=list)]
     """The list of request queue items returned on this page."""
 
 
+@docs_group('Data structures')
 class DatasetItemsListPage(_BaseListPage):
     """Model for a single page of dataset items returned from a collection list method."""
 
     items: Annotated[list[dict], Field(default_factory=list)]
     """The list of dataset items returned on this page."""
 
 
+@docs_group('Data structures')
 class ProlongRequestLockResponse(BaseModel):
     """Response to prolong request lock calls."""
 
@@ -185,6 +201,7 @@ class ProlongRequestLockResponse(BaseModel):
     lock_expires_at: Annotated[datetime, Field(alias='lockExpiresAt')]
 
 
+@docs_group('Data structures')
 class ProcessedRequest(BaseModel):
     """Represents a processed request."""
 
@@ -196,6 +213,7 @@ class ProcessedRequest(BaseModel):
     was_already_handled: Annotated[bool, Field(alias='wasAlreadyHandled')]
 
 
+@docs_group('Data structures')
 class UnprocessedRequest(BaseModel):
     """Represents an unprocessed request."""
 
@@ -206,6 +224,7 @@ class UnprocessedRequest(BaseModel):
     method: Annotated[HttpMethod | None, Field()] = None
 
 
+@docs_group('Data structures')
 class BatchRequestsOperationResponse(BaseModel):
     """Response to batch request deletion calls."""