docs: Add @docs_group decorator (#655)

Adds support for the `@docs_group` decorator, which sets the symbol's
group in the documentation.

Adds the decorators as per the list in #650 .

Hides the undecorated symbols under the project root (but keeps those in
classes etc., so class / object methods + properties are still
documented).


![obrazek](https://github.com/user-attachments/assets/f196f75b-015e-4247-a6bf-f062cb93e97a)

Closes #650

---------

Co-authored-by: Vlada Dusek <[email protected]>

Author: barjin

src/crawlee/_autoscaling/autoscaled_pool.py

@@ -10,6 +10,7 @@
 from typing import TYPE_CHECKING, Awaitable, Callable
 
 from crawlee._types import ConcurrencySettings
+from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
 
 if TYPE_CHECKING:
@@ -32,6 +33,7 @@ def __init__(self) -> None:
         self.result: asyncio.Future = asyncio.Future()
 
 
+@docs_group('Classes')
 class AutoscaledPool:
     """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
 

src/crawlee/_autoscaling/snapshotter.py

@@ -10,6 +10,7 @@
 
 from crawlee._autoscaling.types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, MemorySnapshot, Snapshot
 from crawlee._utils.byte_size import ByteSize
+from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
 from crawlee.events._types import Event, EventSystemInfoData
 
@@ -21,6 +22,7 @@
 logger = getLogger(__name__)
 
 
+@docs_group('Classes')
 class Snapshotter:
     """Monitors and logs system resource usage at predefined intervals for performance optimization.
 

src/crawlee/_autoscaling/system_status.py

@@ -9,6 +9,7 @@
 from more_itertools import pairwise
 
 from crawlee._autoscaling.types import LoadRatioInfo, Snapshot, SystemInfo
+from crawlee._utils.docs import docs_group
 from crawlee._utils.math import compute_weighted_avg
 
 if TYPE_CHECKING:
@@ -17,6 +18,7 @@
 logger = getLogger(__name__)
 
 
+@docs_group('Classes')
 class SystemStatus:
     """Provides a simple interface for evaluating system resource usage from snapshots collected by `Snapshotter`.
 

src/crawlee/_request.py

@@ -21,6 +21,7 @@
 
 from crawlee._types import EnqueueStrategy, HttpHeaders, HttpMethod, HttpPayload, JsonSerializable
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.docs import docs_group
 from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
 from crawlee._utils.urls import extract_query_params, validate_http_url
 
@@ -227,6 +228,7 @@ def get_query_param_from_url(self, param: str, *, default: str | None = None) ->
         return values[0]
 
 
+@docs_group('Data structures')
 class Request(BaseRequestData):
     """Represents a request in the Crawlee framework, containing the necessary information for crawling operations.
 

src/crawlee/_types.py

@@ -19,6 +19,8 @@
 from pydantic import ConfigDict, Field, PlainValidator, RootModel
 from typing_extensions import NotRequired, TypeAlias, TypedDict, Unpack
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     import logging
     import re
@@ -97,6 +99,7 @@ def __len__(self) -> int:
         return len(self.root)
 
 
+@docs_group('Data structures')
 class EnqueueStrategy(str, Enum):
     """Strategy for deciding which links should be followed and which ones should be ignored."""
 
@@ -117,6 +120,7 @@ class EnqueueStrategy(str, Enum):
     the same protocol, domain, and port, ensuring a strict scope for the crawl."""
 
 
+@docs_group('Data structures')
 class ConcurrencySettings:
     """Concurrency settings for AutoscaledPool."""
 
@@ -333,6 +337,7 @@ def __call__(
 
 
 @dataclass(frozen=True)
+@docs_group('Data structures')
 class BasicCrawlingContext:
     """Basic crawling context intended to be extended by crawlers."""
 

src/crawlee/_utils/docs.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import Callable, Literal
+
+GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Errors', 'Functions']
+
+
+def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001
+    """Decorator to mark symbols for rendering and grouping in documentation.
+
+    This decorator is used purely for documentation purposes and does not alter the behavior
+    of the decorated callable.
+    """
+
+    def wrapper(func: Callable) -> Callable:
+        return func
+
+    return wrapper

src/crawlee/base_storage_client/_base_dataset_client.py

@@ -3,13 +3,16 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, AsyncContextManager, AsyncIterator
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from httpx import Response
 
     from crawlee._types import JsonSerializable
     from crawlee.base_storage_client._models import DatasetItemsListPage, DatasetMetadata
 
 
+@docs_group('Abstract classes')
 class BaseDatasetClient(ABC):
     """Abstract base class for dataset resource clients.
 

src/crawlee/base_storage_client/_base_dataset_collection_client.py

@@ -3,10 +3,13 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from crawlee.base_storage_client._models import DatasetListPage, DatasetMetadata
 
 
+@docs_group('Abstract classes')
 class BaseDatasetCollectionClient(ABC):
     """Abstract base class for dataset collection clients.
 

src/crawlee/base_storage_client/_base_key_value_store_client.py

@@ -3,6 +3,8 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, AsyncContextManager
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from httpx import Response
 
@@ -13,6 +15,7 @@
     )
 
 
+@docs_group('Abstract classes')
 class BaseKeyValueStoreClient(ABC):
     """Abstract base class for key-value store resource clients.
 

src/crawlee/base_storage_client/_base_key_value_store_collection_client.py

@@ -3,10 +3,13 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING
 
+from crawlee._utils.docs import docs_group
+
 if TYPE_CHECKING:
     from crawlee.base_storage_client._models import KeyValueStoreListPage, KeyValueStoreMetadata
 
 
+@docs_group('Abstract classes')
 class BaseKeyValueStoreCollectionClient(ABC):
     """Abstract base class for key-value store collection clients.