Skip to content

Commit 9ce6e14

Browse files
barjinvdusek
andauthored
docs: Add @docs_group decorator (#655)
Adds support for the `@docs_group` decorator, which sets the symbol's group in the documentation. Adds the decorators as per the list in #650 . Hides the undecorated symbols under the project root (but keeps those in classes etc., so class / object methods + properties are still documented). ![obrazek](https://github.com/user-attachments/assets/f196f75b-015e-4247-a6bf-f062cb93e97a) Closes #650 --------- Co-authored-by: Vlada Dusek <[email protected]>
1 parent d26aeb7 commit 9ce6e14

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+199
-25
lines changed

src/crawlee/_autoscaling/autoscaled_pool.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from typing import TYPE_CHECKING, Awaitable, Callable
1111

1212
from crawlee._types import ConcurrencySettings
13+
from crawlee._utils.docs import docs_group
1314
from crawlee._utils.recurring_task import RecurringTask
1415

1516
if TYPE_CHECKING:
@@ -32,6 +33,7 @@ def __init__(self) -> None:
3233
self.result: asyncio.Future = asyncio.Future()
3334

3435

36+
@docs_group('Classes')
3537
class AutoscaledPool:
3638
"""Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
3739

src/crawlee/_autoscaling/snapshotter.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
from crawlee._autoscaling.types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, MemorySnapshot, Snapshot
1212
from crawlee._utils.byte_size import ByteSize
13+
from crawlee._utils.docs import docs_group
1314
from crawlee._utils.recurring_task import RecurringTask
1415
from crawlee.events._types import Event, EventSystemInfoData
1516

@@ -21,6 +22,7 @@
2122
logger = getLogger(__name__)
2223

2324

25+
@docs_group('Classes')
2426
class Snapshotter:
2527
"""Monitors and logs system resource usage at predefined intervals for performance optimization.
2628

src/crawlee/_autoscaling/system_status.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from more_itertools import pairwise
1010

1111
from crawlee._autoscaling.types import LoadRatioInfo, Snapshot, SystemInfo
12+
from crawlee._utils.docs import docs_group
1213
from crawlee._utils.math import compute_weighted_avg
1314

1415
if TYPE_CHECKING:
@@ -17,6 +18,7 @@
1718
logger = getLogger(__name__)
1819

1920

21+
@docs_group('Classes')
2022
class SystemStatus:
2123
"""Provides a simple interface for evaluating system resource usage from snapshots collected by `Snapshotter`.
2224

src/crawlee/_request.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
from crawlee._types import EnqueueStrategy, HttpHeaders, HttpMethod, HttpPayload, JsonSerializable
2323
from crawlee._utils.crypto import crypto_random_object_id
24+
from crawlee._utils.docs import docs_group
2425
from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
2526
from crawlee._utils.urls import extract_query_params, validate_http_url
2627

@@ -227,6 +228,7 @@ def get_query_param_from_url(self, param: str, *, default: str | None = None) ->
227228
return values[0]
228229

229230

231+
@docs_group('Data structures')
230232
class Request(BaseRequestData):
231233
"""Represents a request in the Crawlee framework, containing the necessary information for crawling operations.
232234

src/crawlee/_types.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
from pydantic import ConfigDict, Field, PlainValidator, RootModel
2020
from typing_extensions import NotRequired, TypeAlias, TypedDict, Unpack
2121

22+
from crawlee._utils.docs import docs_group
23+
2224
if TYPE_CHECKING:
2325
import logging
2426
import re
@@ -97,6 +99,7 @@ def __len__(self) -> int:
9799
return len(self.root)
98100

99101

102+
@docs_group('Data structures')
100103
class EnqueueStrategy(str, Enum):
101104
"""Strategy for deciding which links should be followed and which ones should be ignored."""
102105

@@ -117,6 +120,7 @@ class EnqueueStrategy(str, Enum):
117120
the same protocol, domain, and port, ensuring a strict scope for the crawl."""
118121

119122

123+
@docs_group('Data structures')
120124
class ConcurrencySettings:
121125
"""Concurrency settings for AutoscaledPool."""
122126

@@ -333,6 +337,7 @@ def __call__(
333337

334338

335339
@dataclass(frozen=True)
340+
@docs_group('Data structures')
336341
class BasicCrawlingContext:
337342
"""Basic crawling context intended to be extended by crawlers."""
338343

src/crawlee/_utils/docs.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from __future__ import annotations
2+
3+
from typing import Callable, Literal
4+
5+
GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Errors', 'Functions']
6+
7+
8+
def docs_group(group_name: GroupName) -> Callable: # noqa: ARG001
9+
"""Decorator to mark symbols for rendering and grouping in documentation.
10+
11+
This decorator is used purely for documentation purposes and does not alter the behavior
12+
of the decorated callable.
13+
"""
14+
15+
def wrapper(func: Callable) -> Callable:
16+
return func
17+
18+
return wrapper

src/crawlee/base_storage_client/_base_dataset_client.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33
from abc import ABC, abstractmethod
44
from typing import TYPE_CHECKING, AsyncContextManager, AsyncIterator
55

6+
from crawlee._utils.docs import docs_group
7+
68
if TYPE_CHECKING:
79
from httpx import Response
810

911
from crawlee._types import JsonSerializable
1012
from crawlee.base_storage_client._models import DatasetItemsListPage, DatasetMetadata
1113

1214

15+
@docs_group('Abstract classes')
1316
class BaseDatasetClient(ABC):
1417
"""Abstract base class for dataset resource clients.
1518

src/crawlee/base_storage_client/_base_dataset_collection_client.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,13 @@
33
from abc import ABC, abstractmethod
44
from typing import TYPE_CHECKING
55

6+
from crawlee._utils.docs import docs_group
7+
68
if TYPE_CHECKING:
79
from crawlee.base_storage_client._models import DatasetListPage, DatasetMetadata
810

911

12+
@docs_group('Abstract classes')
1013
class BaseDatasetCollectionClient(ABC):
1114
"""Abstract base class for dataset collection clients.
1215

src/crawlee/base_storage_client/_base_key_value_store_client.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
from abc import ABC, abstractmethod
44
from typing import TYPE_CHECKING, Any, AsyncContextManager
55

6+
from crawlee._utils.docs import docs_group
7+
68
if TYPE_CHECKING:
79
from httpx import Response
810

@@ -13,6 +15,7 @@
1315
)
1416

1517

18+
@docs_group('Abstract classes')
1619
class BaseKeyValueStoreClient(ABC):
1720
"""Abstract base class for key-value store resource clients.
1821

src/crawlee/base_storage_client/_base_key_value_store_collection_client.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,13 @@
33
from abc import ABC, abstractmethod
44
from typing import TYPE_CHECKING
55

6+
from crawlee._utils.docs import docs_group
7+
68
if TYPE_CHECKING:
79
from crawlee.base_storage_client._models import KeyValueStoreListPage, KeyValueStoreMetadata
810

911

12+
@docs_group('Abstract classes')
1013
class BaseKeyValueStoreCollectionClient(ABC):
1114
"""Abstract base class for key-value store collection clients.
1215

0 commit comments

Comments
 (0)