Skip to content

Commit 2b10d96

Browse files
committed
perf: take component sets from the listing
Calendars piled up on a Synology server, as the caldav library's tests added calendars but could not delete them. With half a thousand of them, listing went really slow: collections() sent one PROPFIND per calendar, 268 s against 6 s for the listing alone. The calendars were deleted some other way, but the number of PROPFINDs was excessive anyway. caldav (from python-caldav/caldav#872, not yet released) keeps the listing's properties on each calendar, and the component sets are now read from there. Bedework 5 also lists "ghost collections" that do not exist. A compatibility hint for this has been added to the caldav library, and where it is set, such ghost collections are weeded out. Prompts: > [pick from the task list: "calendaring: list collections with one > PROPFIND, not one per calendar"] > so let's go for yet another caldav compatibility hint on the ghost > collections > (on the commit message) Please do a QA on the edited text. Didn't > we already from earlier cache propfind results in the caldav library? Assisted-By: Claude Opus 5.5 Reviewed-by: Tobias Brox <tobias@redpill-linpro.com>
1 parent 605f153 commit 2b10d96

9 files changed

Lines changed: 162 additions & 10 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
On CalDAV, ``collections()`` no longer sends one PROPFIND per calendar for its component set, where caldav keeps the listing's properties (after 3.7): 469 calendars on a Synology took 268 s, the listing alone 6 s. With caldav 3.7, or a server whose listing leaves the component set out, each calendar is still asked; so is each on a server whose listing names calendars that are not there (Bedework 5, by caldav's ``propfind.listed-members-exist`` hint), to leave those out. That takes caldav's Bedework 5 profile (``features: bedework_5_0_0``): without hints, Bedework 5's ghost ``tasks/`` is listed.

‎src/calendaring/_async/caldav.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from .._caldav_features import (
2525
backend_capabilities,
2626
collection_capabilities,
27+
lists_missing_collections,
2728
multiget_serves_deleted,
2829
stored_components,
2930
)
@@ -272,11 +273,14 @@ async def _open(cls, url: str, **options: Any) -> AsyncCalDAVBackend:
272273
return cls(AsyncDAV(url, **options), id=backend_id, loss=loss, verify=verify)
273274

274275
async def _wrap(self, calendar: Calendar) -> AsyncCalDAVCollection:
276+
features = self._dav.client.features
277+
# Where the listing names calendars that are not there, a fresh read of each is how they are found.
278+
name = await self._dav.display_name(calendar, cached=not lists_missing_collections(features))
275279
listed = frozenset(
276280
Component(c) for c in await self._dav.supported_components(calendar) if c in {k.value for k in Component}
277281
)
278-
components = stored_components(self._dav.client.features, listed)
279-
return self.collection_class(self, calendar, name=await self._dav.display_name(calendar), components=components)
282+
components = stored_components(features, listed)
283+
return self.collection_class(self, calendar, name=name, components=components)
280284

281285
async def collections(self) -> list[AsyncCollection]:
282286
found: list[AsyncCollection] = []

‎src/calendaring/_async/dav.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
SyncResult,
1616
_awaited,
1717
_etags,
18+
_listed_components,
1819
_raw,
1920
_resource,
2021
_strong,
@@ -57,12 +58,18 @@ async def delete_calendar(self, calendar: Calendar) -> None:
5758
await _awaited(calendar.delete()) # type: ignore[no-untyped-call] # caldav leaves delete() unannotated
5859

5960
async def supported_components(self, calendar: Calendar) -> list[str]:
61+
# From the listing if it said, or one PROPFIND per calendar: 268 s for 469 on a Synology.
62+
if (listed := _listed_components(calendar)) is not None:
63+
return listed
6064
with translated():
6165
return [str(c) for c in await _awaited(calendar.get_supported_components())]
6266

63-
async def display_name(self, calendar: Calendar) -> str | None:
67+
async def display_name(self, calendar: Calendar, *, cached: bool = True) -> str | None:
68+
"""The listed name, or with ``cached=False`` a fresh one: ``NotFoundError`` if the calendar is gone."""
6469
with translated():
65-
return await _awaited(calendar.get_display_name())
70+
if cached:
71+
return await _awaited(calendar.get_display_name())
72+
return await _awaited(calendar.get_property(DisplayName(), use_cached=False))
6673

6774
async def set_display_name(self, calendar: Calendar, name: str) -> None:
6875
with translated():

‎src/calendaring/_caldav_features.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,11 @@ def multiget_serves_deleted(features: Any) -> bool:
288288
return _level(features, "multiget.deleted") is U
289289

290290

291+
def lists_missing_collections(features: Any) -> bool:
292+
"""Bedework 5 lists a calendar, display name and component set and all, that answers 404."""
293+
return _level(features, "propfind.listed-members-exist") is U
294+
295+
291296
def backend_capabilities(features: Any) -> Capabilities:
292297
return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES})
293298

‎src/calendaring/_caldav_io.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
from caldav.calendarobjectresource import CalendarObjectResource, Event, Journal, Todo
2828
from caldav.collection import Calendar
2929
from caldav.davclient import DAVClient
30+
from caldav.elements.cdav import SupportedCalendarComponentSet
3031
from caldav.elements.dav import DisplayName, GetEtag
3132
from caldav.lib import error as dav_error
3233
from caldav.lib.url import URL
@@ -69,6 +70,15 @@
6970

7071
VALID_SYNC_TOKEN = "{DAV:}valid-sync-token" # RFC 6578 §3.2
7172

73+
74+
def _listed_components(calendar: Calendar) -> list[str] | None:
75+
"""The component set the calendar listing answered, if caldav kept it (after 3.7): None to ask.
76+
77+
Read from ``props`` rather than ``get_supported_components(use_cached=True)``, which 3.7 does not take."""
78+
listed = calendar.props.get(SupportedCalendarComponentSet.tag)
79+
return [str(c) for c in listed] if listed else None
80+
81+
7282
# A sync-collection report: hrefs changed, hrefs removed, the new token (None if the server gave
7383
# none), and whether it is incomplete (truncated beyond what caldav itself follows, RFC 6578 §3.6).
7484
SyncResult = tuple[list[str], list[str], str | None, bool]

‎src/calendaring/_sync/caldav.py‎

Lines changed: 6 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/calendaring/_sync/dav.py‎

Lines changed: 9 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/_async/test_caldav_backend.py‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,13 @@
3131

3232
from caldav.collection import Calendar # noqa: E402
3333
from caldav.compatibility_hints import FeatureSet # noqa: E402
34+
from caldav.elements.cdav import SupportedCalendarComponentSet # noqa: E402
35+
from caldav.elements.dav import DisplayName # noqa: E402
3436
from caldav.lib.url import URL # noqa: E402
3537

3638
from calendaring._async.caldav import _name_of # noqa: E402
37-
from calendaring._caldav_io import _resource # noqa: E402
39+
from calendaring._caldav_features import lists_missing_collections # noqa: E402
40+
from calendaring._caldav_io import _listed_components, _resource # noqa: E402
3841

3942

4043
@pytest.fixture
@@ -90,10 +93,64 @@ async def test_a_listed_calendar_that_is_not_there_is_left_out(
9093
assert len(await backend.collections()) == len(real)
9194

9295

96+
async def test_a_listed_calendar_that_is_not_there_is_left_out_where_the_listing_says_otherwise(
97+
collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch
98+
) -> None:
99+
"""Bedework 5 lists its ghost ``tasks/`` with a display name and a component set, all a listing could say."""
100+
caldav_collection: Any = collection
101+
backend = caldav_collection._backend
102+
features = backend._dav.client.features
103+
if "propfind.listed-members-exist" not in features.FEATURES:
104+
pytest.skip("caldav up to 3.7 has no propfind.listed-members-exist hint")
105+
features.set_feature("propfind.listed-members-exist", {"support": "broken"})
106+
real = await backend._dav.calendars()
107+
ghost = Calendar(
108+
client=backend._dav.client,
109+
url=URL.objectify(caldav_collection.url).join("../ghost/"),
110+
props={DisplayName.tag: "tasks", SupportedCalendarComponentSet.tag: ["VTODO"]},
111+
)
112+
monkeypatch.setattr(backend._dav, "calendars", lambda: _returning([*real, ghost]))
113+
assert [c.id for c in await backend.collections()] == [_name_of(str(c.url)) for c in real]
114+
115+
93116
async def _returning(value: Any) -> Any:
94117
return value
95118

96119

120+
async def test_listing_collections_costs_the_same_however_many_there_are(
121+
collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch
122+
) -> None:
123+
"""One PROPFIND per calendar took 268 s for 469 calendars on a Synology; the listing alone takes 6 s."""
124+
backend: Any = collection._backend
125+
if not all(_listed_components(c) for c in await backend._dav.calendars()):
126+
pytest.skip("the listing kept no component set: caldav up to 3.7 keeps only the display name")
127+
if lists_missing_collections(backend._dav.client.features):
128+
pytest.skip("where the listing names calendars that are not there, each is asked")
129+
client = backend._dav.client
130+
request = client.request
131+
methods: list[str] = []
132+
133+
async def counted(url: Any, method: str = "GET", *args: Any, **kwargs: Any) -> Any:
134+
methods.append(method)
135+
return await request(url, method, *args, **kwargs)
136+
137+
monkeypatch.setattr(client, "request", counted)
138+
one = await backend.collections()
139+
one_propfinds, methods[:] = methods.count("PROPFIND"), []
140+
assert one_propfinds >= 1 # the counting sees the requests at all
141+
monkeypatch.setattr(client, "request", request)
142+
other = await backend.create_collection("backend-tests-2", components=(Component.TASK,))
143+
try:
144+
monkeypatch.setattr(client, "request", counted)
145+
two = await backend.collections()
146+
assert len(two) == len(one) + 1
147+
assert methods.count("PROPFIND") == one_propfinds
148+
assert {c.id: c.components for c in two}[collection.id] == {Component.TASK}
149+
finally:
150+
monkeypatch.setattr(client, "request", request)
151+
await other.delete_collection()
152+
153+
97154
async def test_a_token_is_bearer_auth() -> None:
98155
backend = await AsyncBackend.open("https://dav.example.com/", kind="caldav", token="s3cret")
99156
async with backend:

‎tests/_sync/test_caldav_backend.py‎

Lines changed: 58 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)