Skip to content

Commit 2f36a4c

Browse files
Wizard1209claude
andcommitted
Batch expired record processing in tezos domains demo
Process expired records in batches of 1000 instead of loading all at once to avoid memory issues with large datasets. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 3812602 commit 2f36a4c

1 file changed

Lines changed: 31 additions & 15 deletions

File tree

src/demo_tezos_domains/hooks/check_expiration.py

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,39 @@
88
if TYPE_CHECKING:
99
from dipdup.datasources.tezos_tzkt import TezosTzktDatasource
1010

11+
BATCH_SIZE = 1000
12+
1113

1214
async def check_expiration(
1315
ctx: HookContext,
1416
) -> None:
1517
ds = cast('TezosTzktDatasource', next(iter(ctx.datasources.values())))
16-
expiring_records = (
17-
await Record.filter(expired=False, domain__expires_at__lt=datetime.utcnow()).all().prefetch_related('domain')
18-
)
19-
20-
for record in expiring_records:
21-
ctx.logger.info('Record %s expired at %s', record.id, record.domain.expires_at)
22-
record.expired = True
23-
await record.save()
24-
if record.address:
25-
ctx.logger.debug('Invalidating contract metadata for %s @ %s', record.address, record.id)
26-
await ctx.update_contract_metadata(
27-
network=ds.name,
28-
address=record.address,
29-
metadata={}, # TODO: NULL
30-
)
18+
total_processed = 0
19+
20+
while True:
21+
expiring_records = (
22+
await Record.filter(expired=False, domain__expires_at__lt=datetime.utcnow())
23+
.limit(BATCH_SIZE)
24+
.prefetch_related('domain')
25+
)
26+
27+
if not expiring_records:
28+
break
29+
30+
for record in expiring_records:
31+
ctx.logger.info('Record %s expired at %s', record.id, record.domain.expires_at)
32+
record.expired = True
33+
await record.save()
34+
if record.address:
35+
ctx.logger.debug('Invalidating contract metadata for %s @ %s', record.address, record.id)
36+
await ctx.update_contract_metadata(
37+
network=ds.name,
38+
address=record.address,
39+
metadata={}, # TODO: NULL
40+
)
41+
42+
total_processed += len(expiring_records)
43+
ctx.logger.info('Processed %s expired records so far', total_processed)
44+
45+
if total_processed:
46+
ctx.logger.info('Finished processing %s expired records', total_processed)

0 commit comments

Comments
 (0)