We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 6578915 commit 2c74dfdCopy full SHA for 2c74dfd
cdx_toolkit/commoncrawl.py
@@ -60,7 +60,6 @@ def get_cc_endpoints(cc_mirror):
60
if r.status_code != 200:
61
raise RuntimeError('error {} getting list of cc indices from {}'.format(r.status_code, collinfo)) # pragma: no cover
62
set_collinfo_cache(cc_mirror, r.text)
63
- time.sleep(5) # XXX to avoid triggering rate limit
64
col = r.json()
65
66
endpoints = [x['cdx-api'] for x in col]
0 commit comments