diff --git a/brozzler/frontier.py b/brozzler/frontier.py index b199c1c9..e0edd6f8 100644 --- a/brozzler/frontier.py +++ b/brozzler/frontier.py @@ -51,16 +51,18 @@ def filter_claimable_site_ids( ): is_claimable = True - # or site has been disclaimed more than an hour ago + # or site has been claimed more than an hour ago if "last_claimed" in site and site["last_claimed"] <= ( now - datetime.timedelta(hours=1) ): is_claimable = True - # Count number of claimed sites per job_id (optional field) + # Count number of claimed sites per job_id (ignoring sites claimed over an hour ago) + # for enforcing the optional max_claimed_sites per job if site["claimed"] and "max_claimed_sites" in site and "job_id" in site: - job_id = site["job_id"] - job_counts[job_id] = job_counts.get(job_id, 0) + 1 + if not is_claimable: + job_id = site["job_id"] + job_counts[job_id] = job_counts.get(job_id, 0) + 1 if is_claimable: claimable_sites.append(site) diff --git a/tests/test_frontier.py b/tests/test_frontier.py index e030ffdc..b047923b 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -27,6 +27,7 @@ import doublethink import pytest +import rethinkdb as rdb import brozzler.cli @@ -35,6 +36,7 @@ args = arg_parser.parse_args([]) args.log_level = logging.INFO brozzler.cli.configure_logging(args) +r = rdb.RethinkDB() @pytest.fixture(scope="module") @@ -953,10 +955,20 @@ def test_max_claimed_sites(rethinker): sites = list(frontier.job_sites(job.id)) assert len(sites) == 5 - claimed_sites = frontier.claim_sites(1) - assert len(claimed_sites) == 1 - claimed_sites = frontier.claim_sites(3) - assert len(claimed_sites) == 2 + claimed_sites_1 = frontier.claim_sites(1) + assert len(claimed_sites_1) == 1 + claimed_sites_2 = frontier.claim_sites(3) + assert len(claimed_sites_2) == 2 + with pytest.raises(brozzler.NothingToClaim): + frontier.claim_sites(3) + + # if we "wait" an hour, we can reclaim + rr.table("sites").get_all( + r.args([item["id"] for item in (claimed_sites_1 + claimed_sites_2)]) + ).update({"claimed": True, "last_claimed": r.now().sub(61 * 60)}).run() + + claimed_sites_3 = frontier.claim_sites(3) + assert len(claimed_sites_3) == 3 with pytest.raises(brozzler.NothingToClaim): frontier.claim_sites(3)