Resolve upstream conflicts

Author: TheTechRobo

.github/workflows/renovate.yml

@@ -17,6 +17,10 @@ jobs:
           sudo apt-get install chromium-browser
       - name: Install uv
         uses: astral-sh/setup-uv@v5
+      - name: Install Deno
+        uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
       - name: Test new yt-dlp
         run: |
           set -euo pipefail

.github/workflows/setup/action.yml

@@ -28,6 +28,11 @@ runs:
         sudo /etc/init.d/rethinkdb restart
       shell: bash
 
+    - name: Install Deno
+      uses: denoland/setup-deno@v2
+      with:
+        deno-version: v2.x
+
     - name: Install pip dependencies
       run: |
         uv sync --python ${{ inputs.python-version }} --extra rethinkdb --extra warcprox --extra yt-dlp

.github/workflows/tests.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        version: ['3.9', '3.12', '3.14']
+        version: ['3.10', '3.12', '3.14']
     steps:
       - uses: actions/checkout@v4
 

Makefile

@@ -59,3 +59,8 @@ check-format:
 format:
 	$(VIRTUAL_ENV_DIR)/bin/ruff check $(OUTPUT_FLAGS) --select I --fix .
 	$(VIRTUAL_ENV_DIR)/bin/ruff format .
+
+.PHONY: test
+test:
+	uv sync --all-extras
+	uv run py.test tests

brozzler/browser.py

@@ -382,7 +382,10 @@ def __init__(self, **kwargs):
         self.is_browsing = False
         self._command_id = Counter()
         self._wait_interval = 0.5
+        self._max_screenshot_width = kwargs.get("max_screenshot_width", 2000)
+        self._max_screenshot_height = kwargs.get("max_screenshot_height", 20000)
         self.session_id = None
+
         # Set default configuration in case the caller doesn't use
         # configure_browser or browse_page
         self.configure_browser()
@@ -886,8 +889,12 @@ def screenshot(self, full_page=False, timeout=45):
                 lambda: self.websock_thread.received_result(msg_id), timeout=timeout
             )
             message = self.websock_thread.pop_result(msg_id)
-            width = message["result"]["contentSize"]["width"]
-            height = message["result"]["contentSize"]["height"]
+            width = min(
+                message["result"]["contentSize"]["width"], self._max_screenshot_width
+            )
+            height = min(
+                message["result"]["contentSize"]["height"], self._max_screenshot_height
+            )
             clip = dict(x=0, y=0, width=width, height=height, scale=1)
             deviceScaleFactor = 1
             screenOrientation = {"angle": 0, "type": "portraitPrimary"}

brozzler/frontier.py

@@ -65,9 +65,6 @@ def filter_claimable_site_ids(
         if is_claimable:
             claimable_sites.append(site)
 
-        if len(claimable_sites) >= max_sites_to_claim:
-            break
-
     site_ids_to_claim = []
     # gather sites that are under the max without going over
     for site in claimable_sites:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "brozzler"
-version = "1.8.0"
+version = "1.8.1"
 authors = [
   { name="Noah Levitt", email="nlevitt@archive.org" },
 ]
@@ -13,7 +13,7 @@ maintainers = [
 ]
 description = "Distributed web crawling with browsers"
 readme = "README.rst"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Programming Language :: Python :: 3",

tests/test_frontier.py

@@ -1052,6 +1052,45 @@ def test_max_claimed_sites_cross_job(rethinker):
     rr.table("sites").delete().run()
 
 
+def test_many_active_claimed_sites_cross_job(rethinker):
+    rr = rethinker
+    frontier = brozzler.RethinkDbFrontier(rr)
+
+    # clean slate
+    rr.table("jobs").delete().run()
+    rr.table("sites").delete().run()
+
+    job_conf_1 = {
+        "id": 1,
+        "seeds": [{"url": f"http://example.com/{i}"} for i in range(0, 2000)],
+        "max_claimed_sites": 3,
+    }
+    job_conf_2 = {
+        "id": 2,
+        "seeds": [
+            {"url": "http://example.com/1"},
+            {"url": "http://example.com/2"},
+            {"url": "http://example.com/3"},
+            {"url": "http://example.com/4"},
+            {"url": "http://example.com/5"},
+        ],
+        "max_claimed_sites": 5,
+    }
+
+    brozzler.new_job(frontier, job_conf_1)
+
+    # Claim all possible sites from job 1. We should only get 3 due to max_claimed_sites
+    claimed_sites_1 = frontier.claim_sites(4)
+    assert len(claimed_sites_1) == 3
+
+    # Add 5 more seeds
+    brozzler.new_job(frontier, job_conf_2)
+
+    # We shouldn't have trouble getting seeds from job 2
+    claimed_sites_1 = frontier.claim_sites(5)
+    assert len(claimed_sites_1) == 5
+
+
 # Works locally, but reliably fails in CI.
 @pytest.mark.xfail
 def test_max_claimed_sites_load_perf(rethinker):