Merge pull request #243 from aldbr/main_FEAT_pagination

feat(jobs): pagination

Author: chrisburr

diracx-cli/src/diracx/cli/jobs.py

@@ -4,7 +4,8 @@
 __all__ = ("app",)
 
 import json
-from typing import Annotated
+import re
+from typing import Annotated, cast
 
 from rich.console import Console
 from rich.table import Table
@@ -52,29 +53,66 @@ async def search(
     ],
     condition: Annotated[list[SearchSpec], Option(parser=parse_condition)] = [],
     all: bool = False,
+    page: int = 1,
+    per_page: int = 10,
 ):
     async with DiracClient() as api:
-        jobs = await api.jobs.search(
+        jobs, content_range = await api.jobs.search(
             parameters=None if all else parameter,
             search=condition if condition else None,
+            page=page,
+            per_page=per_page,
+            cls=lambda _, jobs, headers: (
+                jobs,
+                ContentRange(headers.get("Content-Range", "jobs")),
+            ),
         )
-    display(jobs, "jobs")
 
-
-def display(data, unit: str):
+    display(jobs, cast(ContentRange, content_range))
+
+
+class ContentRange:
+    unit: str | None = None
+    start: int | None = None
+    end: int | None = None
+    total: int | None = None
+
+    def __init__(self, header: str):
+        if match := re.fullmatch(r"(\w+) (\d+-\d+|\*)/(\d+|\*)", header):
+            self.unit, range, total = match.groups()
+            self.total = int(total)
+            if range != "*":
+                self.start, self.end = map(int, range.split("-"))
+        elif match := re.fullmatch(r"\w+", header):
+            self.unit = match.group()
+
+    @property
+    def caption(self):
+        if self.start is None and self.end is None:
+            range_str = "all"
+        else:
+            range_str = (
+                f"{self.start if self.start is not None else 'unknown'}-"
+                f"{self.end if self.end is not None else 'unknown'} "
+                f"of {self.total or 'unknown'}"
+            )
+        return f"Showing {range_str} {self.unit}"
+
+
+def display(data, content_range: ContentRange):
     output_format = get_diracx_preferences().output_format
     match output_format:
         case OutputFormats.JSON:
             print(json.dumps(data, indent=2))
         case OutputFormats.RICH:
-            display_rich(data, unit)
+            display_rich(data, content_range)
         case _:
             raise NotImplementedError(output_format)
 
 
-def display_rich(data, unit: str) -> None:
+def display_rich(data, content_range: ContentRange) -> None:
     if not data:
-        print(f"No {unit} found")
+        print(f"No {content_range.unit} found")
         return
 
     console = Console()
@@ -83,7 +121,7 @@ def display_rich(data, unit: str) -> None:
         table = Table(
             "Parameter",
             "Value",
-            caption=f"Showing {len(data)} of {len(data)} {unit}",
+            caption=content_range.caption,
             caption_justify="right",
         )
         for job in data:
@@ -93,7 +131,7 @@ def display_rich(data, unit: str) -> None:
     else:
         table = Table(
             *columns,
-            caption=f"Showing {len(data)} of {len(data)} {unit}",
+            caption=content_range.caption,
             caption_justify="right",
         )
         for job in data:

diracx-cli/tests/test_jobs.py

@@ -1,13 +1,122 @@
 from __future__ import annotations
 
 import json
+import os
+import tempfile
+
+import pytest
+from pytest import raises
 
 from diracx import cli
+from diracx.core.models import ScalarSearchSpec
+from diracx.core.preferences import get_diracx_preferences
 
+TEST_JDL = """
+    Arguments = "jobDescription.xml -o LogLevel=INFO";
+    Executable = "dirac-jobexec";
+    JobGroup = jobGroup;
+    JobName = jobName;
+    JobType = User;
+    LogLevel = INFO;
+    OutputSandbox =
+        {
+            Script1_CodeOutput.log,
+            std.err,
+            std.out
+        };
+    Priority = 1;
+    Site = ANY;
+    StdError = std.err;
+    StdOutput = std.out;
+"""
+
+
+@pytest.fixture
+async def jdl_file():
+    with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as temp_file:
+        temp_file.write(TEST_JDL)
+        temp_file.flush()
+        yield temp_file.name
+
+
+async def test_submit(with_cli_login, jdl_file, capfd):
+    """Test submitting a job using a JDL file."""
+
+    with open(jdl_file, "r") as temp_file:
+        await cli.jobs.submit([temp_file])
 
-async def test_search(with_cli_login, capfd):
-    await cli.jobs.search()
     cap = capfd.readouterr()
     assert cap.err == ""
+    assert "Inserted 1 jobs with ids" in cap.out
+
+
+async def test_search(with_cli_login, jdl_file, capfd):
+    """Test searching for jobs."""
+
+    # Submit 20 jobs
+    with open(jdl_file, "r") as temp_file:
+        await cli.jobs.submit([temp_file] * 20)
+
+    cap = capfd.readouterr()
+
     # By default the output should be in JSON format as capfd is not a TTY
-    json.loads(cap.out)
+    await cli.jobs.search()
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    jobs = json.loads(cap.out)
+
+    # There should be 10 jobs by default
+    assert len(jobs) == 10
+    assert "JobID" in jobs[0]
+    assert "JobGroup" in jobs[0]
+
+    # Change per-page to a very large number to get all the jobs at once: the caption should change
+    await cli.jobs.search(per_page=9999)
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    jobs = json.loads(cap.out)
+
+    # There should be 20 jobs at least now
+    assert len(jobs) >= 20
+    assert "JobID" in cap.out
+    assert "JobGroup" in cap.out
+
+    # Search for a job that doesn't exist
+    condition = ScalarSearchSpec(parameter="Status", operator="eq", value="nonexistent")
+    await cli.jobs.search(condition=[condition])
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    assert "[]" == cap.out.strip()
+
+    # Switch to RICH output
+    get_diracx_preferences.cache_clear()
+    os.environ["DIRACX_OUTPUT_FORMAT"] = "RICH"
+
+    await cli.jobs.search()
+    cap = capfd.readouterr()
+    assert cap.err == ""
+
+    with raises(json.JSONDecodeError):
+        json.loads(cap.out)
+
+    assert "JobID" in cap.out
+    assert "JobGroup" in cap.out
+    assert "Showing 0-9 of " in cap.out
+
+    # Change per-page to a very large number to get all the jobs at once: the caption should change
+    await cli.jobs.search(per_page=9999)
+    cap = capfd.readouterr()
+    assert cap.err == ""
+
+    with raises(json.JSONDecodeError):
+        json.loads(cap.out)
+
+    assert "JobID" in cap.out
+    assert "JobGroup" in cap.out
+    assert "Showing all jobs" in cap.out
+
+    # Search for a job that doesn't exist
+    await cli.jobs.search(condition=[condition])
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    assert "No jobs found" in cap.out

diracx-client/src/diracx/client/__init__.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/_client.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/_configuration.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/_vendor.py

@@ -1,5 +1,5 @@
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/__init__.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/_client.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/_configuration.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/_vendor.py

@@ -1,5 +1,5 @@
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------