feat(pulpcore): add JSON content negotiation to the content app

Serve a paginated JSON directory listing when Accept prefers
application/json. Plugins can override Distribution.content_handler_json.
Listing pagination is applied in the database; cache keys use only
normalized JSON limit/offset.

ref #7887

Assisted-By: Cursor

Author: TenSt

CHANGES/7887.feature

@@ -0,0 +1 @@
+Added Accept-header content negotiation to the content app so clients requesting `application/json` receive a paginated JSON directory listing.

CHANGES/plugin_api/7887.feature

@@ -0,0 +1 @@
+Added `Distribution.content_handler_json()` so plugins can serve JSON from the content app when the client prefers `application/json`.

docs/dev/reference/code-api/plugins-api/content-app.md

@@ -13,12 +13,36 @@ Making a custom Handler is a two-step process:
 2. Add the Handler to a route using aiohttp.server's [add_route()](https://aiohttp.readthedocs.io/en/stable/web_reference.html#aiohttp.web.UrlDispatcher.add_route) interface.
 
 If content needs to be served from within the `Distribution`'s base_path,
-overriding the `pulpcore.plugin.models.Distribution.content_handler` and
-`pulpcore.plugin.models.Distribution.content_handler_directory_listing`
-methods in your Distribution is an easier way to serve this content. The
-`pulpcore.plugin.models.Distribution.content_handler` method should
-return an instance of `aiohttp.web_response.Response` or a
-`pulpcore.plugin.models.ContentArtifact`.
+overriding `pulpcore.plugin.models.Distribution.content_handler`,
+`content_handler_json`, and `content_handler_list_directory` is an easier
+way to serve this content.
+
+`content_handler` should return an instance of `aiohttp.web_response.Response`
+or a `pulpcore.plugin.models.ContentArtifact`. It is used for the default
+HTML/binary representation.
+
+`content_handler_json` is invoked when the client's `Accept` header prefers
+JSON (see `pulpcore.cache.accept_prefers_json`). Return `None` (the default)
+to use pulpcore's generic paginated JSON directory listing, a JSON-serializable
+dict/list, or an `aiohttp.web.StreamResponse` for full control over
+headers/status. Concrete artifact paths stay binary unless this method returns
+JSON. Missing/`*/*`/`text/html` Accept headers keep today's HTML/binary
+responses.
+
+The generic JSON listing envelope is:
+
+```json
+{
+  "path": "/pulp/content/my-distro/",
+  "packages": [{"path": "subdir/file.iso", "size": 1024, "date": "..."}],
+  "count": 1,
+  "limit": 1000,
+  "offset": 0
+}
+```
+
+Pagination uses `?limit=` and `?offset=` (default limit 1000, max 10000).
+When more pages exist the body also includes `next_offset`.
 
 ## Creating your Handler
 

pulpcore/app/models/publication.py

@@ -740,6 +740,31 @@ def content_handler_list_directory(self, rel_path):
         """
         return set()
 
+    def content_handler_json(self, path):
+        """
+        Handler to serve a JSON representation of the content at ``path`` for this Distribution.
+
+        This is the JSON counterpart to :meth:`content_handler`. It is invoked instead of (and
+        checked before) the generic, plugin-agnostic JSON directory listing whenever the
+        client's ``Accept`` header indicates a preference for JSON over HTML. Plugins override
+        this to provide type-specific JSON (e.g. package metadata, a de-duplicated "package"
+        listing, etc.) rather than falling back to the generic file/size/date listing that
+        pulpcore builds automatically for every Distribution.
+
+        The default implementation returns ``None`` for every path, which is safe for any
+        Distribution subclass that doesn't override it: pulpcore's generic JSON directory
+        listing (or the normal HTML/binary behavior) is used instead.
+
+        Args:
+            path (str): The path being requested
+        Returns:
+            None if there is no JSON representation to serve at path. Otherwise, a
+            JSON-serializable object (dict/list) to be returned to the client, or an
+            aiohttp.web.StreamResponse (e.g. built via aiohttp.web.json_response) for full
+            control over headers/status.
+        """
+        return None
+
     def content_headers_for(self, path):
         """
         Opportunity for Distribution to specify response-headers for a specific path

pulpcore/cache/__init__.py

@@ -1,9 +1,13 @@
 # ruff: noqa: F401
 from .cache import (
+    JSON_LIST_DEFAULT_LIMIT,
+    JSON_LIST_MAX_LIMIT,
     AsyncCache,
     AsyncContentCache,
     Cache,
     CacheKeys,
     ConnectionError,
     SyncContentCache,
+    accept_prefers_json,
+    json_listing_pagination,
 )

pulpcore/cache/cache.py

@@ -29,6 +29,93 @@ class CacheKeys(enum.Enum):
     path = "path"
     host = "host"
     method = "method"
+    format = "format"
+    query = "query"
+
+
+def accept_prefers_json(accept_header):
+    """
+    Determine whether an HTTP Accept header value prefers application/json over other types.
+
+    A missing/empty header, or one whose highest-quality (per RFC 9110 q-values) entry isn't
+    "application/json" or a "+json" subtype, is treated as "does not prefer JSON". This is the
+    single source of truth for JSON content negotiation in the content app; it lives here (a
+    dependency-free leaf module) rather than in ``pulpcore.content.handler`` so that both the
+    content app's response logic and its cache key (see ``AsyncContentCache.make_key``) can use
+    the exact same decision, avoiding any risk of a JSON response being cached/served for an
+    HTML request or vice versa.
+
+    Args:
+        accept_header (str): The raw value of the request's Accept header, or None.
+
+    Returns:
+        bool: True if the client's top choice is JSON, False otherwise.
+    """
+    if not isinstance(accept_header, str) or not accept_header:
+        return False
+
+    best_type = None
+    best_q = -1.0
+    for part in accept_header.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        media_type, _, params_str = part.partition(";")
+        media_type = media_type.strip().lower()
+        q = 1.0
+        for param in params_str.split(";"):
+            param = param.strip()
+            if param.startswith("q="):
+                try:
+                    q = float(param[2:])
+                except ValueError:
+                    q = 1.0
+        if q > best_q:
+            best_q = q
+            best_type = media_type
+
+    if not best_type or best_q <= 0:
+        return False
+
+    return best_type == "application/json" or best_type.endswith("+json")
+
+
+JSON_LIST_DEFAULT_LIMIT = 1000
+JSON_LIST_MAX_LIMIT = 10000
+
+
+def json_listing_pagination(query):
+    """
+    Parse and bound ``limit``/``offset`` from a request query mapping.
+
+    Invalid or missing values fall back to defaults rather than raising. This is shared by
+    the content app's JSON listing and its cache key so paginated pages cannot collide, and
+    unrecognized query params cannot fragment the cache.
+
+    Args:
+        query: A mapping with ``.get()`` (e.g. aiohttp ``request.query``), or None.
+
+    Returns:
+        tuple: ``(limit, offset)`` integers.
+    """
+
+    def parse_int(name, default, minimum, maximum):
+        if query is None:
+            raw = default
+        else:
+            try:
+                raw = query.get(name, default)
+            except (AttributeError, TypeError):
+                raw = default
+        try:
+            value = int(raw)
+        except (TypeError, ValueError):
+            value = default
+        return max(minimum, min(value, maximum))
+
+    limit = parse_int("limit", JSON_LIST_DEFAULT_LIMIT, 1, JSON_LIST_MAX_LIMIT)
+    offset = parse_int("offset", 0, 0, 2**31 - 1)
+    return limit, offset
 
 
 def connection_error_wrapper(func):
@@ -323,7 +410,10 @@ def __init__(self, base_key=None, expires_ttl=None, keys=None, auth=None):
                       can be a callable taking the request and cache instance as arguments
             expires_ttl: length in seconds entries should live in the cache, EXPIRES_TTL is default
             keys: a list of CacheKeys to use for key creation upon entry placement,
-                (path, method) is default
+                (path, method) is default. Pass CacheKeys.format if responses for the same
+                path/method can differ based on the request's Accept header (e.g. JSON vs.
+                HTML). Pass CacheKeys.query to include normalized JSON ``limit``/``offset``
+                (other query params and HTML requests are ignored).
             auth: a callable to check authorization of the request; takes the request, cache
                   instance, and base_key as arguments.
         """
@@ -444,10 +534,23 @@ async def make_entry(self, key, base_key, handler, args, kwargs, expires=DEFAULT
     def make_key(self, request):
         """Makes the key based off the request"""
         # Might potentially have to make this async if keys require async data from request
+        wants_json = accept_prefers_json(request.headers.get("Accept"))
+        if wants_json:
+            limit, offset = json_listing_pagination(getattr(request, "query", None))
+            query_key = f"{limit}:{offset}"
+        else:
+            query_key = ""
         all_keys = {
             CacheKeys.path: request.path,
             CacheKeys.method: request.method,
             CacheKeys.host: request.url.host,
+            CacheKeys.format: "json" if wants_json else "other",
+            CacheKeys.query: query_key,
         }
-        key = ":".join(all_keys[k] for k in self.keys)
-        return key
+        parts = []
+        for key_name in self.keys:
+            value = all_keys[key_name]
+            if key_name is CacheKeys.query and value == "":
+                continue
+            parts.append(value)
+        return ":".join(parts)