diff --git a/sphinx/domains/__init__.py b/sphinx/domains/__init__.py index 6c2dc7bf817..414b31bd008 100644 --- a/sphinx/domains/__init__.py +++ b/sphinx/domains/__init__.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Sequence + from typing import ClassVar from docutils import nodes from docutils.parsers.rst import Directive @@ -201,6 +202,13 @@ class Domain: data: dict #: data version, bump this when the format of `self.data` changes data_version = 0 + #: Value for an empty inventory of objects for this domain. + #: It must be copy.deepcopy-able. + #: If this value is overridden, then the various intersphinx methods in the domain should + #: probably also be overridden. + #: Intersphinx is not inspecting this dictionary, so the domain has complete freedom in + #: the key and value type. + initial_intersphinx_inventory: ClassVar[dict] = {} def __init__(self, env: BuildEnvironment) -> None: self.env: BuildEnvironment = env @@ -404,3 +412,128 @@ def get_enumerable_node_type(self, node: Node) -> str | None: def get_full_qualified_name(self, node: Element) -> str | None: """Return full qualified name for given node.""" pass + + def intersphinx_add_entries_v2(self, store: Any, + data: dict[str, dict[str, Any]]) -> None: + """Store the given *data* for later intersphinx reference resolution. + + This method is called at most once with all data loaded from inventories in + v1 and v2 format. + + The *data* is a dictionary indexed by **object type**, i.e, a key from + :attr:`object_types`. The value is a dictionary indexed by **object name**, + i.e., the **name** part of each tuple returned by :meth:`get_objects`. + The value of those inner dictionaries are objects with intersphinx data, + that should not be inspected by the domain, but merely returned as a result + of reference resolution. + + For example, for Python the data could look like the following. + + .. code-block:: python + + data = { + 'class': {'pkg.mod.Foo': SomeIntersphinxData(...)}, + 'method': {'pkg.mod.Foo.bar': SomeIntersphinxData(...)}, + } + + The domain must store the given inner intersphinx data in the given *store*, + in whichever way makes sense for later reference resolution. + This *store* was initially a copy of :attr:`initial_intersphinx_inventory`. + + Later in :meth:`intersphinx_resolve_xref` during intersphinx reference resolution, + the domain is again given the *store* object to perform the resolution. + + .. versionadded:: 8.0 + """ + store = cast(dict[str, dict[str, Any]], store) + assert len(store) == 0 # the method is called at most once + store.update(data) # update so the object is changed in-place + + def _intersphinx_adjust_object_types(self, env: BuildEnvironment, + store: Any, + typ: str, target: str, + disabled_object_types: list[str], + node: pending_xref, contnode: Element, + objtypes: list[str]) -> None: + """For implementing backwards compatibility. + + This method is an internal implementation detail used in the std and python domains, + for implementing backwards compatibility. + + The given *objtypes* is the list of object types computed based on the *typ*, + which will be used for lookup. + By overriding this method this list can be manipulated, e.g., adding types + that were removed in earlier Sphinx versions. + After this method returns, the types in *disabled_object_types* are removed + from *objtypes*. This final list is given to the lookup method. + """ + pass + + def _intersphinx_resolve_xref_lookup(self, store: dict[str, dict[str, Any]], + target: str, objtypes: list[str] + ) -> Any | None: + """Default implementation for the actual lookup. + + This method is an internal implementation detail, and may be overridden + by the bundled domains, e.g., std, for customizing the lookup, while + reusing the rest of the default implementation. + """ + for objtype in objtypes: + if objtype not in store: + continue + if target in store[objtype]: + return store[objtype][target] + return None + + def intersphinx_resolve_xref(self, env: BuildEnvironment, + store: Any, + typ: str, target: str, + disabled_object_types: list[str], + node: pending_xref, contnode: Element + ) -> Any | None: + """Resolve the pending_xref *node* with the given *target* via intersphinx. + + This method should perform lookup of the pending cross-reference + in the given *store*, but otherwise behave very similarly to :meth:`resolve_xref`. + + The *typ* may be ``any`` if the cross-references comes from an any-role. + + The *store* was created through a previous call to :meth:`intersphinx_add_entries_v2`. + + The *disabled_object_types* is a list of object types that the reference may not + resolve to, per user request through :confval:`intersphinx_disabled_reftypes`. + These disabled object types are just the names of the types, without a domain prefix. + + If a candidate is found in the store, the associated object must be returned. + + If no candidates can be found, *None* can be returned; and subsequent event + handlers will be given a chance to resolve the reference. + The method can also raise :exc:`sphinx.environment.NoUri` to suppress + any subsequent resolution of this reference. + + .. versionadded:: 8.0 + """ + if typ == 'any': + objtypes = list(self.object_types) + else: + for_role = self.objtypes_for_role(typ) + if not for_role: + return None + objtypes = for_role + + self._intersphinx_adjust_object_types( + env, store, typ, target, disabled_object_types, node, contnode, objtypes) + objtypes = [o for o in objtypes if o not in disabled_object_types] + + typed_store = cast(dict[str, dict[str, Any]], store) + # we try the target either as is, or with full qualification based on the scope of node + res = self._intersphinx_resolve_xref_lookup(typed_store, target, objtypes) + if res is not None: + return res + # try with qualification of the current scope instead + full_qualified_name = self.get_full_qualified_name(node) + if full_qualified_name: + return self._intersphinx_resolve_xref_lookup( + typed_store, full_qualified_name, objtypes) + else: + return None diff --git a/sphinx/domains/c/__init__.py b/sphinx/domains/c/__init__.py index 7a68606bcc8..0344f72afbd 100644 --- a/sphinx/domains/c/__init__.py +++ b/sphinx/domains/c/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, cast from docutils import nodes from docutils.parsers.rst import directives @@ -13,6 +13,7 @@ from sphinx.domains.c._ast import ( ASTDeclaration, ASTIdentifier, + ASTIntersphinx_v2, ASTNestedName, ) from sphinx.domains.c._ids import _macroKeywords, _max_id @@ -34,6 +35,7 @@ if TYPE_CHECKING: from collections.abc import Iterator + from typing import ClassVar from docutils.nodes import Element, Node, TextElement, system_message @@ -666,6 +668,10 @@ class CDomain(Domain): 'objects': {}, # fullname -> docname, node_id, objtype } + initial_intersphinx_inventory: ClassVar[dict[str, Symbol]] = { + 'root_symbol': Symbol(None, None, None, None, None), + } + def clear_doc(self, docname: str) -> None: if Symbol.debug_show_tree: logger.debug("clear_doc: %s", docname) @@ -712,9 +718,10 @@ def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> No ourObjects[fullname] = (fn, id_, objtype) # no need to warn on duplicates, the symbol merge already does that - def _resolve_xref_inner(self, env: BuildEnvironment, fromdocname: str, builder: Builder, - typ: str, target: str, node: pending_xref, - contnode: Element) -> tuple[Element | None, str | None]: + def _resolve_xref_in_tree(self, env: BuildEnvironment, root: Symbol, + softParent: bool, + typ: str, target: str, + node: pending_xref) -> tuple[Symbol, ASTNestedName]: parser = DefinitionParser(target, location=node, config=env.config) try: name = parser.parse_xref_object() @@ -723,20 +730,32 @@ def _resolve_xref_inner(self, env: BuildEnvironment, fromdocname: str, builder: location=node) return None, None parentKey: LookupKey = node.get("c:parent_key", None) - rootSymbol = self.data['root_symbol'] if parentKey: - parentSymbol: Symbol = rootSymbol.direct_lookup(parentKey) + parentSymbol: Symbol = root.direct_lookup(parentKey) if not parentSymbol: - logger.debug("Target: %s", target) - logger.debug("ParentKey: %s", parentKey) - logger.debug(rootSymbol.dump(1)) - assert parentSymbol # should be there + if softParent: + parentSymbol = root + else: + msg = f"Target: {target}\nParentKey: {parentKey}\n{root.dump(1)}\n" + raise AssertionError(msg) else: - parentSymbol = rootSymbol + parentSymbol = root s = parentSymbol.find_declaration(name, typ, matchSelf=True, recurseInAnon=True) if s is None or s.declaration is None: return None, None + # TODO: conditionally warn about xrefs with incorrect tagging? + return s, name + + def _resolve_xref_inner(self, env: BuildEnvironment, fromdocname: str, builder: Builder, + typ: str, target: str, node: pending_xref, + contnode: Element) -> tuple[Element, str]: + if Symbol.debug_lookup: + Symbol.debug_print("C._resolve_xref_inner(type={}, target={})".format(typ, target)) + s, name = self._resolve_xref_in_tree(env, self.data['root_symbol'], + False, typ, target, node) + if s is None: + return None, None # TODO: check role type vs. object type @@ -779,6 +798,52 @@ def get_objects(self) -> Iterator[tuple[str, str, str, str, str, int]]: newestId = symbol.declaration.get_newest_id() yield (name, dispname, objectType, docname, newestId, 1) + def intersphinx_add_entries_v2(self, store: dict[str, Symbol], + data: dict[str, dict[str, Any]]) -> None: + root = store['root_symbol'] + for object_type, per_type_data in data.items(): + for object_name, item_set in per_type_data.items(): + parser = DefinitionParser( + object_name, location=('intersphinx', 0), config=self.env.config) + try: + ast = parser._parse_nested_name() + except DefinitionError as e: + logger.warning("Error in C entry in intersphinx inventory:\n%s", e) + continue + decl = ASTDeclaration(object_type, 'intersphinx', + ASTIntersphinx_v2(ast, item_set)) + root.add_declaration(decl, docname="_$FakeIntersphinxDoc", line=0) + + def _intersphinx_resolve_xref_inner(self, env: BuildEnvironment, + store: dict[str, Symbol], + target: str, + node: pending_xref, + typ: str) -> Any | None: + if Symbol.debug_lookup: + Symbol.debug_print( + f"C._intersphinx_resolve_xref_inner(type={typ}, target={target})") + s, name = self._resolve_xref_in_tree(env, store['root_symbol'], + True, typ, target, node) + if s is None: + return None + assert s.declaration is not None + decl = cast(ASTIntersphinx_v2, s.declaration.declaration) + return decl.data + + def intersphinx_resolve_xref(self, env: BuildEnvironment, + store: dict[str, Symbol], + typ: str, target: str, + disabled_object_types: list[str], + node: pending_xref, contnode: Element + ) -> Any | None: + if typ == 'any': + with logging.suppress_logging(): + return self._intersphinx_resolve_xref_inner( + env, store, target, node, typ) + else: + return self._intersphinx_resolve_xref_inner( + env, store, target, node, typ) + def setup(app: Sphinx) -> ExtensionMetadata: app.add_domain(CDomain) diff --git a/sphinx/domains/c/_ast.py b/sphinx/domains/c/_ast.py index 3a8e2a2a4cb..deee33654bc 100644 --- a/sphinx/domains/c/_ast.py +++ b/sphinx/domains/c/_ast.py @@ -25,6 +25,7 @@ DeclarationType = Union[ "ASTStruct", "ASTUnion", "ASTEnum", "ASTEnumerator", "ASTType", "ASTTypeWithInit", "ASTMacro", + "ASTIntersphinx_v2", ] @@ -1329,6 +1330,28 @@ def describe_signature(self, signode: TextElement, mode: str, self.init.describe_signature(signode, 'markType', env, symbol) +class ASTIntersphinx_v2(ASTBaseBase): + def __init__(self, name: ASTNestedName, data: Any) -> None: + self.name = name + self.data = data + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.name) + " (has data)" + + def get_id(self, version: int, objectType: str, symbol: "Symbol") -> str: + return symbol.get_full_nested_name().get_id(version) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + raise AssertionError # should not happen + + @property + def function_params(self) -> list[ASTFunctionParameter] | None: + # the v2 data does not contain actual declarations, but just names + # so return nothing here + return None + + class ASTDeclaration(ASTBaseBase): def __init__(self, objectType: str, directiveType: str | None, declaration: DeclarationType | ASTFunctionParameter, diff --git a/sphinx/domains/python/__init__.py b/sphinx/domains/python/__init__.py index 542911f8706..179dd2d5cce 100644 --- a/sphinx/domains/python/__init__.py +++ b/sphinx/domains/python/__init__.py @@ -824,6 +824,17 @@ def _make_module_refnode(self, builder: Builder, fromdocname: str, name: str, return make_refnode(builder, fromdocname, module.docname, module.node_id, contnode, title) + def _intersphinx_adjust_object_types(self, env: BuildEnvironment, + store: Any, + typ: str, target: str, + disabled_object_types: list[str], + node: pending_xref, contnode: Element, + objtypes: list[str]) -> None: + # we adjust the object types for backwards compatibility + if 'attribute' in objtypes and 'method' not in objtypes: + # Since Sphinx-2.1, properties are stored as py:method + objtypes.append('method') + def get_objects(self) -> Iterator[tuple[str, str, str, str, str, int]]: for modname, mod in self.modules.items(): yield (modname, modname, 'module', mod.docname, mod.node_id, 0) diff --git a/sphinx/domains/std/__init__.py b/sphinx/domains/std/__init__.py index 30d0977a2f4..c1db574b340 100644 --- a/sphinx/domains/std/__init__.py +++ b/sphinx/domains/std/__init__.py @@ -996,6 +996,38 @@ def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, labelid, contnode))) return results + def _intersphinx_adjust_object_types(self, env: BuildEnvironment, + store: Any, + typ: str, target: str, + disabled_object_types: list[str], + node: pending_xref, contnode: Element, + objtypes: list[str]) -> None: + # we adjust the object types for backwards compatibility + if 'cmdoption' in objtypes: + # until Sphinx-1.6, cmdoptions are stored as std:option + objtypes.append('option') + + def _intersphinx_resolve_xref_lookup(self, store: dict[str, dict[str, Any]], + target: str, objtypes: list[str] + ) -> Any | None: + # Overridden as we also need to do case-insensitive lookup for label and term. + for objtype in objtypes: + if objtype not in store: + continue + + if target in store[objtype]: + # Case-sensitive match, use it + return store[objtype][target] + elif objtype in ('label', 'term'): + # Some types require case-insensitive matches: + # * 'term': https://github.com/sphinx-doc/sphinx/issues/9291 + # * 'label': https://github.com/sphinx-doc/sphinx/issues/12008 + target_lower = target.lower() + for object_name, object_data in store[objtype].items(): + if object_name.lower() == target_lower: + return object_data + return None + def get_objects(self) -> Iterator[tuple[str, str, str, str, str, int]]: # handle the special 'doc' reference here for doc in self.env.all_docs: diff --git a/sphinx/ext/intersphinx.py b/sphinx/ext/intersphinx.py index a8a2cf13161..275a191db8b 100644 --- a/sphinx/ext/intersphinx.py +++ b/sphinx/ext/intersphinx.py @@ -19,6 +19,7 @@ from __future__ import annotations import concurrent.futures +import copy import functools import posixpath import re @@ -29,42 +30,63 @@ from urllib.parse import urlsplit, urlunsplit from docutils import nodes -from docutils.utils import relative_path import sphinx from sphinx.addnodes import pending_xref from sphinx.builders.html import INVENTORY_FILENAME from sphinx.deprecation import _deprecation_warning from sphinx.errors import ExtensionError -from sphinx.locale import _, __ +from sphinx.locale import __ from sphinx.transforms.post_transforms import ReferencesResolver from sphinx.util import logging, requests from sphinx.util.docutils import CustomReSTDispatcher, SphinxRole -from sphinx.util.inventory import InventoryFile +from sphinx.util.inventory import InventoryFile, InventoryItemSet if TYPE_CHECKING: from collections.abc import Iterable from types import ModuleType from typing import IO, Any, Union - from docutils.nodes import Node, TextElement, system_message + from docutils.nodes import Element, Node, TextElement, system_message from docutils.utils import Reporter from sphinx.application import Sphinx from sphinx.config import Config from sphinx.domains import Domain from sphinx.environment import BuildEnvironment - from sphinx.util.typing import ExtensionMetadata, Inventory, InventoryItem, RoleFunction + from sphinx.util.typing import ExtensionMetadata, Inventory, RoleFunction InventoryCacheEntry = tuple[Union[str, None], int, Inventory] logger = logging.getLogger(__name__) +def _process_disabled_reftypes(env: BuildEnvironment) -> None: + # Compile intersphinx_disabled_reftypes into datastructures + # easier to check during reference resolution. + # It is a separate function so the tests can use it. + env.intersphinx_all_disabled = False # type: ignore[attr-defined] + env.intersphinx_all_domain_disabled = set() # type: ignore[attr-defined] + env.intersphinx_disabled_per_domain = {} # type: ignore[attr-defined] + for d in env.config.intersphinx_disabled_reftypes: + if d == '*': + env.intersphinx_all_disabled = True # type: ignore[attr-defined] + elif ':' in d: + domain, typ = d.split(':', 1) + if typ == '*': + env.intersphinx_all_domain_disabled.add(domain) # type: ignore[attr-defined] + else: + env.intersphinx_disabled_per_domain.setdefault( # type: ignore[attr-defined] + domain, []).append(typ) + + class InventoryAdapter: """Inventory adapter for environment""" def __init__(self, env: BuildEnvironment) -> None: + _deprecation_warning( + __name__, f'{self.__class__.__name__}', '', remove=(9, 0) + ) self.env = env if not hasattr(env, 'intersphinx_cache'): @@ -96,7 +118,78 @@ def named_inventory(self) -> dict[str, Inventory]: def clear(self) -> None: self.env.intersphinx_inventory.clear() # type: ignore[attr-defined] - self.env.intersphinx_named_inventory.clear() # type: ignore[attr-defined] + + +class _EnvAdapter: + """Adapter for environment to set inventory data and configuration settings.""" + + def __init__(self, env: BuildEnvironment) -> None: + self.env = env + + if not hasattr(env, 'intersphinx_cache'): + _process_disabled_reftypes(env) + + # old stuff, RemovedInSphinx90, still used in tests + self.env.intersphinx_inventory = {} # type: ignore[attr-defined] + # old stuff end + + # initial storage when fetching inventories before processing + self.env.intersphinx_cache = {} # type: ignore[attr-defined] + + # list of inventory names for validation + self.env.intersphinx_inventory_names = set() # type: ignore[attr-defined] + # store inventory data in domain-specific data structures + self.env.intersphinx_by_domain_inventory = {} # type: ignore[attr-defined] + self._clear_by_domain_inventory() + + @property + def all_objtypes_disabled(self) -> bool: + return self.env.intersphinx_all_disabled # type: ignore[attr-defined] + + def all_domain_objtypes_disabled(self, domain: str) -> bool: + return domain in self.env.intersphinx_all_domain_disabled # type: ignore[attr-defined] + + def disabled_objtypes_in_domain(self, domain: str) -> list[str]: + return self.env.intersphinx_disabled_per_domain.get(domain, []) # type: ignore[attr-defined] + + def _clear_by_domain_inventory(self) -> None: + # reinitialize the domain-specific inventory stores + for domain in self.env.domains.values(): + inv = copy.deepcopy(domain.initial_intersphinx_inventory) + self.env.intersphinx_by_domain_inventory[domain.name] = inv # type: ignore[attr-defined] + + @property + def cache(self) -> dict[str, InventoryCacheEntry]: + """Intersphinx cache. + + - Key is the URI of the remote inventory + - Element one is the key given in the Sphinx intersphinx_mapping + configuration value + - Element two is a time value for cache invalidation, a float + - Element three is the loaded remote inventory, type Inventory + """ + return self.env.intersphinx_cache # type: ignore[attr-defined] + + @property + def names(self) -> set[str | None]: + return self.env.intersphinx_inventory_names # type: ignore[attr-defined] + + @property + def by_domain_inventory(self) -> dict[str, dict[str, Any]]: + return self.env.intersphinx_by_domain_inventory # type: ignore[attr-defined] + + @property + def main_inventory(self) -> Inventory: + # deprecated, still used for setting up old stuff for tests + return self.env.intersphinx_inventory # type: ignore[attr-defined] + + def clear(self) -> None: + # old stuff + self.env.intersphinx_inventory.clear() # type: ignore[attr-defined] + # old stuff end + self.env.intersphinx_inventory_names.clear() # type: ignore[attr-defined] + self.env.intersphinx_by_domain_inventory.clear() # type: ignore[attr-defined] + self._clear_by_domain_inventory() def _strip_basic_auth(url: str) -> str: @@ -251,10 +344,21 @@ def fetch_inventory_group( "with the following issues:") + "\n" + issues) +_debug = False +_debug_indent = 0 +_debug_indent_string = " " + + +def _debug_print(*args: Any) -> None: + msg = _debug_indent_string * _debug_indent + msg += "".join(str(e) for e in args) + print(msg) + + def load_mappings(app: Sphinx) -> None: """Load all intersphinx mappings into the environment.""" now = int(time.time()) - inventories = InventoryAdapter(app.builder.env) + inventories = _EnvAdapter(app.builder.env) intersphinx_cache: dict[str, InventoryCacheEntry] = inventories.cache with concurrent.futures.ThreadPoolExecutor() as pool: @@ -271,183 +375,167 @@ def load_mappings(app: Sphinx) -> None: if any(updated): inventories.clear() - # Duplicate values in different inventories will shadow each - # other; which one will override which can vary between builds - # since they are specified using an unordered dict. To make - # it more consistent, we sort the named inventories and then - # add the unnamed inventories last. This means that the - # unnamed inventories will shadow the named ones but the named - # ones can still be accessed when the name is specified. - named_vals = [] - unnamed_vals = [] - for name, _expiry, invdata in intersphinx_cache.values(): - if name: - named_vals.append((name, invdata)) - else: - unnamed_vals.append((name, invdata)) - for name, invdata in sorted(named_vals) + unnamed_vals: - if name: - inventories.named_inventory[name] = invdata + # old stuff, still used in the tests + cached_vals = list(inventories.cache.values()) + named_vals = sorted(v for v in cached_vals if v[0]) + unnamed_vals = [v for v in cached_vals if not v[0]] + for _name, _, invdata in named_vals + unnamed_vals: for type, objects in invdata.items(): inventories.main_inventory.setdefault(type, {}).update(objects) + # end of old stuff + + # first collect all entries indexed by domain, object name, and object type + # domain -> object_type -> object_name -> InventoryItemSet([(inv_name, inner_data)]) + entries: dict[str, dict[str, dict[str, InventoryItemSet]]] = {} + for inv_name, _, inv_data in inventories.cache.values(): + assert inv_name not in inventories.names + inventories.names.add(inv_name) + + for inv_object_type, inv_objects in inv_data.items(): + domain_name, object_type = inv_object_type.split(':', 1) + # skip objects in domains we don't use + if domain_name not in app.env.domains: + continue - -def _create_element_from_result(domain: Domain, inv_name: str | None, - data: InventoryItem, - node: pending_xref, contnode: TextElement) -> nodes.reference: - proj, version, uri, dispname = data - if '://' not in uri and node.get('refdoc'): - # get correct path in case of subdirectories - uri = posixpath.join(relative_path(node['refdoc'], '.'), uri) - if version: - reftitle = _('(in %s v%s)') % (proj, version) - else: - reftitle = _('(in %s)') % (proj,) - newnode = nodes.reference('', '', internal=False, refuri=uri, reftitle=reftitle) - if node.get('refexplicit'): - # use whatever title was given - newnode.append(contnode) - elif dispname == '-' or \ - (domain.name == 'std' and node['reftype'] == 'keyword'): - # use whatever title was given, but strip prefix - title = contnode.astext() - if inv_name is not None and title.startswith(inv_name + ':'): - newnode.append(contnode.__class__(title[len(inv_name) + 1:], - title[len(inv_name) + 1:])) - else: - newnode.append(contnode) - else: - # else use the given display name (used for :ref:) - newnode.append(contnode.__class__(dispname, dispname)) - return newnode - - -def _resolve_reference_in_domain_by_target( - inv_name: str | None, inventory: Inventory, - domain: Domain, objtypes: Iterable[str], - target: str, - node: pending_xref, contnode: TextElement) -> nodes.reference | None: - for objtype in objtypes: - if objtype not in inventory: - # Continue if there's nothing of this kind in the inventory - continue - - if target in inventory[objtype]: - # Case sensitive match, use it - data = inventory[objtype][target] - elif objtype in {'std:label', 'std:term'}: - # Some types require case insensitive matches: - # * 'term': https://github.com/sphinx-doc/sphinx/issues/9291 - # * 'label': https://github.com/sphinx-doc/sphinx/issues/12008 - target_lower = target.lower() - insensitive_matches = list(filter(lambda k: k.lower() == target_lower, - inventory[objtype].keys())) - if insensitive_matches: - data = inventory[objtype][insensitive_matches[0]] - else: - # No case insensitive match either, continue to the next candidate - continue - else: - # Could reach here if we're not a term but have a case insensitive match. - # This is a fix for terms specifically, but potentially should apply to - # other types. - continue - return _create_element_from_result(domain, inv_name, data, node, contnode) - return None + domain_entries = entries.setdefault(domain_name, {}) + per_type = domain_entries.setdefault(object_type, {}) + for object_name, object_data in inv_objects.items(): + item_set = per_type.setdefault(object_name, InventoryItemSet()) + item_set.append(inv_name, object_data) + + # and then give the data to each domain + for domain_name, domain_entries in entries.items(): + if _debug: + global _debug_indent + _debug_print(f"intersphinx debug(load_mappings): domain={domain_name}") + _debug_indent += 1 + for objtyp, names in domain_entries.items(): + _debug_print(f"objtyp={objtyp}:") + _debug_indent += 1 + for name, data in names.items(): + _debug_print(f"{name}: {data}") + _debug_indent -= 1 + _debug_indent -= 1 + domain = app.env.domains[domain_name] + domain_store = inventories.by_domain_inventory[domain_name] + domain.intersphinx_add_entries_v2(domain_store, domain_entries) def _resolve_reference_in_domain(env: BuildEnvironment, - inv_name: str | None, inventory: Inventory, + inv_name: str | None, honor_disabled_refs: bool, - domain: Domain, objtypes: Iterable[str], - node: pending_xref, contnode: TextElement, - ) -> nodes.reference | None: - obj_types: dict[str, None] = {}.fromkeys(objtypes) - - # we adjust the object types for backwards compatibility - if domain.name == 'std' and 'cmdoption' in obj_types: - # cmdoptions were stored as std:option until Sphinx 1.6 - obj_types['option'] = None - if domain.name == 'py' and 'attribute' in obj_types: - # properties are stored as py:method since Sphinx 2.1 - obj_types['method'] = None - - # the inventory contains domain:type as objtype - domain_name = domain.name - obj_types = {f"{domain_name}:{obj_type}": None for obj_type in obj_types} - - # now that the objtypes list is complete we can remove the disabled ones + domain: Domain, + node: pending_xref, contnode: TextElement + ) -> Element | None: + if _debug: + global _debug_indent + _debug_print("intersphinx debug(_resolve_reference_in_domain):") + _debug_indent += 1 + _debug_print(f"inv_name={inv_name}") + _debug_print(f"honor_disabled_refs={honor_disabled_refs}") + _debug_print(f"domain={domain.name}") + _debug_print(f"node={node}") if honor_disabled_refs: - disabled = set(env.config.intersphinx_disabled_reftypes) - obj_types = {obj_type: None - for obj_type in obj_types - if obj_type not in disabled} - - objtypes = [*obj_types.keys()] - - # without qualification - res = _resolve_reference_in_domain_by_target(inv_name, inventory, domain, objtypes, - node['reftarget'], node, contnode) - if res is not None: - return res - - # try with qualification of the current scope instead - full_qualified_name = domain.get_full_qualified_name(node) - if full_qualified_name is None: + conf = _EnvAdapter(env) # make sure the disabled has been processed + assert not conf.all_objtypes_disabled + assert not conf.all_domain_objtypes_disabled(domain.name) + disabled_refs = conf.disabled_objtypes_in_domain(domain.name) + else: + disabled_refs = [] + + if _debug: + _debug_print(f"disabled_refs={disabled_refs}") + + domain_store = _EnvAdapter(env).by_domain_inventory[domain.name] + inv_set = domain.intersphinx_resolve_xref( + env, domain_store, node['reftype'], node['reftarget'], disabled_refs, node, contnode) + if _debug: + _debug_print(f"inv_set={inv_set}") + if inv_set is None: + if _debug: + _debug_indent -= 1 + return None + inv_set_restricted = inv_set.select_inventory(inv_name) + if _debug: + _debug_print(f"res restricted to inv_name={inv_name}: {inv_set_restricted}") + _debug_indent -= 1 + try: + return inv_set_restricted.make_reference_node(domain.name, node, contnode) + except ValueError: return None - return _resolve_reference_in_domain_by_target(inv_name, inventory, domain, objtypes, - full_qualified_name, node, contnode) -def _resolve_reference(env: BuildEnvironment, inv_name: str | None, inventory: Inventory, +def _resolve_reference(env: BuildEnvironment, inv_name: str | None, honor_disabled_refs: bool, - node: pending_xref, contnode: TextElement) -> nodes.reference | None: + node: pending_xref, contnode: TextElement) -> Element | None: + if _debug: + global _debug_indent + _debug_print("intersphinx debug(_resolve_reference):") + _debug_indent += 1 + _debug_print(f"inv_name={inv_name}") + _debug_print(f"honor_disabled_refs={honor_disabled_refs}") + _debug_print(f"node={node}") # disabling should only be done if no inventory is given honor_disabled_refs = honor_disabled_refs and inv_name is None - if honor_disabled_refs and '*' in env.config.intersphinx_disabled_reftypes: + if honor_disabled_refs and _EnvAdapter(env).all_objtypes_disabled: + if _debug: + _debug_print("res=None, honor_disabled_refs and all_objtypes_disabled") + _debug_indent -= 1 return None - typ = node['reftype'] - if typ == 'any': + if node['reftype'] == 'any': for domain_name, domain in env.domains.items(): + if _debug: + _debug_print(f"typ=any, trying domain={domain_name}") if (honor_disabled_refs - and (domain_name + ":*") in env.config.intersphinx_disabled_reftypes): + and _EnvAdapter(env).all_domain_objtypes_disabled(domain_name)): + if _debug: + msg = _debug_indent_string + msg += "skipping, honor_disabled_refs and all_domain_objtypes_disabled" + _debug_print(msg) continue - objtypes: Iterable[str] = domain.object_types.keys() - res = _resolve_reference_in_domain(env, inv_name, inventory, - honor_disabled_refs, - domain, objtypes, - node, contnode) + res = _resolve_reference_in_domain(env, inv_name, honor_disabled_refs, + domain, node, contnode) if res is not None: + if _debug: + _debug_print(f"res={res}") + _debug_indent -= 1 return res + if _debug: + _debug_print("res=None, no matches in any domain") + _debug_indent -= 1 return None else: domain_name = node.get('refdomain') if not domain_name: + if _debug: + _debug_print("res=None, no domain in reference") + _debug_indent -= 1 # only objects in domains are in the inventory return None - if honor_disabled_refs \ - and (domain_name + ":*") in env.config.intersphinx_disabled_reftypes: + if honor_disabled_refs and _EnvAdapter(env).all_domain_objtypes_disabled(domain_name): + if _debug: + _debug_print("res=None, honor_disabled_refs and all_domain_objtypes_disabled") + _debug_indent -= 1 return None domain = env.get_domain(domain_name) - objtypes = domain.objtypes_for_role(typ) or () - if not objtypes: - return None - return _resolve_reference_in_domain(env, inv_name, inventory, - honor_disabled_refs, - domain, objtypes, - node, contnode) + res = _resolve_reference_in_domain(env, inv_name, honor_disabled_refs, + domain, node, contnode) + if _debug: + _debug_print(f"res={res}") + _debug_indent -= 1 + return res def inventory_exists(env: BuildEnvironment, inv_name: str) -> bool: - return inv_name in InventoryAdapter(env).named_inventory + return inv_name in _EnvAdapter(env).names def resolve_reference_in_inventory(env: BuildEnvironment, inv_name: str, node: pending_xref, contnode: TextElement, - ) -> nodes.reference | None: + ) -> Element | None: """Attempt to resolve a missing reference via intersphinx references. Resolution is tried in the given inventory with the target as is. @@ -455,26 +543,39 @@ def resolve_reference_in_inventory(env: BuildEnvironment, Requires ``inventory_exists(env, inv_name)``. """ assert inventory_exists(env, inv_name) - return _resolve_reference(env, inv_name, InventoryAdapter(env).named_inventory[inv_name], - False, node, contnode) + if _debug: + global _debug_indent + _debug_print("intersphinx debug(resolve_reference_in_inventory):") + _debug_indent += 1 + res = _resolve_reference(env, inv_name, False, node, contnode) + if _debug: + _debug_print(f"res={res}") + _debug_indent -= 1 + return res def resolve_reference_any_inventory(env: BuildEnvironment, honor_disabled_refs: bool, node: pending_xref, contnode: TextElement, - ) -> nodes.reference | None: + ) -> Element | None: """Attempt to resolve a missing reference via intersphinx references. Resolution is tried with the target as is in any inventory. """ - return _resolve_reference(env, None, InventoryAdapter(env).main_inventory, - honor_disabled_refs, - node, contnode) + if _debug: + global _debug_indent + _debug_print("intersphinx debug(resolve_reference_any_inventory):") + _debug_indent += 1 + res = _resolve_reference(env, None, honor_disabled_refs, node, contnode) + if _debug: + _debug_print(f"res={res}") + _debug_indent -= 1 + return res def resolve_reference_detect_inventory(env: BuildEnvironment, node: pending_xref, contnode: TextElement, - ) -> nodes.reference | None: + ) -> Element | None: """Attempt to resolve a missing reference via intersphinx references. Resolution is tried first with the target as is in any inventory. @@ -482,26 +583,45 @@ def resolve_reference_detect_inventory(env: BuildEnvironment, to form ``inv_name:newtarget``. If ``inv_name`` is a named inventory, then resolution is tried in that inventory with the new target. """ + if _debug: + global _debug_indent + _debug_print("intersphinx debug(resolve_reference_detect_inventory):") + _debug_indent += 1 + _debug_print(f"node={node}") # ordinary direct lookup, use data as is res = resolve_reference_any_inventory(env, True, node, contnode) if res is not None: + if _debug: + _debug_print(f"res={res}") + _debug_indent -= 1 return res # try splitting the target into 'inv_name:target' target = node['reftarget'] if ':' not in target: + if _debug: + _debug_print("res=None, can't split into inv_name:target") + _debug_indent -= 1 return None inv_name, newtarget = target.split(':', 1) if not inventory_exists(env, inv_name): + if _debug: + _debug_print(f"res=None, inventory ({inv_name}) doesn't exist") + _debug_indent -= 1 return None node['reftarget'] = newtarget + node['origtarget'] = target res_inv = resolve_reference_in_inventory(env, inv_name, node, contnode) node['reftarget'] = target + del node['origtarget'] + if _debug: + _debug_print(f"res={res_inv}") + _debug_indent -= 1 return res_inv def missing_reference(app: Sphinx, env: BuildEnvironment, node: pending_xref, - contnode: TextElement) -> nodes.reference | None: + contnode: TextElement) -> Element | None: """Attempt to resolve a missing reference via intersphinx references.""" return resolve_reference_detect_inventory(env, node, contnode) @@ -810,7 +930,7 @@ def setup(app: Sphinx) -> ExtensionMetadata: app.add_post_transform(IntersphinxRoleResolver) return { 'version': sphinx.__display_version__, - 'env_version': 1, + 'env_version': 2, 'parallel_read_safe': True, } diff --git a/sphinx/util/inventory.py b/sphinx/util/inventory.py index a43fd0379ea..86dd84f41df 100644 --- a/sphinx/util/inventory.py +++ b/sphinx/util/inventory.py @@ -2,10 +2,15 @@ from __future__ import annotations import os +import posixpath import re import zlib from typing import IO, TYPE_CHECKING, Callable +from docutils import nodes +from docutils.utils import relative_path + +from sphinx.locale import _ from sphinx.util import logging BUFSIZE = 16 * 1024 @@ -14,6 +19,9 @@ if TYPE_CHECKING: from collections.abc import Iterator + from docutils.nodes import TextElement + + from sphinx.addnodes import pending_xref from sphinx.builders import Builder from sphinx.environment import BuildEnvironment from sphinx.util.typing import Inventory, InventoryItem @@ -150,7 +158,7 @@ def load_v2( if location.endswith('$'): location = location[:-1] + name location = join(uri, location) - inv_item: InventoryItem = projname, version, location, dispname + inv_item: InventoryItem = (projname, version, location, dispname) invdata.setdefault(type, {})[name] = inv_item return invdata @@ -187,3 +195,94 @@ def escape(string: str) -> str: (name, domainname, typ, prio, uri, dispname)) f.write(compressor.compress(entry.encode())) f.write(compressor.flush()) + + +class InventoryItemSet: + """Container with intersphinx resolution data. + + Instances of this class is stored by domains and later returned + during intersphinx reference resolution. + The implementation details of this class is thus private for intersphinx. + + The data stored is a list of tuples: + + - Element one is a unique reference given in the intersphinx_mapping + configuration variable. + - Element two is data about an inventory item in the form of an + InventoryItem tuple. + """ + + def __init__(self, __items: dict[str | None, list[InventoryItem]] | None = None) -> None: + if __items is None: + self._items: dict[str | None, list[InventoryItem]] = {} + else: + self._items = __items + + def __repr__(self) -> str: + return "InventoryItemSet({})".format(self._items) + + def append(self, inventory_name: str | None, item: InventoryItem) -> None: + self._items.setdefault(inventory_name, []).append(item) + + def select_inventory(self, inv_name: str | None) -> InventoryItemSet: + """Return inventory items from ``inv_name``. + + If ``inv_name`` is ``None``, return all inventories. + """ + if inv_name is None: + return InventoryItemSet(self._items.copy()) + try: + return InventoryItemSet({inv_name: self._items[inv_name].copy()}) + except KeyError: + # If inv_name doesn't exist within self._items + return InventoryItemSet() + + def make_reference_node( + self, + domain_name: str, + node: pending_xref, + contnode: TextElement, + ) -> nodes.reference: + # TODO: document and test + if len(self._items) == 0: + msg = "No inventory items!" + raise ValueError(msg) + + legacy_mapping_items = self._items.get(None, []) + if len(legacy_mapping_items) == 0: + inv_name = min(filter(None, self._items)) + proj, version, uri, dispname = self._items[inv_name][0] + elif len(legacy_mapping_items) == 1: + # Deprecated path for handling pre-Sphinx 1.0 intersphinx_mapping + # xref RemovedInSphinx70Warning + inv_name = "" + proj, version, uri, dispname = legacy_mapping_items[0] + else: + raise AssertionError + + if '://' not in uri and 'refdoc' in node: + # get correct path in case of subdirectories + uri = posixpath.join(relative_path(node['refdoc'], '.'), uri) + if version: + reftitle = _('(in %s v%s)') % (proj, version) + else: + reftitle = _('(in %s)') % proj + + newnode = nodes.reference('', '', internal=False, refuri=uri, reftitle=reftitle) + if node.get('refexplicit'): + # use whatever title was given + newnode.append(contnode) + elif (dispname == '-' + or (domain_name == 'std' and node['reftype'] == 'keyword')): + # use whatever title was given, but strip prefix + title = contnode.astext() + if (node.get('origtarget') and node['origtarget'] != node['reftarget'] + and title.startswith(inv_name + ':')): + new_title = title[len(inv_name + ':'):] + newnode.append(contnode.__class__(new_title, new_title)) + else: + newnode.append(contnode) + else: + # else use the given display name (used for :ref:) + newnode.append(contnode.__class__(dispname, dispname)) + return newnode diff --git a/tests/roots/test-domain-c-intersphinx/index.rst b/tests/roots/test-domain-c-intersphinx/index.rst index 5d6d3e09891..c97cc695c0e 100644 --- a/tests/roots/test-domain-c-intersphinx/index.rst +++ b/tests/roots/test-domain-c-intersphinx/index.rst @@ -60,3 +60,12 @@ - :c:member:`_functionParam.param` - :c:var:`_functionParam.param` - :c:data:`_functionParam.param` + +- :any:`_struct.@anon.i` +- :c:member:`_struct.@anon.i` +- :c:var:`_struct.@anon.i` +- :c:data:`_struct.@anon.i` +- :any:`_struct.i` +- :c:member:`_struct.i` +- :c:var:`_struct.i` +- :c:data:`_struct.i` diff --git a/tests/test_domains/test_domain_c.py b/tests/test_domains/test_domain_c.py index a8a92cb82ae..b627623c832 100644 --- a/tests/test_domains/test_domain_c.py +++ b/tests/test_domains/test_domain_c.py @@ -742,10 +742,15 @@ def test_domain_c_build_intersphinx(tmp_path, app, status, warning): .. c:function:: void _function() .. c:macro:: _macro .. c:struct:: _struct + + .. c:union:: @anon + + .. c:var:: int i + .. c:union:: _union .. c:enum:: _enum - .. c:enumerator:: _enumerator + .. c:enumerator:: _enumerator .. c:type:: _type .. c:function:: void _functionParam(int param) @@ -766,6 +771,8 @@ def test_domain_c_build_intersphinx(tmp_path, app, status, warning): _macro c:macro 1 index.html#c.$ - _member c:member 1 index.html#c.$ - _struct c:struct 1 index.html#c.$ - +_struct.@anon c:union 1 index.html#c.$ _struct.[anonymous] +_struct.@anon.i c:member 1 index.html#c.$ _struct.[anonymous].i _type c:type 1 index.html#c.$ - _union c:union 1 index.html#c.$ - _var c:member 1 index.html#c.$ - diff --git a/tests/test_extensions/test_ext_intersphinx.py b/tests/test_extensions/test_ext_intersphinx.py index ef5a9b145b4..770ae60db5c 100644 --- a/tests/test_extensions/test_ext_intersphinx.py +++ b/tests/test_extensions/test_ext_intersphinx.py @@ -1,6 +1,7 @@ """Test the intersphinx extension.""" import http.server +import zlib from unittest import mock import pytest @@ -10,6 +11,7 @@ from sphinx.ext.intersphinx import ( INVENTORY_FILENAME, _get_safe_url, + _process_disabled_reftypes, _strip_basic_auth, fetch_inventory, inspect_main, @@ -44,6 +46,7 @@ def set_config(app, mapping): app.config.intersphinx_mapping = mapping app.config.intersphinx_cache_limit = 0 app.config.intersphinx_disabled_reftypes = [] + _process_disabled_reftypes(app.env) @mock.patch('sphinx.ext.intersphinx.InventoryFile') @@ -198,6 +201,31 @@ def test_missing_reference_pydomain(tmp_path, app, status, warning): assert rn.astext() == 'Foo.bar' +def test_py_old_property(tmp_path, app, status, warning): + inv_file = tmp_path / 'inventory' + inv_file.write_bytes(b'''\ +# Sphinx inventory version 2 +# Project: foo +# Version: 2.0 +# The remainder of this file is compressed with zlib. +''' + zlib.compress(b'''\ +module1.Foo.bar py:method 1 index.html#foo.Bar.baz - +''')) + set_config(app, { + 'https://docs.python.org/': str(inv_file), + }) + + # load the inventory and check if it's done correctly + normalize_intersphinx_mapping(app, app.config) + load_mappings(app) + + # py:attr context helps to search objects + kwargs = {'py:module': 'module1'} + node, contnode = fake_node('py', 'attr', 'Foo.bar', 'Foo.bar', **kwargs) + rn = missing_reference(app, app.env, node, contnode) + assert rn.astext() == 'Foo.bar' + + def test_missing_reference_stddomain(tmp_path, app, status, warning): inv_file = tmp_path / 'inventory' inv_file.write_bytes(INVENTORY_V2) @@ -248,6 +276,30 @@ def test_missing_reference_stddomain(tmp_path, app, status, warning): assert rn.astext() == 'The Julia Domain' +def test_std_old_option(tmp_path, app, status, warning): + inv_file = tmp_path / 'inventory' + inv_file.write_bytes(b'''\ +# Sphinx inventory version 2 +# Project: foo +# Version: 2.0 +# The remainder of this file is compressed with zlib. +''' + zlib.compress(b'''\ +ls.-l std:option 1 index.html#cmdoption-ls-l - +''')) + set_config(app, { + 'cmd': ('https://docs.python.org/', str(inv_file)), + }) + + # load the inventory and check if it's done correctly + normalize_intersphinx_mapping(app, app.config) + load_mappings(app) + + kwargs = {'std:program': 'ls'} + node, contnode = fake_node('std', 'option', '-l', 'ls -l', **kwargs) + rn = missing_reference(app, app.env, node, contnode) + assert rn.astext() == 'ls -l' + + @pytest.mark.sphinx('html', testroot='ext-intersphinx-cppdomain') def test_missing_reference_cppdomain(tmp_path, app, status, warning): inv_file = tmp_path / 'inventory' @@ -346,18 +398,22 @@ def assert_(rn, expected): # the base case, everything should resolve assert app.config.intersphinx_disabled_reftypes == [] + _process_disabled_reftypes(app.env) case(term=True, doc=True, py=True) # disabled a single ref type app.config.intersphinx_disabled_reftypes = ['std:doc'] + _process_disabled_reftypes(app.env) case(term=True, doc=False, py=True) # disabled a whole domain app.config.intersphinx_disabled_reftypes = ['std:*'] + _process_disabled_reftypes(app.env) case(term=False, doc=False, py=True) # disabled all domains app.config.intersphinx_disabled_reftypes = ['*'] + _process_disabled_reftypes(app.env) case(term=False, doc=False, py=False) diff --git a/tests/test_util/test_util_inventory.py b/tests/test_util/test_util_inventory.py index 81d31b0ef44..b094cc98235 100644 --- a/tests/test_util/test_util_inventory.py +++ b/tests/test_util/test_util_inventory.py @@ -5,7 +5,7 @@ import sphinx.locale from sphinx.testing.util import SphinxTestApp -from sphinx.util.inventory import InventoryFile +from sphinx.util.inventory import InventoryFile, InventoryItemSet from tests.test_util.intersphinx_data import ( INVENTORY_V1, @@ -76,3 +76,24 @@ def test_inventory_localization(tmp_path): # Ensure that the inventory contents differ assert inventory_et.read_bytes() != inventory_en.read_bytes() + + +def test_inventory_item_set_repr(): + # Given + item_set = InventoryItemSet() + + # When + item_set._items = { + "test_inventory_name": [ + ( + "project name", + "project version", + "https://project.example/page.html#anchor", + "display name" + ), + ] + } + + # Then + assert repr(item_set) == "InventoryItemSet({'test_inventory_name': [('project name', 'project version', 'https://project.example/page.html#anchor', 'display name')]})" + assert str(item_set) == repr(item_set)