Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 72 additions & 50 deletions astroid/brain/brain_builtin_inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,36 +192,74 @@ def on_bootstrap():
)


def _builtin_filter_predicate(node, builtin_name) -> bool:
# Mapping of builtin name β†’ inference function. Populated by
# ``register_builtin_transform``; consumed by the single dispatcher transform
# registered for ``nodes.Call``. This collapses what was previously one
# transform-list entry per builtin (~19 entries each running their own
# predicate per Call node) into a single entry that does the
# ``isinstance``/name lookup once and dispatches via dict.
_BUILTIN_INFERENCE_FUNCS: dict[
str,
Callable[[nodes.Call, InferenceContext | None], SuccessfulInferenceResult | None],
] = {}


def _builtin_dispatch_predicate(node: nodes.Call) -> bool:
# pylint: disable = too-many-boolean-expressions
if (
builtin_name == "type"
and node.root().name == "re"
and isinstance(node.func, nodes.Name)
and node.func.name == "type"
and isinstance(node.parent, nodes.Assign)
and len(node.parent.targets) == 1
and isinstance(node.parent.targets[0], nodes.AssignName)
and node.parent.targets[0].name in {"Pattern", "Match"}
):
# Handle re.Pattern and re.Match in brain_re
# Match these patterns from stdlib/re.py
# ```py
# Pattern = type(...)
# Match = type(...)
# ```
return False
if isinstance(node.func, nodes.Name):
return node.func.name == builtin_name
if isinstance(node.func, nodes.Attribute):
func = node.func
if isinstance(func, nodes.Name):
name = func.name
if name not in _BUILTIN_INFERENCE_FUNCS:
return False
if (
name == "type"
and node.root().name == "re"
and isinstance(node.parent, nodes.Assign)
and len(node.parent.targets) == 1
and isinstance(node.parent.targets[0], nodes.AssignName)
and node.parent.targets[0].name in {"Pattern", "Match"}
):
# Handle re.Pattern and re.Match in brain_re β€” they look like
# ``Pattern = type(...)`` / ``Match = type(...)`` in stdlib/re.py
# and must not be inferred as the builtin ``type`` call.
return False
return True
if isinstance(func, nodes.Attribute):
return (
node.func.attrname == "fromkeys"
and isinstance(node.func.expr, nodes.Name)
and node.func.expr.name == "dict"
func.attrname == "fromkeys"
and isinstance(func.expr, nodes.Name)
and func.expr.name == "dict"
and "dict.fromkeys" in _BUILTIN_INFERENCE_FUNCS
)
return False


def _builtin_dispatch_transform(
node: nodes.Call, context: InferenceContext | None = None
) -> Iterator:
func = node.func
if isinstance(func, nodes.Name):
builtin_name = func.name
else:
builtin_name = "dict.fromkeys"
transform = _BUILTIN_INFERENCE_FUNCS[builtin_name]
result = transform(node, context=context)
if result:
if not result.parent:
# Let the transformation function determine
# the parent for its result. Otherwise,
# we set it to be the node we transformed from.
result.parent = node

if result.lineno is None:
result.lineno = node.lineno
# Can be a 'Module' see https://github.com/pylint-dev/pylint/issues/4671
# We don't have a regression test on this one: tread carefully
if hasattr(result, "col_offset") and result.col_offset is None:
result.col_offset = node.col_offset
return iter([result])


def register_builtin_transform(
manager: AstroidManager, transform, builtin_name
) -> None:
Expand All @@ -230,31 +268,8 @@ def register_builtin_transform(
The transform function must accept two parameters, a node and
an optional context.
"""

def _transform_wrapper(
node: nodes.Call, context: InferenceContext | None = None
) -> Iterator:
result = transform(node, context=context)
if result:
if not result.parent:
# Let the transformation function determine
# the parent for its result. Otherwise,
# we set it to be the node we transformed from.
result.parent = node

if result.lineno is None:
result.lineno = node.lineno
# Can be a 'Module' see https://github.com/pylint-dev/pylint/issues/4671
# We don't have a regression test on this one: tread carefully
if hasattr(result, "col_offset") and result.col_offset is None:
result.col_offset = node.col_offset
return iter([result])

manager.register_transform(
nodes.Call,
inference_tip(_transform_wrapper),
partial(_builtin_filter_predicate, builtin_name=builtin_name),
)
del manager # No longer needed; dispatcher is registered once globally.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This feels dangerous. Is the del really necessary?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, we could use a noqa instead. But the other discussion might make this one obsolete.

_BUILTIN_INFERENCE_FUNCS[builtin_name] = transform


def _container_generic_inference(
Expand Down Expand Up @@ -1158,7 +1173,14 @@ def _infer_str_format_call(


def register(manager: AstroidManager) -> None:
# Builtins inference
# Builtins inference β€” registered through a single dispatcher to avoid
# running 19 separate predicates on every Call node walked by the
# transform visitor (#1115).
manager.register_transform(
nodes.Call,
inference_tip(_builtin_dispatch_transform),
_builtin_dispatch_predicate,
)
register_builtin_transform(manager, infer_bool, "bool")
register_builtin_transform(manager, infer_super, "super")
register_builtin_transform(manager, infer_callable, "callable")
Comment on lines +1176 to 1186

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we still registering the transforms here?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Didn't want to modify the API for register_builtin_transform (no leading underscore). If we remove this constraint, there's a lot we can do in a simpler way.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't expect non core astroid to ever need to call register_builtin_transform, so I'd be fine with changing the API :)

Expand Down
7 changes: 7 additions & 0 deletions doc/whatsnew/fragments/3167.performance
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
The 19 ``register_builtin_transform`` registrations are collapsed into a single
dispatcher transform on ``nodes.Call``. The transform visitor previously ran 19
predicates per ``Call`` node, each repeating the same ``isinstance`` checks; the
dispatcher does them once and routes by name via a dict.

Refs #3167
Refs #1115
93 changes: 93 additions & 0 deletions tests/test_transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import pytest

from astroid import MANAGER, builder, nodes, parse, transforms
from astroid.brain import brain_builtin_inference
from astroid.brain.brain_builtin_inference import _builtin_dispatch_predicate
from astroid.brain.brain_dataclasses import _looks_like_dataclass_field_call
from astroid.const import IS_PYPY
from astroid.manager import AstroidManager
Expand Down Expand Up @@ -273,3 +275,94 @@ def transform_call(node: nodes.Call) -> nodes.Const:
assert "sys.setrecursionlimit" in records[0].message.args[0]
finally:
sys.setrecursionlimit(original_limit)


class TestBuiltinDispatcher(unittest.TestCase):
"""Cover the per-Call dispatcher that replaced 19 individual transforms.

The predicate is the hot path of the transform visitor β€” wrong answers
here either silently drop builtin inference (false negatives) or apply
builtin inference to user-defined names (false positives), so each
branch is worth a dedicated regression test.
"""

def test_known_builtin_name_dispatches(self) -> None:
"""``bool(...)`` resolves to the builtin's inferred constant."""
node = builder.extract_node("bool(1) #@")
inferred = next(node.infer())
assert isinstance(inferred, nodes.Const)
assert inferred.value is True

def test_dict_fromkeys_attribute_dispatches(self) -> None:
"""``dict.fromkeys(...)`` flows through the Attribute branch of the predicate."""
node = builder.extract_node("dict.fromkeys(['a', 'b']) #@")
inferred = next(node.infer())
assert isinstance(inferred, nodes.Dict)
keys = sorted(k.value for k, _ in inferred.items)
assert keys == ["a", "b"]

def test_unknown_name_is_not_dispatched(self) -> None:
"""A Call to a non-builtin name must not be claimed by the dispatcher."""
node = builder.extract_node("not_a_builtin(1, 2) #@")
assert _builtin_dispatch_predicate(node) is False

def test_non_dict_attribute_is_not_dispatched(self) -> None:
"""Only ``dict.fromkeys`` Attributes are claimed; e.g., ``list.fromkeys``
is not, and neither are unrelated ``dict`` methods."""
list_fromkeys = builder.extract_node("list.fromkeys([1, 2]) #@")
assert _builtin_dispatch_predicate(list_fromkeys) is False

dict_other = builder.extract_node("dict.values() #@")
assert _builtin_dispatch_predicate(dict_other) is False

def test_dynamic_call_target_is_not_dispatched(self) -> None:
"""Calls whose ``func`` is neither a Name nor an Attribute are skipped."""
node = builder.extract_node("(lambda x: x)(1) #@")
assert _builtin_dispatch_predicate(node) is False

def test_re_pattern_and_match_type_calls_are_not_dispatched(self) -> None:
"""``Pattern = type(...)`` / ``Match = type(...)`` in stdlib ``re`` are
handed off to ``brain_re`` rather than the builtin ``type`` inference."""
re_ast = MANAGER.ast_from_module_name("re")
# In modern Python's stdlib these are produced via
# ``Pattern = type(...)`` / ``Match = type(...)`` assignments.
for name in ("Pattern", "Match"):
assign = re_ast.locals[name][0].parent
assert isinstance(assign, nodes.Assign)
assert isinstance(assign.value, nodes.Call)
assert isinstance(assign.value.func, nodes.Name)
assert assign.value.func.name == "type"
# The re-module type() call must not be claimed by the builtin
# dispatcher; brain_re owns its inference.
assert _builtin_dispatch_predicate(assign.value) is False

def test_register_builtin_transform_populates_dispatch_table(self) -> None:
"""``register_builtin_transform`` must add the function to the dispatch
dict so the global dispatcher can route to it."""
marker = object()

def fake_transform(node, context=None):
return marker

original = brain_builtin_inference._BUILTIN_INFERENCE_FUNCS.get(
"_test_register_builtin"
)
try:
brain_builtin_inference.register_builtin_transform(
AstroidManager(), fake_transform, "_test_register_builtin"
)
assert (
brain_builtin_inference._BUILTIN_INFERENCE_FUNCS[
"_test_register_builtin"
]
is fake_transform
)
finally:
if original is None:
brain_builtin_inference._BUILTIN_INFERENCE_FUNCS.pop(
"_test_register_builtin", None
)
else:
brain_builtin_inference._BUILTIN_INFERENCE_FUNCS[
"_test_register_builtin"
] = original