Skip to content

Commit 65ab7b1

Browse files
tamirdtamird
committed
Test generated Python stubs with type checkers
Add generated-stub golden and syntax coverage together with a hermetic mypy, ty, Pyright, and Pyrefly consumer test. Exercise imports, maps, well-known types, proto2 self-field collisions, and realistic construction. Snapshot each checker's inferred types and diagnostics from one expression fixture, including invalid field values and unknown constructor arguments. Run the checker test once in the Linux and macOS Bazel lanes instead of every Python and UPB matrix lane. Keep its dependencies development-only. Integrate generated-stub goldens with stale-file regeneration and provide an explicit fixer for the semantic checker snapshots.
1 parent caa55cb commit 65ab7b1

25 files changed

Lines changed: 800 additions & 1 deletion

‎MODULE.bazel‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,31 @@ bazel_dep(name = "rules_python", version = "1.6.0")
4343
bazel_dep(name = "rules_rust", version = "0.69.0")
4444

4545
bazel_dep(name = "rules_ruby", version = "0.20.1", dev_dependency = True)
46+
bazel_dep(name = "rules_multitool", version = "1.11.1", dev_dependency = True)
47+
bazel_dep(name = "rules_nodejs", version = "6.7.5", dev_dependency = True)
48+
49+
multitool = use_extension("@rules_multitool//multitool:extension.bzl", "multitool", dev_dependency = True)
50+
multitool.hub(
51+
hub_name = "pyi_test_tools",
52+
lockfile = "//python/pyi_test:multitool.lock.json",
53+
)
54+
use_repo(multitool, "multitool", "pyi_test_tools")
55+
register_toolchains("@pyi_test_tools//toolchains:all", dev_dependency = True)
56+
57+
node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node", dev_dependency = True)
58+
node.toolchain(node_version = "22.23.1")
59+
use_repo(node, "nodejs_toolchains")
60+
61+
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
62+
63+
http_archive(
64+
name = "pyright",
65+
build_file_content = 'exports_files(["index.js"], visibility = ["//visibility:public"])\nfilegroup(name = "dist", srcs = glob(["dist/**"], exclude = ["dist/typeshed-fallback/stubs/**"]), visibility = ["//visibility:public"])',
66+
dev_dependency = True,
67+
sha256 = "bd5c488fc20fa237a944279bf32cae2f986cf10d5d5d9e8705819859daeb2f4a",
68+
strip_prefix = "package",
69+
urls = ["https://registry.npmjs.org/pyright/-/pyright-1.1.411.tgz"],
70+
)
4671

4772
# Workaround for https://github.com/bazelbuild/bazel-central-registry/issues/4230
4873
# rules_fuzzing 0.5.3 is not yet available in BCR.
@@ -153,7 +178,17 @@ pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_depen
153178
for python_version in SUPPORTED_PYTHON_VERSIONS
154179
]
155180

156-
use_repo(pip, "protobuf_pip_deps")
181+
[
182+
pip.parse(
183+
hub_name = "pyi_test_pip_deps",
184+
python_interpreter_target = "@system_python//:interpreter",
185+
python_version = python_version,
186+
requirements_lock = "//python/pyi_test:requirements.txt",
187+
)
188+
for python_version in SUPPORTED_PYTHON_VERSIONS
189+
]
190+
191+
use_repo(pip, "protobuf_pip_deps", "pyi_test_pip_deps")
157192

158193
local_runtime_repo = use_repo_rule(
159194
"@rules_python//python/local_toolchains:repos.bzl",

‎bazel/tests/BUILD‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ cc_proto_library_test_suite(name = "cc_proto_library_tests")
2929

3030
proto_bzl_test_suite_tests(name = "proto_bzl_test_suite")
3131

32+
test_suite(
33+
name = "pyi_typecheck_tests",
34+
tests = ["//python/pyi_test:typecheck_golden_test"],
35+
)
36+
3237
bzl_library(
3338
name = "proto_bzl_test_suite_bzl",
3439
srcs = ["proto_bzl_test_suite.bzl"],

‎python/pyi_test/BUILD.bazel‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
load("@rules_python//python:py_binary.bzl", "py_binary")
2+
load("@rules_python//python:py_test.bzl", "py_test")
3+
load("//bazel:proto_library.bzl", "proto_library")
4+
load("//bazel:py_proto_library.bzl", "py_proto_library")
5+
load(":pyi_test.bzl", "node_tool", "pyi_files")
6+
7+
package(default_applicable_licenses = ["//:license"])
8+
9+
proto_library(
10+
name = "fixture_import_proto",
11+
srcs = ["fixture_import.proto"],
12+
)
13+
14+
py_proto_library(
15+
name = "fixture_import_py_pb2",
16+
deps = [":fixture_import_proto"],
17+
)
18+
19+
proto_library(
20+
name = "fixture_proto",
21+
srcs = ["fixture.proto"],
22+
deps = [
23+
":fixture_import_proto",
24+
"//:duration_proto",
25+
"//:timestamp_proto",
26+
],
27+
)
28+
29+
py_proto_library(
30+
name = "fixture_py_pb2",
31+
deps = [":fixture_proto"],
32+
)
33+
34+
pyi_files(
35+
name = "fixture_pyi",
36+
dep = ":fixture_py_pb2",
37+
)
38+
39+
proto_library(
40+
name = "self_fields_proto",
41+
srcs = ["self_fields.proto"],
42+
)
43+
44+
py_proto_library(
45+
name = "self_fields_py_pb2",
46+
deps = [":self_fields_proto"],
47+
)
48+
49+
pyi_files(
50+
name = "self_fields_pyi",
51+
dep = ":self_fields_py_pb2",
52+
)
53+
54+
py_test(
55+
name = "golden_test",
56+
size = "small",
57+
srcs = ["golden_test.py"],
58+
args = [
59+
"$(locations :fixture_pyi)",
60+
"$(locations :self_fields_pyi)",
61+
],
62+
data = glob(["*_pb2.pyi.golden"]) + [
63+
":fixture_pyi",
64+
":self_fields_pyi",
65+
],
66+
tags = [
67+
"manual",
68+
"staleness_test",
69+
],
70+
)
71+
72+
py_binary(
73+
name = "mypy_bin",
74+
srcs = ["mypy_main.py"],
75+
main = "mypy_main.py",
76+
deps = ["@pyi_test_pip_deps//mypy"],
77+
tags = ["manual"],
78+
)
79+
80+
node_tool(
81+
name = "node",
82+
tags = ["manual"],
83+
)
84+
85+
py_test(
86+
name = "typecheck_golden_test",
87+
size = "small",
88+
srcs = ["typecheck_test.py"],
89+
main = "typecheck_test.py",
90+
data = [
91+
"mypy.golden",
92+
"mypy.ini",
93+
"pyrefly.golden",
94+
"pyrefly.toml",
95+
"pyright.golden",
96+
"pyrightconfig.json",
97+
"ty.golden",
98+
"ty.toml",
99+
"typing_test.py",
100+
":fixture_py_pb2",
101+
":fixture_pyi",
102+
":mypy_bin",
103+
":node",
104+
":self_fields_pyi",
105+
":self_fields_py_pb2",
106+
"//:protobuf_python",
107+
"@pyi_test_tools//tools/pyrefly",
108+
"@pyi_test_tools//tools/ty",
109+
"@pyright//:dist",
110+
"@pyright//:index.js",
111+
],
112+
tags = [
113+
"manual",
114+
"staleness_test",
115+
],
116+
target_compatible_with = select({
117+
"@platforms//os:windows": ["@platforms//:incompatible"],
118+
"//conditions:default": [],
119+
}),
120+
visibility = ["//bazel/tests:__pkg__"],
121+
)

‎python/pyi_test/fixture.proto‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Protocol Buffers - Google's data interchange format
2+
// Copyright 2008 Google Inc. All rights reserved.
3+
//
4+
// Use of this source code is governed by a BSD-style
5+
// license that can be found in the LICENSE file or at
6+
// https://developers.google.com/open-source/licenses/bsd
7+
8+
syntax = "proto3";
9+
10+
package protobuf.pyi_test;
11+
12+
import "google/protobuf/duration.proto";
13+
import "google/protobuf/timestamp.proto";
14+
import "python/pyi_test/fixture_import.proto";
15+
16+
enum State {
17+
STATE_UNSPECIFIED = 0;
18+
STATE_READY = 1;
19+
}
20+
21+
message Child {
22+
string value = 1;
23+
}
24+
25+
message Step {
26+
enum Kind {
27+
KIND_UNSPECIFIED = 0;
28+
KIND_VALUE = 1;
29+
}
30+
31+
message Nested {
32+
oneof event {
33+
string started = 1;
34+
string finished = 2;
35+
}
36+
}
37+
38+
oneof action {
39+
string a = 1;
40+
int32 b = 2;
41+
}
42+
43+
oneof result {
44+
bool ok = 3;
45+
string error = 4;
46+
}
47+
48+
optional string detail = 5;
49+
repeated string labels = 6;
50+
repeated Child children = 7;
51+
map<string, Child> children_by_name = 8;
52+
State state = 9;
53+
bytes payload = 10;
54+
Imported imported = 11;
55+
google.protobuf.Timestamp created_at = 12;
56+
google.protobuf.Duration elapsed = 13;
57+
}

‎python/pyi_test/fixture_import.proto‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Protocol Buffers - Google's data interchange format
2+
// Copyright 2008 Google Inc. All rights reserved.
3+
//
4+
// Use of this source code is governed by a BSD-style
5+
// license that can be found in the LICENSE file or at
6+
// https://developers.google.com/open-source/licenses/bsd
7+
8+
syntax = "proto3";
9+
10+
package protobuf.pyi_test;
11+
12+
message Imported {
13+
string value = 1;
14+
}

‎python/pyi_test/fixture_import_pb2.pyi.golden‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from google.protobuf import descriptor as _descriptor
2+
from google.protobuf import message as _message
3+
from typing import ClassVar as _ClassVar, Optional as _Optional
4+
5+
DESCRIPTOR: _descriptor.FileDescriptor
6+
7+
class Imported(_message.Message):
8+
__slots__ = ("value",)
9+
VALUE_FIELD_NUMBER: _ClassVar[int]
10+
value: str
11+
def __init__(self, value: _Optional[str] = ...) -> None: ...

‎python/pyi_test/fixture_pb2.pyi.golden‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import datetime
2+
3+
from google.protobuf import duration_pb2 as _duration_pb2
4+
from google.protobuf import timestamp_pb2 as _timestamp_pb2
5+
from python.pyi_test import fixture_import_pb2 as _fixture_import_pb2
6+
from google.protobuf.internal import containers as _containers
7+
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
8+
from google.protobuf import descriptor as _descriptor
9+
from google.protobuf import message as _message
10+
from collections.abc import Iterable as _Iterable, Mapping as _Mapping
11+
from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
12+
13+
DESCRIPTOR: _descriptor.FileDescriptor
14+
15+
class State(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
16+
__slots__ = ()
17+
STATE_UNSPECIFIED: _ClassVar[State]
18+
STATE_READY: _ClassVar[State]
19+
STATE_UNSPECIFIED: State
20+
STATE_READY: State
21+
22+
class Child(_message.Message):
23+
__slots__ = ("value",)
24+
VALUE_FIELD_NUMBER: _ClassVar[int]
25+
value: str
26+
def __init__(self, value: _Optional[str] = ...) -> None: ...
27+
28+
class Step(_message.Message):
29+
__slots__ = ("a", "b", "ok", "error", "detail", "labels", "children", "children_by_name", "state", "payload", "imported", "created_at", "elapsed")
30+
class Kind(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
31+
__slots__ = ()
32+
KIND_UNSPECIFIED: _ClassVar[Step.Kind]
33+
KIND_VALUE: _ClassVar[Step.Kind]
34+
KIND_UNSPECIFIED: Step.Kind
35+
KIND_VALUE: Step.Kind
36+
class Nested(_message.Message):
37+
__slots__ = ("started", "finished")
38+
STARTED_FIELD_NUMBER: _ClassVar[int]
39+
FINISHED_FIELD_NUMBER: _ClassVar[int]
40+
started: str
41+
finished: str
42+
def __init__(self, started: _Optional[str] = ..., finished: _Optional[str] = ...) -> None: ...
43+
class ChildrenByNameEntry(_message.Message):
44+
__slots__ = ("key", "value")
45+
KEY_FIELD_NUMBER: _ClassVar[int]
46+
VALUE_FIELD_NUMBER: _ClassVar[int]
47+
key: str
48+
value: Child
49+
def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[Child, _Mapping]] = ...) -> None: ...
50+
A_FIELD_NUMBER: _ClassVar[int]
51+
B_FIELD_NUMBER: _ClassVar[int]
52+
OK_FIELD_NUMBER: _ClassVar[int]
53+
ERROR_FIELD_NUMBER: _ClassVar[int]
54+
DETAIL_FIELD_NUMBER: _ClassVar[int]
55+
LABELS_FIELD_NUMBER: _ClassVar[int]
56+
CHILDREN_FIELD_NUMBER: _ClassVar[int]
57+
CHILDREN_BY_NAME_FIELD_NUMBER: _ClassVar[int]
58+
STATE_FIELD_NUMBER: _ClassVar[int]
59+
PAYLOAD_FIELD_NUMBER: _ClassVar[int]
60+
IMPORTED_FIELD_NUMBER: _ClassVar[int]
61+
CREATED_AT_FIELD_NUMBER: _ClassVar[int]
62+
ELAPSED_FIELD_NUMBER: _ClassVar[int]
63+
a: str
64+
b: int
65+
ok: bool
66+
error: str
67+
detail: str
68+
labels: _containers.RepeatedScalarFieldContainer[str]
69+
children: _containers.RepeatedCompositeFieldContainer[Child]
70+
children_by_name: _containers.MessageMap[str, Child]
71+
state: State
72+
payload: bytes
73+
imported: _fixture_import_pb2.Imported
74+
created_at: _timestamp_pb2.Timestamp
75+
elapsed: _duration_pb2.Duration
76+
def __init__(self, a: _Optional[str] = ..., b: _Optional[int] = ..., ok: _Optional[bool] = ..., error: _Optional[str] = ..., detail: _Optional[str] = ..., labels: _Optional[_Iterable[str]] = ..., children: _Optional[_Iterable[_Union[Child, _Mapping]]] = ..., children_by_name: _Optional[_Mapping[str, Child]] = ..., state: _Optional[_Union[State, str]] = ..., payload: _Optional[bytes] = ..., imported: _Optional[_Union[_fixture_import_pb2.Imported, _Mapping]] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., elapsed: _Optional[_Union[datetime.timedelta, _duration_pb2.Duration, _Mapping]] = ...) -> None: ...

0 commit comments

Comments
 (0)