bazel: Migrate py_proto_library to @com_github_grpc_grpc (#74)

py_proto_library provided by @com_google_protobuf//:protobuf.bzl has been deprecated for a while now:

This is provided for backwards compatibility only. Bazel 5.3 will
introduce support for py_proto_library, which should be used instead.
https://github.com/protocolbuffers/protobuf/blob/32af7d211b85f71920acdfa9b796db008f8c2a45/protobuf.bzl#L642-L644

However, native py_proto_library has never been provided, see bazelbuild/bazel#3935. Instead @rules_python//python:proto.bzl is recommended. I attempted switching to this library, but it's not compatible with @com_google_googleapis's py_proto_library targets, see #69. I found a hacky workaround by using cc_proto_library to generate python targets, but downstream integration into Envoy failed (envoyproxy/envoy#30159).

This PR migrates py_proto_library implementation to to @com_github_grpc_grpc. This implementation is used by @com_google_googleapis's, and, more importantly, uses bazel aspects. Which decouples cncf/xds and Envoy's dependencies from concrete upstream py_proto_library implementations.

This resulted in a significant code improvement of bazel/api_build_system.bzl:

No more custom @com_google_googleapis dependency mapping needed via py_proto_library rules override.
No more hardcoded dependencies _xds_py_proto_library - proto dependency tree is determined from proto_library definitions via Basel aspects.
No more EXTERNAL_PROTO_PY_BAZEL_DEP_MAP dependency map needed - for the same reason.
Similar work in Envoy: envoyproxy/envoy#30834.

Signed-off-by: Sergii Tkachenko <[email protected]>

Author: sergiitk

bazel/api_build_system.bzl

@@ -1,13 +1,12 @@
 load("@com_envoyproxy_protoc_gen_validate//bazel:pgv_proto_library.bzl", "pgv_cc_proto_library")
-load("@com_google_protobuf//:protobuf.bzl", _py_proto_library = "py_proto_library")
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", _py_proto_library = "py_proto_library")
 load("@io_bazel_rules_go//go:def.bzl", "go_test")
 load("@io_bazel_rules_go//proto:def.bzl", "go_grpc_library", "go_proto_library")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 load(
     "//bazel:external_proto_deps.bzl",
     "EXTERNAL_PROTO_CC_BAZEL_DEP_MAP",
     "EXTERNAL_PROTO_GO_BAZEL_DEP_MAP",
-    "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP",
 )
 
 _PY_PROTO_SUFFIX = "_py_proto"
@@ -43,65 +42,6 @@ def _go_proto_mapping(dep):
 def _cc_proto_mapping(dep):
     return _proto_mapping(dep, EXTERNAL_PROTO_CC_BAZEL_DEP_MAP, _CC_PROTO_SUFFIX)
 
-def _py_proto_mapping(dep):
-    return _proto_mapping(dep, EXTERNAL_PROTO_PY_BAZEL_DEP_MAP, _PY_PROTO_SUFFIX)
-
-# TODO(htuch): Convert this to native py_proto_library once
-# https://github.com/bazelbuild/bazel/issues/3935 and/or
-# https://github.com/bazelbuild/bazel/issues/2626 are resolved.
-def _xds_py_proto_library(name, srcs = [], deps = []):
-    mapped_deps = [_py_proto_mapping(dep) for dep in deps]
-    mapped_unique_deps = []
-    [mapped_unique_deps.append(d) for d in mapped_deps if d not in mapped_unique_deps]
-    _py_proto_library(
-        name = name + _PY_PROTO_SUFFIX,
-        srcs = srcs,
-        default_runtime = "@com_google_protobuf//:protobuf_python",
-        protoc = "@com_google_protobuf//:protoc",
-        deps = mapped_unique_deps + [
-            "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
-            "@com_google_googleapis//google/rpc:status_py_proto",
-            "@com_google_googleapis//google/api:annotations_py_proto",
-            "@com_google_googleapis//google/api:http_py_proto",
-            "@com_google_googleapis//google/api:httpbody_py_proto",
-        ],
-        visibility = ["//visibility:public"],
-    )
-
-# This defines googleapis py_proto_library. The repository does not provide its definition and requires
-# overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details).
-def py_proto_library(name, deps = [], plugin = None):
-    srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps]
-    proto_deps = []
-
-    # py_proto_library in googleapis specifies *_proto rules in dependencies.
-    # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved.
-    # As a workaround, manually specify the proto dependencies for the imported python rules.
-    if name == "annotations_py_proto":
-        proto_deps = proto_deps + [":http_py_proto"]
-
-    # checked.proto depends on syntax.proto, we have to add this dependency manually as well.
-    if name == "checked_py_proto":
-        proto_deps = proto_deps + [":syntax_py_proto"]
-
-    # Special handling for expr_proto target
-    if srcs[0] == ":expr_moved.proto":
-        srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto",]
-        proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"]
-
-
-    # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0:
-    # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72.
-    # plugin should also be passed in here when gRPC version is greater than v1.25.x.
-    _py_proto_library(
-        name = name,
-        srcs = srcs,
-        default_runtime = "@com_google_protobuf//:protobuf_python",
-        protoc = "@com_google_protobuf//:protoc",
-        deps = proto_deps + ["@com_google_protobuf//:protobuf_python"],
-        visibility = ["//visibility:public"],
-    )
-
 def _xds_cc_py_proto_library(
         name,
         visibility = ["//visibility:private"],
@@ -129,7 +69,15 @@ def _xds_cc_py_proto_library(
         deps = [relative_name],
         visibility = ["//visibility:public"],
     )
-    _xds_py_proto_library(name, srcs, deps)
+
+    # Uses gRPC implementation of py_proto_library.
+    # https://github.com/grpc/grpc/blob/v1.59.1/bazel/python_rules.bzl#L160
+    _py_proto_library(
+        name = name + _PY_PROTO_SUFFIX,
+        # Actual dependencies are resolved automatically from the proto_library dep tree.
+        deps = [relative_name],
+        visibility = ["//visibility:public"],
+    )
 
     # Optionally define gRPC services
     if has_services:

bazel/dependency_imports.bzl

@@ -3,25 +3,34 @@ load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
 load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
 load("@com_envoyproxy_protoc_gen_validate//bazel:repositories.bzl", "pgv_dependencies")
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 
 # go version for rules_go
 GO_VERSION = "1.20.2"
 
 def xds_dependency_imports(go_version = GO_VERSION):
+    rules_proto_dependencies()
+    rules_proto_toolchains()
     protobuf_deps()
     go_rules_dependencies()
     go_register_toolchains(go_version)
     gazelle_dependencies()
     pgv_dependencies()
 
+    # Needed for grpc's @com_github_grpc_grpc//bazel:python_rules.bzl
+    # Used in place of calling grpc_deps() because it needs to be called before
+    # loading `grpc_extra_deps.bzl` - which is not allowed in a method def context.
+    native.bind(
+        name = "protocol_compiler",
+        actual = "@com_google_protobuf//:protoc",
+    )
+
     switched_rules_by_language(
         name = "com_google_googleapis_imports",
         cc = True,
         go = True,
+        python = True,
         grpc = True,
-        rules_override = {
-            "py_proto_library": ["@com_github_cncf_xds//bazel:api_build_system.bzl", "",],
-        },
     )
 
     # These dependencies, like most of the Go in this repository, exist only for the API.

bazel/external_proto_deps.bzl

@@ -36,9 +36,3 @@ EXTERNAL_PROTO_CC_BAZEL_DEP_MAP = {
     "@com_google_googleapis//google/api/expr/v1alpha1:checked_proto": "@com_google_googleapis//google/api/expr/v1alpha1:checked_cc_proto",
     "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:syntax_cc_proto",
 }
-
-# This maps from the Bazel proto_library target to the Python language binding target for external dependencies.
-EXTERNAL_PROTO_PY_BAZEL_DEP_MAP = {
-    "@com_google_googleapis//google/api/expr/v1alpha1:checked_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto",
-    "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto",
-}

bazel/repositories.bzl

@@ -26,6 +26,10 @@ def xds_api_dependencies():
         "io_bazel_rules_go",
         locations = REPOSITORY_LOCATIONS,
     )
+    xds_http_archive(
+        "rules_proto",
+        locations = REPOSITORY_LOCATIONS,
+    )
 
 # Old name for backward compatibility.
 # TODO(roth): Remove once all callers are updated to use the new name.

bazel/repository_locations.bzl

@@ -13,9 +13,9 @@ REPOSITORY_LOCATIONS = dict(
         urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/refs/tags/v0.6.1.tar.gz"],
     ),
     com_github_grpc_grpc = dict(
-        sha256 = "13e7c6460cd979726e5b3b129bb01c34532f115883ac696a75eb7f1d6a9765ed",
-        strip_prefix = "grpc-1.40.0",
-        urls = ["https://github.com/grpc/grpc/archive/refs/tags/v1.40.0.tar.gz"],
+        sha256 = "916f88a34f06b56432611aaa8c55befee96d0a7b7d7457733b9deeacbc016f99",
+        strip_prefix = "grpc-1.59.1",
+        urls = ["https://github.com/grpc/grpc/archive/refs/tags/v1.59.1.tar.gz"],
     ),
     com_google_googleapis = dict(
         # TODO(dio): Consider writing a Skylark macro for importing Google API proto.
@@ -24,9 +24,9 @@ REPOSITORY_LOCATIONS = dict(
         urls = ["https://github.com/googleapis/googleapis/archive/114a745b2841a044e98cdbb19358ed29fcf4a5f1.tar.gz"],
     ),
     com_google_protobuf = dict(
-        sha256 = "52b6160ae9266630adb5e96a9fc645215336371a740e87d411bfb63ea2f268a0",
-        strip_prefix = "protobuf-3.18.0",
-        urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v3.18.0/protobuf-all-3.18.0.tar.gz"],
+        sha256 = "8242327e5df8c80ba49e4165250b8f79a76bd11765facefaaecfca7747dc8da2",
+        strip_prefix = "protobuf-3.21.5",
+        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.21.5.zip"],
     ),
     io_bazel_rules_go = dict(
         sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996",
@@ -35,4 +35,9 @@ REPOSITORY_LOCATIONS = dict(
             "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip",
         ],
     ),
+    rules_proto = dict(
+        sha256 = "80d3a4ec17354cccc898bfe32118edd934f851b03029d63ef3fc7c8663a7415c",
+        strip_prefix = "rules_proto-5.3.0-21.5",
+        urls = ["https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.5.tar.gz",],
+    ),
 )