From ea7d1c299090bcd4438fea1478d792c9cfdb56c0 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 10 Nov 2023 19:46:27 -0800 Subject: [PATCH] bazel: Migrate py_proto_library to `@com_github_grpc_grpc` Signed-off-by: Sergii Tkachenko --- api/bazel/BUILD | 2 - api/bazel/api_build_system.bzl | 70 +++++------------------------- api/bazel/external_proto_deps.bzl | 12 ----- api/bazel/repository_locations.bzl | 4 +- bazel/repositories.bzl | 4 +- tools/protodoc/BUILD | 23 +++++++--- tools/protoprint/BUILD | 8 +++- tools/protoxform/BUILD | 10 ++++- 8 files changed, 47 insertions(+), 86 deletions(-) diff --git a/api/bazel/BUILD b/api/bazel/BUILD index 5ac7a0e55c365..c4116598f74c7 100644 --- a/api/bazel/BUILD +++ b/api/bazel/BUILD @@ -7,7 +7,6 @@ load( "EXTERNAL_PROTO_CC_BAZEL_DEP_MAP", "EXTERNAL_PROTO_GO_BAZEL_DEP_MAP", "EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP", - "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP", ) licenses(["notice"]) # Apache 2 @@ -38,6 +37,5 @@ json_data( cc = EXTERNAL_PROTO_CC_BAZEL_DEP_MAP, go = EXTERNAL_PROTO_GO_BAZEL_DEP_MAP, imports = EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP, - py = EXTERNAL_PROTO_PY_BAZEL_DEP_MAP, ), ) diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index 8832aec434d21..1f24149b9f927 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -1,6 +1,6 @@ load("@com_envoyproxy_protoc_gen_validate//bazel:pgv_proto_library.bzl", "pgv_cc_proto_library") load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") -load("@com_google_protobuf//:protobuf.bzl", _py_proto_library = "py_proto_library") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", _py_proto_library = "py_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") load("@io_bazel_rules_go//go:def.bzl", "go_test") load("@rules_proto//proto:defs.bzl", "proto_library") @@ -8,7 +8,6 @@ load( "//bazel:external_proto_deps.bzl", "EXTERNAL_PROTO_CC_BAZEL_DEP_MAP", "EXTERNAL_PROTO_GO_BAZEL_DEP_MAP", - "EXTERNAL_PROTO_PY_BAZEL_DEP_MAP", ) load( "//bazel/cc_proto_descriptor_library:builddefs.bzl", @@ -52,63 +51,6 @@ def _go_proto_mapping(dep): def _cc_proto_mapping(dep): return _proto_mapping(dep, EXTERNAL_PROTO_CC_BAZEL_DEP_MAP, _CC_PROTO_SUFFIX) -def _py_proto_mapping(dep): - return _proto_mapping(dep, EXTERNAL_PROTO_PY_BAZEL_DEP_MAP, _PY_PROTO_SUFFIX) - -# TODO(htuch): Convert this to native py_proto_library once -# https://github.com/bazelbuild/bazel/issues/3935 and/or -# https://github.com/bazelbuild/bazel/issues/2626 are resolved. -def _api_py_proto_library(name, srcs = [], deps = []): - mapped_deps = [_py_proto_mapping(dep) for dep in deps] - mapped_unique_deps = {k: True for k in mapped_deps}.keys() - _py_proto_library( - name = name + _PY_PROTO_SUFFIX, - srcs = srcs, - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - deps = mapped_unique_deps + [ - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", - "@com_google_googleapis//google/rpc:status_py_proto", - "@com_google_googleapis//google/api:annotations_py_proto", - "@com_google_googleapis//google/api:http_py_proto", - "@com_google_googleapis//google/api:httpbody_py_proto", - ], - visibility = ["//visibility:public"], - ) - -# This defines googleapis py_proto_library. The repository does not provide its definition and requires -# overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details). -def py_proto_library(name, deps = [], plugin = None): - srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps] - proto_deps = [] - - # py_proto_library in googleapis specifies *_proto rules in dependencies. - # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved. - # As a workaround, manually specify the proto dependencies for the imported python rules. - if name == "annotations_py_proto": - proto_deps = proto_deps + [":http_py_proto"] - - # checked.proto depends on syntax.proto, we have to add this dependency manually as well. - if name == "checked_py_proto": - proto_deps = proto_deps + [":syntax_py_proto"] - - # Special handling for expr_proto target - if srcs[0] == ":expr_moved.proto": - srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto"] - proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"] - - # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0: - # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72. - # plugin should also be passed in here when gRPC version is greater than v1.25.x. - _py_proto_library( - name = name, - srcs = srcs, - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - deps = proto_deps + ["@com_google_protobuf//:protobuf_python"], - visibility = ["//visibility:public"], - ) - def _api_cc_grpc_library(name, proto, deps = []): cc_grpc_library( name = name, @@ -157,7 +99,15 @@ def api_cc_py_proto_library( deps = [relative_name], visibility = ["//visibility:public"], ) - _api_py_proto_library(name, srcs, deps) + + # Uses gRPC implementation of py_proto_library. + # https://github.com/grpc/grpc/blob/v1.59.1/bazel/python_rules.bzl#L160 + _py_proto_library( + name = name + _PY_PROTO_SUFFIX, + # Actual dependencies are resolved automatically from the proto_library dep tree. + deps = [relative_name], + visibility = ["//visibility:public"], + ) # Optionally define gRPC services if has_services: diff --git a/api/bazel/external_proto_deps.bzl b/api/bazel/external_proto_deps.bzl index 212850dd752ad..331934132ab61 100644 --- a/api/bazel/external_proto_deps.bzl +++ b/api/bazel/external_proto_deps.bzl @@ -49,15 +49,3 @@ EXTERNAL_PROTO_CC_BAZEL_DEP_MAP = { "@opentelemetry_proto//:metrics": "@opentelemetry_proto//:metrics_cc_proto", "@opentelemetry_proto//:common": "@opentelemetry_proto//:common_cc_proto", } - -# This maps from the Bazel proto_library target to the Python language binding target for external dependencies. -EXTERNAL_PROTO_PY_BAZEL_DEP_MAP = { - "@com_google_googleapis//google/api/expr/v1alpha1:checked_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto", - "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_py_proto", - "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_py", - "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto_py", - "@opentelemetry_proto//:trace": "@opentelemetry_proto//:trace_py_proto", - "@opentelemetry_proto//:logs": "@opentelemetry_proto//:logs_py_proto", - "@opentelemetry_proto//:metrics": "@opentelemetry_proto//:metrics_py_proto", - "@opentelemetry_proto//:common": "@opentelemetry_proto//:common_py_proto", -} diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 80b100d45f394..bb021c9808ea2 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -39,8 +39,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_desc = "xDS API Working Group (xDS-WG)", project_url = "https://github.com/cncf/xds", # During the UDPA -> xDS migration, we aren't working with releases. - version = "523115ebc1014a83e9cf1e85194ef8f8739d87c7", - sha256 = "3bd28b29380372d54848111b12e24ec684f890032b42b2719ee6971658016b72", + version = "83c031ea693357fdf7a7fea9a68a785d97864a38", + sha256 = "35bdcdbcd2e437058ad1f74a91b49525339b028a6b52760ab019fd9efa5a6d44", release_date = "2023-06-07", strip_prefix = "xds-{version}", urls = ["https://github.com/cncf/xds/archive/{version}.tar.gz"], diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index a28ce2e347adf..5b09395533f80 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -368,10 +368,8 @@ def envoy_dependencies(skip_targets = []): name = "com_google_googleapis_imports", cc = True, go = True, + python = True, grpc = True, - rules_override = { - "py_proto_library": ["@envoy_api//bazel:api_build_system.bzl", ""], - }, ) native.bind( name = "bazel_runfiles", diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 2802576bca3c1..16178951349a1 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -1,5 +1,6 @@ load("@base_pip3//:requirements.bzl", "requirement") -load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") load("//bazel:envoy_build_system.bzl", "envoy_package") load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_jinja_env", "envoy_py_data", "envoy_pytool_binary", "envoy_pytool_library") load("//tools/protodoc:protodoc.bzl", "protodoc_rule") @@ -21,10 +22,22 @@ envoy_pytool_binary( ], ) -py_proto_library( +proto_library( name = "manifest_proto", srcs = ["manifest.proto"], - deps = ["@com_google_protobuf//:protobuf_python"], + deps = [ + "@com_google_protobuf//:struct_proto", + ], +) + +py_proto_library( + name = "manifest_py_pb2", + deps = [":manifest_proto"], +) + +py_proto_library( + name = "validate_py_pb2", + deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"], ) envoy_py_data( @@ -39,7 +52,7 @@ envoy_pytool_binary( data = ["@envoy_api//:v3_proto_set"], deps = [ requirement("envoy.base.utils"), - ":manifest_proto", + ":manifest_py_pb2", ":protodoc_manifest_untyped", "@com_google_protobuf//:protobuf_python", ], @@ -110,8 +123,8 @@ envoy_pytool_binary( deps = [ ":data", ":jinja", + ":validate_py_pb2", "//tools/api_proto_plugin", - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto", requirement("envoy.code.check"), diff --git a/tools/protoprint/BUILD b/tools/protoprint/BUILD index 7b3b786a79c15..02bca13066b8a 100644 --- a/tools/protoprint/BUILD +++ b/tools/protoprint/BUILD @@ -1,4 +1,5 @@ load("@base_pip3//:requirements.bzl", "requirement") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") load("//bazel:envoy_build_system.bzl", "envoy_package") @@ -22,6 +23,7 @@ envoy_pytool_binary( ], visibility = ["//visibility:public"], deps = [ + ":validate_py_pb2", "//tools/api_versioning:utils", "//tools/protoxform:options", "//tools/protoxform:utils", @@ -29,7 +31,6 @@ envoy_pytool_binary( requirement("packaging"), "//tools/type_whisperer", "//tools/type_whisperer:api_type_db_proto_py_proto", - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto", "@com_google_googleapis//google/api:annotations_py_proto", @@ -38,6 +39,11 @@ envoy_pytool_binary( ], ) +py_proto_library( + name = "validate_py_pb2", + deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"], +) + protoprint_rule( name = "protoprinted_srcs", deps = [ diff --git a/tools/protoxform/BUILD b/tools/protoxform/BUILD index d1be2d6e7a810..500f801e547e0 100644 --- a/tools/protoxform/BUILD +++ b/tools/protoxform/BUILD @@ -1,4 +1,5 @@ load("@aspect_bazel_lib//lib:jq.bzl", "jq") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") load("@rules_python//python:defs.bzl", "py_binary", "py_library") @@ -15,7 +16,6 @@ py_binary( ":utils", "//tools/api_proto_plugin", "//tools/type_whisperer:api_type_db_proto_py_proto", - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_github_cncf_xds//udpa/annotations:pkg_py_proto", "@com_github_cncf_xds//xds/annotations/v3:pkg_py_proto", "@com_google_googleapis//google/api:annotations_py_proto", @@ -33,6 +33,14 @@ py_library( name = "utils", srcs = ["utils.py"], visibility = ["//visibility:public"], + deps = [ + ":validate_py_pb2", + ], +) + +py_proto_library( + name = "validate_py_pb2", + deps = ["@com_envoyproxy_protoc_gen_validate//validate:validate_proto"], ) protoxform_rule(