henry/whisper-com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ADD: added new version of protobuf

Browse Source
This commit is contained in:
Henry Winkel
2022-12-20 10:09:28 +01:00
parent 4a79559129
commit 1e2b3dda7b
1513 changed files with 123720 additions and 83381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

536
libs/protobuf/protobuf.bzl
View File

@@ -1,7 +1,7 @@
load("@bazel_skylib//lib:versions.bzl", "versions")
load("@rules_cc//cc:defs.bzl", "objc_library")
load("@rules_cc//cc:defs.bzl", "cc_library")
load("@rules_proto//proto:defs.bzl", "ProtoInfo")
load("@rules_python//python:defs.bzl", "py_library")
load("@rules_python//python:defs.bzl", "py_library", "py_test")
def _GetPath(ctx, path):
if ctx.label.workspace_root:
@@ -37,24 +37,20 @@ def _SourceDir(ctx):
return _GetPath(ctx, ctx.attr.includes[0])
return _GetPath(ctx, ctx.label.package + "/" + ctx.attr.includes[0])
def _ObjcBase(srcs):
return [
"".join([token.capitalize() for token in src[:-len(".proto")].split("_")])
for src in srcs
]
def _CcHdrs(srcs, use_grpc_plugin = False):
ret = [s[:-len(".proto")] + ".pb.h" for s in srcs]
if use_grpc_plugin:
ret += [s[:-len(".proto")] + ".grpc.pb.h" for s in srcs]
return ret
def _ObjcHdrs(srcs):
return [src + ".pbobjc.h" for src in _ObjcBase(srcs)]
def _CcSrcs(srcs, use_grpc_plugin = False):
ret = [s[:-len(".proto")] + ".pb.cc" for s in srcs]
if use_grpc_plugin:
ret += [s[:-len(".proto")] + ".grpc.pb.cc" for s in srcs]
return ret
def _ObjcSrcs(srcs):
return [src + ".pbobjc.m" for src in _ObjcBase(srcs)]
def _ObjcOuts(srcs, out_type):
if out_type == "hdrs":
return _ObjcHdrs(srcs)
if out_type == "srcs":
return _ObjcSrcs(srcs)
return _ObjcHdrs(srcs) + _ObjcSrcs(srcs)
def _CcOuts(srcs, use_grpc_plugin = False):
return _CcHdrs(srcs, use_grpc_plugin) + _CcSrcs(srcs, use_grpc_plugin)
def _PyOuts(srcs, use_grpc_plugin = False):
ret = [s[:-len(".proto")] + "_pb2.py" for s in srcs]
@@ -62,29 +58,29 @@ def _PyOuts(srcs, use_grpc_plugin = False):
ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs]
return ret
def _RubyOuts(srcs):
return [s[:-len(".proto")] + "_pb.rb" for s in srcs]
def _RelativeOutputPath(path, include, dest = ""):
if include == None:
return path
def _CsharpOuts(srcs):
return [
"".join([token.capitalize() for token in src[:-len(".proto")].split("_")]) + ".cs"
for src in srcs
]
if not path.startswith(include):
fail("Include path %s isn't part of the path %s." % (include, path))
ProtoGenInfo = provider(
fields = ["srcs", "import_flags", "deps"],
)
if include and include[-1] != "/":
include = include + "/"
if dest and dest[-1] != "/":
dest = dest + "/"
path = path[len(include):]
return dest + path
def _proto_gen_impl(ctx):
"""General implementation for generating protos"""
srcs = ctx.files.srcs
langs = ctx.attr.langs or []
out_type = ctx.attr.out_type
deps = depset(direct = ctx.files.srcs)
deps = depset(direct=ctx.files.srcs)
source_dir = _SourceDir(ctx)
gen_dir = _GenDir(ctx).rstrip("/")
import_flags = []
if source_dir:
has_sources = any([src.is_source for src in srcs])
if has_sources:
@@ -96,38 +92,27 @@ def _proto_gen_impl(ctx):
if has_generated:
import_flags += ["-I" + gen_dir]
if ctx.attr.includes:
for include in ctx.attr.includes:
import_flags += ["-I" + _GetPath(ctx, include)]
import_flags = depset(direct = import_flags)
import_flags = depset(direct=import_flags)
for dep in ctx.attr.deps:
dep_proto = dep[ProtoGenInfo]
if type(dep_proto.import_flags) == "list":
import_flags = depset(
transitive = [import_flags],
direct = dep_proto.import_flags,
)
if type(dep.proto.import_flags) == "list":
import_flags = depset(transitive=[import_flags], direct=dep.proto.import_flags)
else:
import_flags = depset(
transitive = [import_flags, dep_proto.import_flags],
)
if type(dep_proto.deps) == "list":
deps = depset(transitive = [deps], direct = dep_proto.deps)
import_flags = depset(transitive=[import_flags, dep.proto.import_flags])
if type(dep.proto.deps) == "list":
deps = depset(transitive=[deps], direct=dep.proto.deps)
else:
deps = depset(transitive = [deps, dep_proto.deps])
deps = depset(transitive=[deps, dep.proto.deps])
if not langs and not ctx.executable.plugin:
return [
ProtoGenInfo(
if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin:
return struct(
proto = struct(
srcs = srcs,
import_flags = import_flags,
deps = deps,
),
]
)
generated_files = []
for src in srcs:
args = []
@@ -138,28 +123,17 @@ def _proto_gen_impl(ctx):
path = f.replace("-I", "")
import_flags_real.append("-I$(realpath -s %s)" % path)
outs = []
use_grpc_plugin = (ctx.attr.plugin_language == "grpc" and ctx.attr.plugin)
path_tpl = "$(realpath %s)" if in_gen_dir else "%s"
if ctx.attr.gen_cc:
args += [("--cpp_out=" + path_tpl) % gen_dir]
outs.extend(_CcOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
if ctx.attr.gen_py:
args += [("--python_out=" + path_tpl) % gen_dir]
outs.extend(_PyOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
outs = []
for lang in langs:
if lang == "csharp":
outs.extend(_CsharpOuts([src.basename]))
elif lang == "objc":
outs.extend(_ObjcOuts([src.basename], out_type = out_type))
elif lang == "python":
outs.extend(_PyOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
elif lang == "ruby":
outs.extend(_RubyOuts([src.basename]))
# Otherwise, rely on user-supplied outs.
args += [("--%s_out=" + path_tpl) % (lang, gen_dir)]
if ctx.attr.outs:
outs.extend(ctx.attr.outs)
outs = [ctx.actions.declare_file(out, sibling = src) for out in outs]
generated_files.extend(outs)
inputs = [src] + deps.to_list()
tools = [ctx.executable.protoc]
if ctx.executable.plugin:
@@ -212,15 +186,39 @@ def _proto_gen_impl(ctx):
use_default_shell_env = True,
)
return [
ProtoGenInfo(
return struct(
proto = struct(
srcs = srcs,
import_flags = import_flags,
deps = deps,
),
DefaultInfo(files = depset(generated_files)),
]
)
proto_gen = rule(
attrs = {
"srcs": attr.label_list(allow_files = True),
"deps": attr.label_list(providers = ["proto"]),
"includes": attr.string_list(),
"protoc": attr.label(
cfg = "exec",
executable = True,
allow_single_file = True,
mandatory = True,
),
"plugin": attr.label(
cfg = "exec",
allow_files = True,
executable = True,
),
"plugin_language": attr.string(),
"plugin_options": attr.string_list(),
"gen_cc": attr.bool(),
"gen_py": attr.bool(),
"outs": attr.output_list(),
},
output_to_genfiles = True,
implementation = _proto_gen_impl,
)
"""Generates codes from Protocol Buffers definitions.
This rule helps you to implement Skylark macros specific to the target
@@ -237,40 +235,103 @@ Args:
compiler.
plugin_language: the language of the generated sources
plugin_options: a list of options to be passed to the plugin
langs: generates sources in addition to the ones from the plugin for each
specified language.
gen_cc: generates C++ sources in addition to the ones from the plugin.
gen_py: generates Python sources in addition to the ones from the plugin.
outs: a list of labels of the expected outputs from the protocol compiler.
out_type: only generated a single type of source file for languages that have
split sources (e.g. *.h and *.cc in C++)
"""
_proto_gen = rule(
def _adapt_proto_library_impl(ctx):
deps = [dep[ProtoInfo] for dep in ctx.attr.deps]
srcs = [src for dep in deps for src in dep.direct_sources]
return struct(
proto = struct(
srcs = srcs,
import_flags = ["-I{}".format(path) for dep in deps for path in dep.transitive_proto_path.to_list()],
deps = srcs,
),
)
adapt_proto_library = rule(
implementation = _adapt_proto_library_impl,
attrs = {
"srcs": attr.label_list(allow_files = True),
"deps": attr.label_list(providers = [ProtoGenInfo]),
"includes": attr.string_list(),
"protoc": attr.label(
cfg = "exec",
executable = True,
allow_single_file = True,
"deps": attr.label_list(
mandatory = True,
),
"plugin": attr.label(
cfg = "exec",
allow_files = True,
executable = True,
),
"plugin_language": attr.string(),
"plugin_options": attr.string_list(),
"langs": attr.string_list(),
"outs": attr.string_list(),
"out_type": attr.string(
default = "all",
providers = [ProtoInfo],
),
},
output_to_genfiles = True,
implementation = _proto_gen_impl,
doc = "Adapts `proto_library` from `@rules_proto` to be used with `{cc,py}_proto_library` from this file.",
)
def cc_proto_library(
name,
srcs = [],
deps = [],
cc_libs = [],
include = None,
protoc = "@com_google_protobuf//:protoc",
use_grpc_plugin = False,
default_runtime = "@com_google_protobuf//:protobuf",
**kargs):
"""Bazel rule to create a C++ protobuf library from proto source files
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
Args:
name: the name of the cc_proto_library.
srcs: the .proto files of the cc_proto_library.
deps: a list of dependency labels; must be cc_proto_library.
cc_libs: a list of other cc_library targets depended by the generated
cc_library.
include: a string indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources.
use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
when processing the proto files.
default_runtime: the implicitly default runtime which will be depended on by
the generated cc_library target.
**kargs: other keyword arguments that are passed to cc_library.
"""
includes = []
if include != None:
includes = [include]
grpc_cpp_plugin = None
if use_grpc_plugin:
grpc_cpp_plugin = "//external:grpc_cpp_plugin"
gen_srcs = _CcSrcs(srcs, use_grpc_plugin)
gen_hdrs = _CcHdrs(srcs, use_grpc_plugin)
outs = gen_srcs + gen_hdrs
proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = [s + "_genproto" for s in deps],
includes = includes,
protoc = protoc,
plugin = grpc_cpp_plugin,
plugin_language = "grpc",
gen_cc = 1,
outs = outs,
visibility = ["//visibility:public"],
)
if default_runtime and not default_runtime in cc_libs:
cc_libs = cc_libs + [default_runtime]
if use_grpc_plugin:
cc_libs = cc_libs + ["//external:grpc_lib"]
cc_library(
name = name,
srcs = gen_srcs,
hdrs = gen_hdrs,
deps = cc_libs + deps,
includes = includes,
**kargs
)
def _internal_gen_well_known_protos_java_impl(ctx):
args = ctx.actions.args()
@@ -395,97 +456,42 @@ internal_gen_kt_protos = rule(
},
)
def internal_objc_proto_library(
name,
srcs = [],
deps = [],
outs = [],
proto_deps = [],
includes = ["."],
default_runtime = "@com_google_protobuf//:protobuf_objc",
protoc = "@com_google_protobuf//:protoc",
testonly = None,
visibility = ["//visibility:public"],
**kwargs):
"""Bazel rule to create a Objective-C protobuf library from proto source
files
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
"""Macro to copy files to a different directory and then create a filegroup.
This is used by the //:protobuf_python py_proto_library target to work around
an issue caused by Python source files that are part of the same Python
package being in separate directories.
Args:
name: the name of the objc_proto_library.
srcs: the .proto files to compile.
deps: a list of dependency labels; must be objc_proto_library.
outs: a list of expected output files.
proto_deps: a list of proto file dependencies that don't have a
objc_proto_library rule.
include: a string indicating the include path of the .proto files.
default_runtime: the Objective-C Protobuf runtime
protoc: the label of the protocol compiler to generate the sources.
testonly: common rule attribute (see:
https://bazel.build/reference/be/common-definitions#common-attributes)
visibility: the visibility of the generated files.
**kwargs: other keyword arguments that are passed to py_library.
srcs: The source files to copy and add to the filegroup.
strip_prefix: Path to the root of the files to copy.
dest: The directory to copy the source files into.
**kwargs: extra arguments that will be passesd to the filegroup.
"""
full_deps = [d + "_genproto" for d in deps]
outs = [_RelativeOutputPath(s, strip_prefix, dest) for s in srcs]
if proto_deps:
_proto_gen(
name = name + "_deps_genproto",
testonly = testonly,
srcs = proto_deps,
protoc = protoc,
includes = includes,
)
full_deps.append(":%s_deps_genproto" % name)
# Note: we need to run the protoc build twice to get separate targets for
# the generated header and the source files.
_proto_gen(
name = name + "_genproto_hdrs",
native.genrule(
name = name + "_genrule",
srcs = srcs,
deps = full_deps,
langs = ["objc"],
out_type = "hdrs",
includes = includes,
protoc = protoc,
testonly = testonly,
visibility = visibility,
tags = ["manual"],
outs = outs,
cmd_bash = " && ".join(
["cp $(location %s) $(location %s)" %
(s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs]),
cmd_bat = " && ".join(
["@copy /Y $(location %s) $(location %s) >NUL" %
(s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs]),
)
_proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = full_deps,
langs = ["objc"],
out_type = "srcs",
includes = includes,
protoc = protoc,
testonly = testonly,
visibility = visibility,
tags = ["manual"],
)
objc_library(
native.filegroup(
name = name,
hdrs = [name + "_genproto_hdrs"],
non_arc_srcs = [name + "_genproto"],
deps = [default_runtime],
includes = includes,
testonly = testonly,
visibility = visibility,
# Don't auto-expand these targets until target_compatible_with
# works. See https://github.com/bazelbuild/bazel/issues/12897.
tags = ["manual"],
target_compatible_with = ["@platforms//os:osx"],
srcs = outs,
**kwargs
)
def internal_py_proto_library(
def py_proto_library(
name,
srcs = [],
deps = [],
@@ -495,7 +501,6 @@ def internal_py_proto_library(
default_runtime = "@com_google_protobuf//:protobuf_python",
protoc = "@com_google_protobuf//:protoc",
use_grpc_plugin = False,
testonly = None,
**kargs):
"""Bazel rule to create a Python protobuf library from proto source files
@@ -517,11 +522,11 @@ def internal_py_proto_library(
protoc: the label of the protocol compiler to generate the sources.
use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin
when processing the proto files.
testonly: common rule attribute (see:
https://bazel.build/reference/be/common-definitions#common-attributes)
**kargs: other keyword arguments that are passed to py_library.
"""
outs = _PyOuts(srcs, use_grpc_plugin)
includes = []
if include != None:
includes = [include]
@@ -533,14 +538,14 @@ def internal_py_proto_library(
# is not explicitly listed in py_libs. Instead, host system is assumed to
# have grpc installed.
_proto_gen(
proto_gen(
name = name + "_genproto",
testonly = testonly,
srcs = srcs,
deps = [s + "_genproto" for s in deps],
includes = includes,
protoc = protoc,
langs = ["python"],
gen_py = 1,
outs = outs,
visibility = ["//visibility:public"],
plugin = grpc_python_plugin,
plugin_language = "grpc",
@@ -550,158 +555,33 @@ def internal_py_proto_library(
py_libs = py_libs + [default_runtime]
py_library(
name = name,
testonly = testonly,
srcs = [name + "_genproto"] + py_extra_srcs,
srcs = outs + py_extra_srcs,
deps = py_libs + deps,
imports = includes,
**kargs
)
def py_proto_library(
*args,
**kwargs):
"""Deprecated alias for use before Bazel 5.3.
Args:
*args: the name of the py_proto_library.
**kwargs: other keyword arguments that are passed to py_library.
Deprecated:
This is provided for backwards compatibility only. Bazel 5.3 will
introduce support for py_proto_library, which should be used instead.
"""
internal_py_proto_library(*args, **kwargs)
def _source_proto_library(
def internal_protobuf_py_tests(
name,
srcs = [],
deps = [],
proto_deps = [],
outs = [],
lang = None,
includes = ["."],
protoc = "@com_google_protobuf//:protoc",
testonly = None,
visibility = ["//visibility:public"],
**kwargs):
"""Bazel rule to create generated protobuf code from proto source files for
languages not well supported by Bazel yet. This will output the generated
code as-is without any compilation. This is most useful for interpreted
languages that don't require it.
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
modules = [],
**kargs):
"""Bazel rules to create batch tests for protobuf internal.
Args:
name: the name of the unsupported_proto_library.
srcs: the .proto files to compile. Note, that for languages where out
needs to be provided, only a single source file is allowed.
deps: a list of dependency labels; must be unsupported_proto_library.
proto_deps: a list of proto file dependencies that don't have a
unsupported_proto_library rule.
lang: the language to (optionally) generate code for.
outs: a list of expected output files. This is only required for
languages where we can't predict the outputs.
includes: strings indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources.
testonly: common rule attribute (see:
https://bazel.build/reference/be/common-definitions#common-attributes)
visibility: the visibility of the generated files.
**kwargs: other keyword arguments that are passed to py_library.
name: the name of the rule.
modules: a list of modules for tests. The macro will create a py_test for
each of the parameter with the source "google/protobuf/%s.py"
kargs: extra parameters that will be passed into the py_test.
"""
if outs and len(srcs) != 1:
fail("Custom outputs only allowed for single proto targets.")
langs = []
if lang != None:
langs = [lang]
full_deps = [d + "_genproto" for d in deps]
if proto_deps:
_proto_gen(
name = name + "_deps_genproto",
testonly = testonly,
srcs = proto_deps,
protoc = protoc,
includes = includes,
for m in modules:
s = "python/google/protobuf/internal/%s.py" % m
py_test(
name = "py_%s" % m,
srcs = [s],
main = s,
**kargs
)
full_deps.append(":%s_deps_genproto" % name)
_proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = full_deps,
langs = langs,
outs = outs,
includes = includes,
protoc = protoc,
testonly = testonly,
visibility = visibility,
)
native.filegroup(
name = name,
srcs = [":%s_genproto" % name],
testonly = testonly,
visibility = visibility,
**kwargs
)
def internal_csharp_proto_library(**kwargs):
"""Bazel rule to create a C# protobuf library from proto source files
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
Args:
**kwargs: arguments that are passed to unsupported_proto_library.
"""
_source_proto_library(
lang = "csharp",
**kwargs
)
def internal_php_proto_library(**kwargs):
"""Bazel rule to create a PHP protobuf library from proto source files
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
Args:
**kwargs: arguments that are passed to unsupported_proto_library.
"""
if not kwargs.get("outs"):
fail("Unable to predict the outputs for php_proto_library. Please specify them via `outs`.")
_source_proto_library(
lang = "php",
**kwargs
)
def internal_ruby_proto_library(**kwargs):
"""Bazel rule to create a Ruby protobuf library from proto source files
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
Args:
**kwargs: arguments that are passed to unsupported_proto_library.
"""
_source_proto_library(
lang = "ruby",
**kwargs
)
def check_protobuf_required_bazel_version():
"""For WORKSPACE files, to check the installed version of bazel.