From 01ed4ce3d4323c38cb3aeb91053a2a3adb4dfc94 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Fri, 2 Nov 2018 12:49:15 -0700 Subject: WIP. --- WORKSPACE | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 WORKSPACE (limited to 'WORKSPACE') diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..b03116f --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,15 @@ + +workspace(name = "upb") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "lua", + build_file = "//:lua.BUILD", + sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b", + strip_prefix = "lua-5.2.4", + urls = [ + "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz", + "https://www.lua.org/ftp/lua-5.2.4.tar.gz", + ], +) -- cgit v1.2.3 From 2655ed9cf5e448e6ce7195e9afb9361c0a901df0 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Sat, 3 Nov 2018 13:04:10 -0700 Subject: Bazel builds everything except conformance_upb. --- BUILD | 275 +++++++++++++++++++++++++++++++++++++++++++++++++++-- WORKSPACE | 19 ++++ build_defs.bzl | 106 +++++++++++++++++++-- upb/json/parser.c | 26 +++-- upb/json/parser.rl | 2 +- 5 files changed, 398 insertions(+), 30 deletions(-) (limited to 'WORKSPACE') diff --git a/BUILD b/BUILD index 2dd5818..6f295a0 100644 --- a/BUILD +++ b/BUILD @@ -1,4 +1,14 @@ -load(":build_defs.bzl", "lua_cclibrary", "lua_library", "lua_binary") +load( + ":build_defs.bzl", + "lua_cclibrary", + "lua_library", + "lua_binary", + "lua_test", + "generated_file_staleness_test", + "upb_amalgamation", +) + +# C/C++ rules ################################################################## cc_library( name = "upb", @@ -33,6 +43,7 @@ cc_library( "upb/sink.h", "upb/upb.h", ], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -46,6 +57,7 @@ cc_library( "upb/descriptor/reader.h", ], deps = [":upb"], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -70,6 +82,7 @@ cc_library( ":upb", ":upb_descriptor", ], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -83,8 +96,114 @@ cc_library( "upb/json/printer.h", ], deps = [":upb"], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) +cc_library( + name = "upb_cc_bindings", + hdrs = [ + "upb/bindings/stdc++/string.h", + ], + deps = [":upb"], +) + +# Amalgamation ################################################################# + +py_binary( + name = "amalgamate", + srcs = ["tools/amalgamate.py"], +) + +upb_amalgamation( + name = "gen_amalgamation", + amalgamator = ":amalgamate", + libs = [ + ":upb", + ":upb_descriptor", + ":upb_pb", + ":upb_json", + ], + outs = [ + "upb.h", + "upb.c", + ], +) + +cc_library( + name = "amalgamation", + hdrs = ["upb.h"], + srcs = ["upb.c"], +) + +# C/C++ tests ################################################################## + +cc_library( + testonly = 1, + name = "upb_test", + hdrs = [ + "tests/upb_test.h", + "tests/test_util.h", + ], + srcs = [ + "tests/testmain.cc", + ], +) + +cc_test( + name = "test_varint", + srcs = ["tests/pb/test_varint.c"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_def", + srcs = ["tests/test_def.c"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_handlers", + srcs = ["tests/test_handlers.c"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_decoder", + srcs = ["tests/pb/test_decoder.cc"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_encoder", + srcs = ["tests/pb/test_encoder.cc"], + deps = [":upb_pb", ":upb_test", ":upb_cc_bindings"], + data = ["upb/descriptor/descriptor.pb"], +) + +cc_test( + name = "test_cpp", + srcs = ["tests/test_cpp.cc"], + deps = [":upb_descriptor", ":upb", ":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_table", + srcs = ["tests/test_table.cc"], + deps = [":upb", ":upb_test"], +) + +cc_test( + name = "test_json", + srcs = [ + "tests/json/test_json.cc", + "tests/json/test.upbdefs.h", + "tests/json/test.upbdefs.c", + ], + deps = [":upb_json", ":upb_test"], +) + +# Lua libraries. ############################################################### + lua_cclibrary( name = "lua/upb_c", srcs = [ @@ -103,10 +222,10 @@ lua_cclibrary( lua_library( name = "lua/upb", - base = "upb/bindings/lua", srcs = [ - "upb/bindings/lua/upb.lua" + "upb/bindings/lua/upb.lua", ], + strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb_c", ], @@ -127,10 +246,10 @@ lua_cclibrary( lua_library( name = "lua/upb/table", - base = "upb/bindings/lua", srcs = [ "upb/bindings/lua/upb/table.lua", ], + strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", "lua/upb/table_c", @@ -152,10 +271,10 @@ lua_cclibrary( lua_library( name = "lua/upb/pb", - base = "upb/bindings/lua", srcs = [ "upb/bindings/lua/upb/pb.lua", ], + strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", "lua/upb/pb_c", @@ -164,20 +283,158 @@ lua_library( lua_library( name = "lua/upbc_lib", - base = "tools", srcs = [ "tools/dump_cinit.lua", "tools/make_c_api.lua", ], + strip_prefix = "tools", luadeps = [ "lua/upb", - ] + "lua/upb/table", + ], +) + +# Lua tests. ################################################################### + +lua_test( + name = "lua/test_upb", + luamain = "tests/bindings/lua/test_upb.lua", + luadeps = ["lua/upb"], +) + +lua_test( + name = "lua/test_upb_pb", + luamain = "tests/bindings/lua/test_upb.pb.lua", + luadeps = ["lua/upb/pb"], ) +# upb compiler ################################################################# + lua_binary( name = "upbc", - main = "tools/upbc.lua", luadeps = [ "lua/upbc_lib", - ] + ], + luamain = "tools/upbc.lua", +) + +# Generated files ############################################################## + +exports_files(["staleness_test.py"]) + +py_library( + name = "staleness_test_lib", + testonly = 1, + srcs = ["staleness_test_lib.py"], +) + +genrule( + name = "make_dynasm_decoder", + srcs = [ + "third_party/dynasm/dynasm.lua", + "third_party/dynasm/dasm_x64.lua", + "third_party/dynasm/dasm_x86.lua", + "upb/pb/compile_decoder_x64.dasc", + ], + outs = ["generated/upb/pb/compile_decoder_x64.h"], + cmd = "LUA_PATH=third_party/dynasm/?.lua $(location @lua//:lua) third_party/dynasm/dynasm.lua -c upb/pb/compile_decoder_x64.dasc > $@", + tools = ["@lua"], +) + +proto_library( + name = "upb_descriptor_proto", + srcs = [ + "upb/descriptor/descriptor.proto" + ], +) + +genrule( + name = "copy_upb_descriptor_pb", + outs = ["generated/upb/descriptor/descriptor.pb"], + srcs = [":upb_descriptor_proto"], + cmd = "cp $< $@", +) + +genrule( + name = "generate_old_upbdefs", + srcs = ["generated/upb/descriptor/descriptor.pb"], + tools = [":upbc"], + outs = [ + "generated/upb/descriptor/descriptor.upbdefs.h", + "generated/upb/descriptor/descriptor.upbdefs.c", + ], + cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", +) + +proto_library( + name = "google_descriptor_proto", + srcs = [ + "google/protobuf/descriptor.proto" + ], +) + +genrule( + name = "copy_google_descriptor_pb", + outs = ["generated/google/protobuf/descriptor.pb"], + srcs = [":google_descriptor_proto"], + cmd = "cp $< $@", +) + +genrule( + name = "generate_descriptor_c", + srcs = ["generated/google/protobuf/descriptor.pb"], + tools = [":upbc"], + outs = [ + "generated/google/protobuf/descriptor.upb.h", + "generated/google/protobuf/descriptor.upb.c", + ], + cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC $$INFILE", +) + +proto_library( + name = "json_test_proto", + srcs = ["tests/json/test.proto"], +) + +genrule( + name = "copy_json_test_proto", + srcs = [":json_test_proto"], + outs = ["generated/tests/json/test.proto.pb"], + cmd = "cp $< $@", +) + +genrule( + name = "generated_json_test_proto_upbdefs", + srcs = ["generated/tests/json/test.proto.pb"], + tools = [":upbc"], + outs = [ + "generated/tests/json/test.upbdefs.h", + "generated/tests/json/test.upbdefs.c", + ], + cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", +) + +genrule( + name = "generate_json_ragel", + srcs = ["upb/json/parser.rl"], + outs = ["generated/upb/json/parser.c"], + tools = ["@ragel//:ragel"], + cmd = "$(location @ragel//:ragel) -C -o upb/json/parser.c $< && mv upb/json/parser.c $@", +) + +generated_file_staleness_test( + name = "test_generated_files", + outs = [ + "google/protobuf/descriptor.upb.h", + "google/protobuf/descriptor.upb.c", + "upb/pb/compile_decoder_x64.h", + "upb/descriptor/descriptor.upbdefs.c", + "upb/descriptor/descriptor.pb", + "upb/descriptor/descriptor.upbdefs.h", + "upb/json/parser.c", + "tests/json/test.upbdefs.c", + "tests/json/test.upbdefs.h", + "tests/json/test.proto.pb", + ], + generated_pattern = "generated/%s", ) diff --git a/WORKSPACE b/WORKSPACE index b03116f..299bfa7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -13,3 +13,22 @@ http_archive( "https://www.lua.org/ftp/lua-5.2.4.tar.gz", ], ) + +http_archive( + name = "com_google_protobuf", + sha256 = "d7a221b3d4fb4f05b7473795ccea9e05dab3b8721f6286a95fffbffc2d926f8b", + strip_prefix = "protobuf-3.6.1", + urls = [ + "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.zip" + ], +) + +http_archive( + name = "ragel", + sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", + build_file = "//:ragel.BUILD", + strip_prefix = "ragel-6.10", + urls = [ + "http://www.colm.net/files/ragel/ragel-6.10.tar.gz" + ], +) diff --git a/build_defs.bzl b/build_defs.bzl index 6023bf6..f9cadd4 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -62,8 +62,8 @@ def _remove_prefix(str, prefix): fail("%s doesn't start with %s" % (str, prefix)) return str[len(prefix):] -def lua_library(name, srcs, base, luadeps = []): - outs = [_remove_prefix(src, base + "/") for src in srcs] +def lua_library(name, srcs, strip_prefix, luadeps = []): + outs = [_remove_prefix(src, strip_prefix + "/") for src in srcs] native.genrule( name = name + "_copy", srcs = srcs, @@ -76,7 +76,7 @@ def lua_library(name, srcs, base, luadeps = []): data = outs + luadeps, ) -def lua_binary(name, main, luadeps=[]): +def _lua_binary_or_test(name, luamain, luadeps, rule): script = name + ".sh" script_contents = (shell_find_runfiles + """ @@ -85,16 +85,110 @@ export LUA_CPATH="$BASE/?.so" export LUA_PATH="$BASE/?.lua" $(rlocation lua/lua) $(rlocation upb/tools/upbc.lua) "$@" """).replace("$", "$$") - print(native.repository_name()) native.genrule( name = "gen_" + name, outs = [script], cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % script_contents, ) - native.sh_binary( + rule( name = name, srcs = [script], - data = ["@lua//:lua", "@bazel_tools//tools/bash/runfiles", main] + luadeps, + data = ["@lua//:lua", "@bazel_tools//tools/bash/runfiles", luamain] + luadeps, ) +def lua_binary(name, luamain, luadeps=[]): + _lua_binary_or_test(name, luamain, luadeps, native.sh_binary) + +def lua_test(name, luamain, luadeps=[]): + _lua_binary_or_test(name, luamain, luadeps, native.sh_test) + +def generated_file_staleness_test(name, outs, generated_pattern): + """Tests that checked-in file(s) match the contents of generated file(s). + + The resulting test will verify that all output files exist and have the + correct contents. If the test fails, it can be invoked with --fix to + bring the checked-in files up to date. + + Args: + name: Name of the rule. + outs: the checked-in files that are copied from generated files. + generated_pattern: the pattern for transforming each "out" file into a + generated file. For example, if generated_pattern="generated/%s" then + a file foo.txt will look for generated file generated/foo.txt. + """ + + script_name = name + ".py" + script_src = "//:staleness_test.py" + + # Filter out non-existing rules so Blaze doesn't error out before we even + # run the test. + existing_outs = native.glob(include = outs) + + # The file list contains a few extra bits of information at the end. + # These get unpacked by the Config class in staleness_test_lib.py. + file_list = outs + [generated_pattern, native.package_name() or ".", name] + + native.genrule( + name = name + "_makescript", + outs = [script_name], + srcs = [script_src], + testonly = 1, + cmd = "cat $(location " + script_src + ") > $@; " + + "sed -i 's|INSERT_FILE_LIST_HERE|" + "\\n ".join(file_list) + "|' $@", + ) + + native.py_test( + name = name, + srcs = [script_name], + data = existing_outs + [generated_pattern % file for file in outs], + deps = [ + "//:staleness_test_lib", + ], + ) + +SrcList = provider( + fields = { + 'srcs' : 'list of srcs', + 'hdrs' : 'list of hdrs', + } +) + +def _file_list_aspect_impl(target, ctx): + srcs = [] + hdrs = [] + for src in ctx.rule.attr.srcs: + srcs += src.files.to_list() + for hdr in ctx.rule.attr.hdrs: + hdrs += hdr.files.to_list() + return [SrcList(srcs = srcs, hdrs = hdrs)] + +_file_list_aspect = aspect( + implementation = _file_list_aspect_impl, +) + +def _upb_amalgamation(ctx): + srcs = [] + hdrs = [] + for lib in ctx.attr.libs: + srcs += lib[SrcList].srcs + hdrs += lib[SrcList].hdrs + ctx.actions.run( + inputs = srcs + hdrs, + outputs = ctx.outputs.outs, + arguments = ["", ctx.bin_dir.path + "/"] + [f.path for f in srcs], + progress_message = "Making amalgamation", + executable = ctx.executable.amalgamator, + ) + +upb_amalgamation = rule( + implementation = _upb_amalgamation, + attrs = { + "amalgamator": attr.label( + executable = True, + cfg = "host", + ), + "libs": attr.label_list(aspects = [_file_list_aspect]), + "outs": attr.output_list(), + } +) diff --git a/upb/json/parser.c b/upb/json/parser.c index 3ae4722..be74366 100644 --- a/upb/json/parser.c +++ b/upb/json/parser.c @@ -2366,7 +2366,7 @@ _match: break; case 2: #line 2023 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 24; goto _again;} } + { p--; {stack[top++] = cs; cs = 24;goto _again;} } break; case 3: #line 2027 "upb/json/parser.rl" @@ -2440,17 +2440,17 @@ _match: #line 2082 "upb/json/parser.rl" { if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { - {stack[top++] = cs; cs = 48; goto _again;} + {stack[top++] = cs; cs = 48;goto _again;} } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { - {stack[top++] = cs; cs = 41; goto _again;} + {stack[top++] = cs; cs = 41;goto _again;} } else { - {stack[top++] = cs; cs = 33; goto _again;} + {stack[top++] = cs; cs = 33;goto _again;} } } break; case 21: #line 2093 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 76; goto _again;} } + { p--; {stack[top++] = cs; cs = 76;goto _again;} } break; case 22: #line 2098 "upb/json/parser.rl" @@ -2538,7 +2538,9 @@ _again: switch ( *__acts++ ) { case 0: #line 2019 "upb/json/parser.rl" - { p--; {cs = stack[--top]; goto _again;} } + { p--; {cs = stack[--top]; if ( p == pe ) + goto _test_eof; +goto _again;} } break; case 26: #line 2111 "upb/json/parser.rl" @@ -2564,7 +2566,7 @@ _again: #line 2139 "upb/json/parser.rl" { end_subobject_full(parser); } break; -#line 2568 "upb/json/parser.c" +#line 2570 "upb/json/parser.c" } } } @@ -2589,7 +2591,7 @@ error: return p - buf; } -bool end(void *closure, const void *hd) { +static bool end(void *closure, const void *hd) { upb_json_parser *parser = closure; /* Prevent compile warning on unused static constants. */ @@ -2603,11 +2605,7 @@ bool end(void *closure, const void *hd) { parse(parser, hd, &eof_ch, 0, NULL); - return parser->current_state >= -#line 2608 "upb/json/parser.c" -105 -#line 2202 "upb/json/parser.rl" -; + return parser->current_state >= 105; } static void json_parser_reset(upb_json_parser *p) { @@ -2622,7 +2620,7 @@ static void json_parser_reset(upb_json_parser *p) { /* Emit Ragel initialization of the parser. */ -#line 2626 "upb/json/parser.c" +#line 2624 "upb/json/parser.c" { cs = json_start; top = 0; diff --git a/upb/json/parser.rl b/upb/json/parser.rl index 29efd09..0d59082 100644 --- a/upb/json/parser.rl +++ b/upb/json/parser.rl @@ -2185,7 +2185,7 @@ error: return p - buf; } -bool end(void *closure, const void *hd) { +static bool end(void *closure, const void *hd) { upb_json_parser *parser = closure; /* Prevent compile warning on unused static constants. */ -- cgit v1.2.3 From a7ebe54eb1cc09cc50cf97af8436835ead4cbbee Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Sat, 3 Nov 2018 18:18:47 -0700 Subject: Conformance tests work under Bazel. --- BUILD | 79 ++++++++++++++++++++++++---------------- WORKSPACE | 24 ++++++++----- build_defs.bzl | 95 ++++++++++++++++++++++++++++++++++++++++++++----- tests/conformance_upb.c | 4 +-- tools/upbc.lua | 22 +++++++++--- 5 files changed, 170 insertions(+), 54 deletions(-) (limited to 'WORKSPACE') diff --git a/BUILD b/BUILD index 6f295a0..b636e15 100644 --- a/BUILD +++ b/BUILD @@ -5,7 +5,9 @@ load( "lua_binary", "lua_test", "generated_file_staleness_test", + "make_shell_script", "upb_amalgamation", + "upb_proto_library", ) # C/C++ rules ################################################################## @@ -202,6 +204,43 @@ cc_test( deps = [":upb_json", ":upb_test"], ) +upb_proto_library( + name = "conformance_proto_upb", + deps = [ + "@com_google_protobuf//:conformance_proto", + "@com_google_protobuf//:test_messages_proto3_proto", + ], + upbc = ":upbc", +) + +cc_binary( + name = "conformance_upb", + srcs = [ + "tests/conformance_upb.c", + ], + deps = [ + ":upb", + ":conformance_proto_upb", + ], +) + +make_shell_script( + name = "gen_test_conformance_upb", + out = "test_conformance_upb.sh", + contents = "$(rlocation com_google_protobuf/conformance_test_runner) $(rlocation upb/conformance_upb)", +) + +sh_test( + name = "test_conformance_upb", + srcs = ["test_conformance_upb.sh"], + data = [ + ":conformance_upb", + "@com_google_protobuf//:conformance_test_runner", + "tests/conformance_upb_failures.txt", + "@bazel_tools//tools/bash/runfiles" + ] +) + # Lua libraries. ############################################################### lua_cclibrary( @@ -222,33 +261,21 @@ lua_cclibrary( lua_library( name = "lua/upb", - srcs = [ - "upb/bindings/lua/upb.lua", - ], + srcs = ["upb/bindings/lua/upb.lua"], strip_prefix = "upb/bindings/lua", - luadeps = [ - "lua/upb_c", - ], + luadeps = ["lua/upb_c"], ) lua_cclibrary( name = "lua/upb/table_c", - srcs = [ - "upb/bindings/lua/upb/table.c", - ], - luadeps = [ - "lua/upb_c", - ], - deps = [ - "upb", - ], + srcs = ["upb/bindings/lua/upb/table.c"], + luadeps = ["lua/upb_c"], + deps = ["upb"], ) lua_library( name = "lua/upb/table", - srcs = [ - "upb/bindings/lua/upb/table.lua", - ], + srcs = ["upb/bindings/lua/upb/table.lua"], strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", @@ -258,22 +285,14 @@ lua_library( lua_cclibrary( name = "lua/upb/pb_c", - srcs = [ - "upb/bindings/lua/upb/pb.c", - ], - luadeps = [ - "lua/upb_c", - ], - deps = [ - "upb_pb", - ], + srcs = ["upb/bindings/lua/upb/pb.c"], + luadeps = ["lua/upb_c"], + deps = ["upb_pb"], ) lua_library( name = "lua/upb/pb", - srcs = [ - "upb/bindings/lua/upb/pb.lua", - ], + srcs = ["upb/bindings/lua/upb/pb.lua"], strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", diff --git a/WORKSPACE b/WORKSPACE index 299bfa7..b4d19c5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,6 +2,7 @@ workspace(name = "upb") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") http_archive( name = "lua", @@ -14,13 +15,12 @@ http_archive( ], ) -http_archive( +git_repository( name = "com_google_protobuf", - sha256 = "d7a221b3d4fb4f05b7473795ccea9e05dab3b8721f6286a95fffbffc2d926f8b", - strip_prefix = "protobuf-3.6.1", - urls = [ - "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.zip" - ], + commit = "25feb59620627b673df76813dfd66e3f565765e7", + #sha256 = "d7a221b3d4fb4f05b7473795ccea9e05dab3b8721f6286a95fffbffc2d926f8b", + remote = "https://github.com/haberman/protobuf.git", + #tag = "conformance-build-tag", ) http_archive( @@ -28,7 +28,13 @@ http_archive( sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", build_file = "//:ragel.BUILD", strip_prefix = "ragel-6.10", - urls = [ - "http://www.colm.net/files/ragel/ragel-6.10.tar.gz" - ], + urls = ["http://www.colm.net/files/ragel/ragel-6.10.tar.gz"], +) + +# Used by protobuf. +http_archive( + name = "bazel_skylib", + sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d", + strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], ) diff --git a/build_defs.bzl b/build_defs.bzl index f9cadd4..8df5660 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -1,5 +1,5 @@ -shell_find_runfiles = """ +_shell_find_runfiles = """ # --- begin runfiles.bash initialization --- # Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). set -euo pipefail @@ -62,6 +62,11 @@ def _remove_prefix(str, prefix): fail("%s doesn't start with %s" % (str, prefix)) return str[len(prefix):] +def _remove_suffix(str, suffix): + if not str.endswith(suffix): + fail("%s doesn't end with %s" % (str, suffix)) + return str[:-len(suffix)] + def lua_library(name, srcs, strip_prefix, luadeps = []): outs = [_remove_prefix(src, strip_prefix + "/") for src in srcs] native.genrule( @@ -76,21 +81,28 @@ def lua_library(name, srcs, strip_prefix, luadeps = []): data = outs + luadeps, ) +def make_shell_script(name, contents, out): + script_contents = (_shell_find_runfiles + contents).replace("$", "$$") + native.genrule( + name = "gen_" + name, + outs = [out], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % script_contents, + ) + def _lua_binary_or_test(name, luamain, luadeps, rule): script = name + ".sh" - script_contents = (shell_find_runfiles + """ + make_shell_script( + name = "gen_" + name, + out = script, + contents = """ BASE=$(dirname $(rlocation upb/upb_c.so)) export LUA_CPATH="$BASE/?.so" export LUA_PATH="$BASE/?.lua" $(rlocation lua/lua) $(rlocation upb/tools/upbc.lua) "$@" -""").replace("$", "$$") - - native.genrule( - name = "gen_" + name, - outs = [script], - cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % script_contents, +""" ) + rule( name = name, srcs = [script], @@ -147,6 +159,8 @@ def generated_file_staleness_test(name, outs, generated_pattern): ], ) +# upb_amalgamation() rule, with file_list aspect. + SrcList = provider( fields = { 'srcs' : 'list of srcs', @@ -192,3 +206,68 @@ upb_amalgamation = rule( "outs": attr.output_list(), } ) + +# upb_proto_library() rule + +def _remove_up(string): + if string.startswith("../"): + string = string[3:] + pos = string.find("/") + string = string[pos + 1:] + + return _remove_suffix(string, ".proto") + +def _upb_proto_library_srcs_impl(ctx): + descriptors = [] + outs = [] + for dep in ctx.attr.deps: + if hasattr(dep, 'proto'): + for desc in dep.proto.transitive_descriptor_sets: + descriptors.append(desc) + for src in dep.proto.transitive_sources: + outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.h")) + outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.c")) + outdir = _remove_suffix(outs[-1].path, _remove_up(src.short_path) + ".upb.c") + + concatenated = ctx.actions.declare_file(ctx.label.name + "_concatenated_descriptor.bin") + descriptor_paths = [d.path for d in descriptors] + + ctx.actions.run_shell( + inputs = descriptors, + outputs = [concatenated], + progress_message = "Concatenating descriptors", + command = "cat %s > %s" % (" ".join(descriptor_paths), concatenated.path), + ) + ctx.actions.run( + inputs = [concatenated], + outputs = outs, + executable = ctx.executable.upbc, + arguments = ["--outdir", outdir, concatenated.path], + progress_message = "Generating upb protos", + ) + + return [DefaultInfo(files = depset(outs))] + +_upb_proto_library_srcs = rule( + implementation = _upb_proto_library_srcs_impl, + attrs = { + "upbc": attr.label( + executable = True, + cfg = "host", + ), + "deps": attr.label_list(), + } +) + +def upb_proto_library(name, deps, upbc): + srcs_rule = name + "_srcs.cc" + _upb_proto_library_srcs( + name = srcs_rule, + upbc = upbc, + deps = deps, + ) + native.cc_library( + name = name, + srcs = [":" + srcs_rule], + deps = [":upb"], + ) diff --git a/tests/conformance_upb.c b/tests/conformance_upb.c index ccc8a94..e0a7d8c 100644 --- a/tests/conformance_upb.c +++ b/tests/conformance_upb.c @@ -8,8 +8,8 @@ #include #include -#include "conformance.upb.h" -#include "google/protobuf/test_messages_proto3.upb.h" +#include "conformance/conformance.upb.h" +#include "src/google/protobuf/test_messages_proto3.upb.h" int test_count = 0; diff --git a/tools/upbc.lua b/tools/upbc.lua index 8ac4f73..adea373 100644 --- a/tools/upbc.lua +++ b/tools/upbc.lua @@ -13,11 +13,17 @@ local make_c_api = require "make_c_api" local upb = require "upb" local generate_upbdefs = false +local outdir = "." -for _, argument in ipairs(arg) do +i = 1 +while i <= #arg do + argument = arg[i] if argument.sub(argument, 1, 2) == "--" then if argument == "--generate-upbdefs" then generate_upbdefs = true + elseif argument == "--outdir" then + i = i + 1 + outdir = arg[i] else print("Unknown flag: " .. argument) return 1 @@ -29,6 +35,7 @@ for _, argument in ipairs(arg) do end src = argument end + i = i + 1 end if not src then @@ -40,6 +47,11 @@ function strip_proto(filename) return string.gsub(filename, '%.proto$','') end +local function open(filename) + local full_name = outdir .. "/" .. filename + return assert(io.open(full_name, "w"), "couldn't open " .. full_name) +end + -- Open input/output files. local f = assert(io.open(src, "r"), "couldn't open input file " .. src) local descriptor = f:read("*all") @@ -67,8 +79,8 @@ for _, file in ipairs(files) do if generate_upbdefs then -- Legacy generated defs. - local hfile = assert(io.open(hfilename, "w"), "couldn't open " .. hfilename) - local cfile = assert(io.open(cfilename, "w"), "couldn't open " .. cfilename) + local hfile = open(hfilename) + local cfile = open(cfilename) local happend = dump_cinit.file_appender(hfile) local cappend = dump_cinit.file_appender(cfile) @@ -90,8 +102,8 @@ for _, file in ipairs(files) do print(string.format(" cfilename=%s", cfilename)) end - local hfile = assert(io.open(hfilename, "w"), "couldn't open " .. hfilename) - local cfile = assert(io.open(cfilename, "w"), "couldn't open " .. cfilename) + local hfile = open(hfilename) + local cfile = open(cfilename) local happend = dump_cinit.file_appender(hfile) local cappend = dump_cinit.file_appender(cfile) -- cgit v1.2.3 From 9ce4a77f61c134bbed28bfd5be5cd7dc0e80f5e3 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Wed, 14 Nov 2018 16:46:19 -0800 Subject: Updates to build system for upb C++ compiler. --- BUILD | 42 +++++++++++++++++++++++++++++++++--------- WORKSPACE | 6 ++++++ build_defs.bzl | 28 ++++++++++++++-------------- tools/make_cmakelists.py | 2 +- 4 files changed, 54 insertions(+), 24 deletions(-) (limited to 'WORKSPACE') diff --git a/BUILD b/BUILD index 458463e..6503ed0 100644 --- a/BUILD +++ b/BUILD @@ -249,7 +249,7 @@ cc_test( upb_proto_library( name = "conformance_proto_upb", - upbc = ":upbc", + upbc = ":protoc-gen-upb", deps = [ "@com_google_protobuf//:conformance_proto", "@com_google_protobuf//:test_messages_proto3_proto", @@ -373,13 +373,33 @@ lua_test( # upb compiler ################################################################# lua_binary( - name = "upbc", + name = "lua_upbc", luadeps = [ "lua/upbc_lib", ], luamain = "tools/upbc.lua", ) +cc_library( + name = "upbc_generator", + hdrs = ["upbc/generator.h"], + srcs = ["upbc/generator.cc", "upbc/message_layout.h", "upbc/message_layout.cc"], + deps = [ + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:protoc_lib", + "@absl//absl/strings", + ], +) + +cc_binary( + name = "protoc-gen-upb", + srcs = ["upbc/main.cc"], + deps = [ + ":upbc_generator", + "@com_google_protobuf//:protoc_lib", + ], +) + # Test the CMake build ######################################################### make_shell_script( @@ -394,6 +414,7 @@ sh_test( data = glob([ "CMakeLists.txt", "google/**/*", + "upbc/**/*", "upb/**/*", "tests/**/*", ]) + [ @@ -458,8 +479,8 @@ genrule( "generated/upb/descriptor/descriptor.upbdefs.h", "generated/upb/descriptor/descriptor.upbdefs.c", ], - cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", - tools = [":upbc"], + cmd = "UPBC=$$PWD/$(location :lua_upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", + tools = [":lua_upbc"], ) proto_library( @@ -478,13 +499,16 @@ genrule( genrule( name = "generate_descriptor_c", - srcs = ["generated/google/protobuf/descriptor.pb"], + srcs = ["google/protobuf/descriptor.proto"], outs = [ "generated/google/protobuf/descriptor.upb.h", "generated/google/protobuf/descriptor.upb.c", ], - cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC $$INFILE", - tools = [":upbc"], + cmd = "$(location @com_google_protobuf//:protoc) $< --upb_out=$(GENDIR)/generated --plugin=protoc-gen-upb=$(location :protoc-gen-upb)", + tools = [ + "@com_google_protobuf//:protoc", + ":protoc-gen-upb" + ], ) proto_library( @@ -506,8 +530,8 @@ genrule( "generated/tests/json/test.upbdefs.h", "generated/tests/json/test.upbdefs.c", ], - cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", - tools = [":upbc"], + cmd = "UPBC=$$PWD/$(location :lua_upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", + tools = [":lua_upbc"], ) genrule( diff --git a/WORKSPACE b/WORKSPACE index b4d19c5..8d20852 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,6 +23,12 @@ git_repository( #tag = "conformance-build-tag", ) +git_repository( + name = "absl", + commit = "070f6e47b33a2909d039e620c873204f78809492", + remote = "https://github.com/abseil/abseil-cpp.git", +) + http_archive( name = "ragel", sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", diff --git a/build_defs.bzl b/build_defs.bzl index 0e58bbb..c1dd422 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -218,31 +218,26 @@ def _remove_up(string): return _remove_suffix(string, ".proto") def _upb_proto_library_srcs_impl(ctx): - descriptors = [] + sources = [] outs = [] + include_dirs = {} for dep in ctx.attr.deps: if hasattr(dep, 'proto'): - for desc in dep.proto.transitive_descriptor_sets: - descriptors.append(desc) for src in dep.proto.transitive_sources: + sources.append(src) + include_dirs[_remove_suffix(src.path, _remove_up(src.short_path) + "." + src.extension)] = True outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.h")) outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.c")) outdir = _remove_suffix(outs[-1].path, _remove_up(src.short_path) + ".upb.c") - concatenated = ctx.actions.declare_file(ctx.label.name + "_concatenated_descriptor.bin") - descriptor_paths = [d.path for d in descriptors] + source_paths = [d.path for d in sources] + include_args = ["-I" + root for root in include_dirs.keys()] - ctx.actions.run_shell( - inputs = descriptors, - outputs = [concatenated], - progress_message = "Concatenating descriptors", - command = "cat %s > %s" % (" ".join(descriptor_paths), concatenated.path), - ) ctx.actions.run( - inputs = [concatenated], + inputs = [ctx.executable.upbc] + sources, outputs = outs, - executable = ctx.executable.upbc, - arguments = ["--outdir", outdir, concatenated.path], + executable = ctx.executable.protoc, + arguments = ["--upb_out", outdir, "--plugin=protoc-gen-upb=" + ctx.executable.upbc.path] + include_args + source_paths, progress_message = "Generating upb protos", ) @@ -255,6 +250,11 @@ _upb_proto_library_srcs = rule( executable = True, cfg = "host", ), + "protoc": attr.label( + executable = True, + cfg = "host", + default = "@com_google_protobuf//:protoc", + ), "deps": attr.label_list(), } ) diff --git a/tools/make_cmakelists.py b/tools/make_cmakelists.py index 30e1b93..d7996d8 100755 --- a/tools/make_cmakelists.py +++ b/tools/make_cmakelists.py @@ -34,7 +34,7 @@ class BuildFileFunctions(object): pass def cc_library(self, **kwargs): - if kwargs["name"] == "amalgamation": + if kwargs["name"] == "amalgamation" or kwargs["name"] == "upbc_generator": return files = kwargs.get("srcs", []) + kwargs.get("hdrs", []) self.converter.toplevel += "add_library(%s\n %s)\n" % ( -- cgit v1.2.3