diff options
| -rw-r--r-- | .travis.yml | 23 | ||||
| -rw-r--r-- | BUILD | 1 | ||||
| -rw-r--r-- | Makefile | 457 | ||||
| -rw-r--r-- | kokoro/ubuntu/build.sh | 7 | ||||
| -rw-r--r-- | tools/make_c_api.lua | 575 | ||||
| -rw-r--r-- | tools/upbc.lua | 43 |
6 files changed, 16 insertions, 1090 deletions
diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 56ab91c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: cpp -compiler: - - gcc - - clang -install: - - ./travis.sh install - - export PATH=$PATH:$PWD/protoc # used by genfiles_install() -script: ./travis.sh script -after_success: ./travis.sh after_success -after_failure: ./travis.sh after_failure -env: - global: - - secure: "Rk5UxBrSGzyuJsOc7DO3ZTSMj0LbKTpBKS7PlvYIPgcU8+DYd3ZbkvWuJb1qD3CDp0J/9X8XWK2c51taiNlSsiqwS6/ympj7VX/k5VbtG0NauWwMoNZnpB0JHaIW1Qn5O/rI4B0zLCraObD/ythXeFOzevpMbfZcB5DIbNsgD8c=" - - secure: "E9G9109pmBVh085+f3ZDFCUqObCjHJymZW/knx0/ABJ3xK9O91RXEEkQTIaKDDasHcx9eujU9yK8F6gFgFulEZLxLvS5if5RFXeK5G1YX/MxWjA6Jh2j0dnSbGdd5Q+Lj8/tBqo50ry59qGYqaUPZ9aCXanal3ymbl2lA96n5qE=" - matrix: - - UPB_TRAVIS_BUILD=bare - - UPB_TRAVIS_BUILD=barejit - - UPB_TRAVIS_BUILD=core32 - - UPB_TRAVIS_BUILD=lua - - UPB_TRAVIS_BUILD=ndebug - - UPB_TRAVIS_BUILD=coverage - - UPB_TRAVIS_BUILD=genfiles - - UPB_TRAVIS_BUILD=amalgamated @@ -302,7 +302,6 @@ lua_library( name = "lua/upbc_lib", srcs = [ "tools/dump_cinit.lua", - "tools/make_c_api.lua", ], luadeps = [ "lua/upb", diff --git a/Makefile b/Makefile deleted file mode 100644 index 0a17e48..0000000 --- a/Makefile +++ /dev/null @@ -1,457 +0,0 @@ -# -# This Makefile builds the upb library as well as associated tests, tools, and -# language extensions. -# -# It does not use autoconf/automake/libtool in order to stay lightweight and -# avoid the need for running ./configure. -# -# Summary of compiler flags you may want to use: -# -# * -UNDEBUG: enables assertions() (makes binary larger and slower) -# * -O0: disable optimizations -# * -g: enable debug symbols -# * -fomit-frame-pointer: makes code smaller and faster by freeing up a reg. -# -# Threading: -# * -DUPB_THREAD_UNSAFE: remove all thread-safety. - -.PHONY: all lib clean tests test descriptorgen amalgamate -.PHONY: clean_leave_profile genfiles - -# Prevents the deletion of intermediate files. -.SECONDARY: - -UPB_MODULES = upb upb.pb upb.json upb.descriptor -UPB_LIBS = $(patsubst %,lib/lib%.a,$(UPB_MODULES)) -UPB_PICLIBS = $(patsubst %,lib/lib%_pic.a,$(UPB_MODULES)) - -# Default rule: build all upb libraries (but no bindings) -default: $(UPB_LIBS) - -# All: build absolutely everything -all: lib tests tools/upbc lua python -testall: test pythontest - -# Set this to have user-specific flags (especially things like -O0 and -g). -USER_CPPFLAGS?= - -# Build with "make WITH_JIT=yes" (or anything besides "no") to enable the JIT. -WITH_JIT=no - -# Build with "make UPB_FAIL_WARNINGS=yes" (or anything besides "no") to turn -# warnings into errors. -UPB_FAIL_WARNINGS?=no - -# Basic compiler/flag setup. -# We are C89/C++98, with the one exception that we need stdint and "long long." -CC?=cc -CXX?=c++ -AR?=ar -CFLAGS= -CXXFLAGS= -INCLUDE=-I. -CSTD=-std=c89 -pedantic -Wno-long-long -CXXSTD=-std=c++98 -pedantic -Wno-long-long -WARNFLAGS=-Wall -Wextra -Wpointer-arith -WARNFLAGS_CXX=$(WARNFLAGS) -Wno-unused-private-field -CPPFLAGS=$(INCLUDE) -DNDEBUG $(USER_CPPFLAGS) -LUA=lua # 5.1 and 5.2 should both be supported - -ifneq ($(WITH_JIT), no) - USE_JIT=true - CPPFLAGS += -DUPB_USE_JIT_X64 - EXTRA_LIBS += -ldl -endif - -ifeq ($(CC), clang) - WARNFLAGS += -Wconditional-uninitialized -endif - -ifeq ($(CXX), clang++) - WARNFLAGS_CXX += -Wconditional-uninitialized -endif - -ifneq ($(UPB_FAIL_WARNINGS), no) - WARNFLAGS += -Werror -Wno-keyword-macro - WARNFLAGS_CXX += -Werror -Wno-keyword-macro -endif - -# Build with "make Q=" to see all commands that are being executed. -Q?=@ - -# Function to expand a wildcard pattern recursively. -rwildcard=$(strip $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2)$(filter $(subst *,%,$2),$d))) - -ifeq ($(Q), @) - E=@echo -else - E=@: -endif - -install: - test -f upb/bindings/ruby/upb.so && cd upb/bindings/ruby && make install - -# Dependency generating. ####################################################### - --include deps -# Unfortuantely we can't easily generate deps for benchmarks or tests because -# of the scheme we use that compiles the same source file multiple times with -# different -D options, which can include different header files. -dep: - $(E) Regenerating dependencies for upb/... - @set -e - @rm -f deps - @for file in $$(find . -name '*.c'); do \ - gcc -MM $$file -MT $${file%.*}.o $(CPPFLAGS) -I. >> deps 2> /dev/null; \ - done - - -clean_leave_profile: - @rm -rf obj lib - @rm -f tests/google_message?.h - @rm -f tests/json/test.upbdefs.o - @rm -f $(TESTS) tests/testmain.o tests/t.* tests/conformance_upb - @rm -rf tools/upbc deps - @rm -rf upb/bindings/python/build - @rm -f upb/bindings/ruby/Makefile - @rm -f upb/bindings/ruby/upb.o - @rm -f upb/bindings/ruby/upb.so - @rm -f upb/bindings/ruby/mkmf.log - @rm -f tests/google_messages.pb.* - @rm -f tests/google_messages.proto.pb - @rm -f upb.c upb.h - @find . | grep dSYM | xargs rm -rf - -clean: clean_leave_profile clean_lua - @rm -rf $(call rwildcard,,*.gcno) $(call rwildcard,,*.gcda) - -# A little bit of Make voodoo: you can call this from the deps of a patterned -# rule like so: -# -# all: lib/libfoo.bar.a -# -# foo_bar_SRCS = a.c b.c -# -# # This will expand into a.o b.o -# lib/lib%.a: $(call make_objs,o) -# gcc -c -o $@ $^ -# -# SECONDEXPANSION: flips on a bit essentially that allows this "secondary -# expansion": it must appear before anything that uses make_objs. -.SECONDEXPANSION: -to_srcs = $(subst .,_,$(1)_SRCS) -pc = % -make_objs = $$(patsubst $$(pc).c,obj/$$(pc).$(1),$$($$(call to_srcs,$$*))) -make_objs_cc = $$(patsubst $$(pc).cc,obj/$$(pc).$(1),$$($$(call to_srcs,$$*))) - - -# Core libraries (ie. not bindings). ############################################################### - -upb_SRCS = \ - google/protobuf/descriptor.upb.c \ - upb/decode.c \ - upb/def.c \ - upb/encode.c \ - upb/handlers.c \ - upb/msg.c \ - upb/msgfactory.c \ - upb/refcounted.c \ - upb/sink.c \ - upb/table.c \ - upb/upb.c \ - -upb_descriptor_SRCS = \ - upb/descriptor/descriptor.upbdefs.c \ - upb/descriptor/reader.c \ - -upb_pb_SRCS = \ - upb/pb/compile_decoder.c \ - upb/pb/decoder.c \ - upb/pb/encoder.c \ - upb/pb/glue.c \ - upb/pb/textprinter.c \ - upb/pb/varint.c \ - -# If the JIT is enabled we include its source. -# If Lua is present we can use DynASM to regenerate the .h file. -ifdef USE_JIT -upb_pb_SRCS += upb/pb/compile_decoder_x64.c - -# The JIT can't compile with -Wpedantic, since it does some inherently -# platform-specific things like casting between data pointers and function -# pointers. Also DynASM emits some GNU extensions. -obj/upb/pb/compile_decoder_x64.o : CSTD = -std=gnu89 -obj/upb/pb/compile_decoder_x64.lo : CSTD = -std=gnu89 - -upb/pb/compile_decoder_x64.h: upb/pb/compile_decoder_x64.dasc - $(E) DYNASM $< - $(Q) $(LUA) third_party/dynasm/dynasm.lua -c upb/pb/compile_decoder_x64.dasc > upb/pb/compile_decoder_x64.h || (rm upb/pb/compile_decoder_x64.h ; false) -endif - -upb_json_SRCS = \ - upb/json/parser.c \ - upb/json/printer.c \ - upb/pb/encoder.c \ - upb/pb/varint.c \ - upb/sink.c \ - -# Ideally we could keep this uncommented, but Git apparently sometimes skews -# timestamps slightly at "clone" time, which makes "Make" think that it needs -# to rebuild upb/json/parser.c when it actually doesn't. This would be harmless -# except that the user might not have Ragel installed. -# -# So instead we require an excplicit "make ragel" to rebuild this (for now). -# More pain for people developing upb/json/parser.rl, but less pain for everyone -# else. -# -# upb/json/parser.c: upb/json/parser.rl -# $(E) RAGEL $< -# $(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl -ragel: - $(E) RAGEL upb/json/parser.rl - $(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl - -# If the user doesn't specify an -O setting, we use -O3 for critical-path -# code and -Os for the rest. -ifeq (, $(findstring -O, $(USER_CPPFLAGS))) -OPT = -O3 -lib/libupb.a : OPT = -Os -lib/libupb.descriptor.a : OPT = -Os -obj/upb/pb/compile_decoder.o : OPT = -Os -obj/upb/pb/compile_decoder_64.o : OPT = -Os - -ifdef USE_JIT -obj/upb/pb/compile_decoder_x64.o: OPT=-Os -endif - -endif - -$(UPB_PICLIBS): lib/lib%_pic.a: $(call make_objs,lo) - $(E) AR $@ - $(Q) mkdir -p lib && $(AR) rcs $@ $^ - -$(UPB_LIBS): lib/lib%.a: $(call make_objs,o) - $(E) AR $@ - $(Q) mkdir -p lib && $(AR) rcs $@ $^ - - -obj/%.o: %.c | $$(@D)/. - $(E) CC $< - $(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $< - -obj/%.o: %.cc | $$(@D)/. - $(E) CXX $< - $(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< - -obj/%.lo: %.c | $$(@D)/. - $(E) 'CC -fPIC' $< - $(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $< -fPIC - -obj/%.lo: %.cc | $$(@D)/. - $(E) CXX -fPIC $< - $(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< -fPIC - -# Note: mkdir -p is technically susceptible to races when used with make -j. -%/.: - $(Q) mkdir -p $@ - -# Regenerating the auto-generated files in upb/. -upb/descriptor/descriptor.pb: upb/descriptor/descriptor.proto - $(E) PROTOC upb/descriptor/descriptor.proto - $(Q) protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb - -# "genfiles" includes Proto schemas we need for tests -# For the moment we check in the *.upbdefs.* generated files so that people -# can build and run the tests without Lua as a build dependency. - -genfiles: tools/upbc - @# TODO: replace protoc with upbc when upb can parse .proto files - $(E) PROTOC upb/descriptor/descriptor.proto - $(Q) protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb - $(E) PROTOC google/protobuf/descriptor.proto - $(Q) protoc google/protobuf/descriptor.proto -ogoogle/protobuf/descriptor.pb - $(E) UPBC upb/descriptor/descriptor.pb - $(Q) ./tools/upbc --generate-upbdefs upb/descriptor/descriptor.pb - $(E) UPBC google/protobuf/descriptor.pb - $(Q) ./tools/upbc google/protobuf/descriptor.pb - $(E) PROTOC tests/json/test.proto - $(Q) protoc tests/json/test.proto -otests/json/test.proto.pb - $(E) UPBC tests/json/test.proto.pb - $(Q) ./tools/upbc --generate-upbdefs tests/json/test.proto.pb - $(E) DYNASM upb/pb/compile_decoder_x64.dasc - $(Q) $(LUA) third_party/dynasm/dynasm.lua -c upb/pb/compile_decoder_x64.dasc > upb/pb/compile_decoder_x64.h || (rm upb/pb/compile_decoder_x64.h ; false) - -# upbc depends on these Lua extensions. -UPBC_LUA_EXTS = \ - upb/bindings/lua/upb_c.so \ - upb/bindings/lua/upb/pb_c.so \ - upb/bindings/lua/upb/table_c.so \ - -tools/upbc: $(UPBC_LUA_EXTS) Makefile - $(E) ECHO tools/upbc - $(Q) echo "#!/bin/sh" > tools/upbc - $(Q) echo 'BASE=`dirname "$$0"`' >> tools/upbc - $(Q) echo 'export LUA_CPATH="$$BASE/../upb/bindings/lua/?.so"' >> tools/upbc - $(Q) echo 'export LUA_PATH="$$BASE/?.lua;$$BASE/../upb/bindings/lua/?.lua"' >> tools/upbc - $(Q) echo 'lua $$BASE/upbc.lua "$$@"' >> tools/upbc - $(Q) chmod a+x tools/upbc - -examples/msg: examples/msg.c $(LIBUPB) - $(E) CC $< - $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ $< $(LIBUPB) - -# Tests. ####################################################################### - -# This section contains only the tests that don't depend on any external -# libraries. - -C_TESTS = \ - tests/pb/test_varint \ - tests/test_def \ - tests/test_handlers \ - -CC_TESTS = \ - tests/pb/test_decoder \ - tests/pb/test_encoder \ - tests/json/test_json \ - tests/test_cpp \ - tests/test_table \ - -TESTS=$(C_TESTS) $(CC_TESTS) -tests: $(TESTS) - -tests/json/test.upbdefs.o: tests/json/test.upbdefs.c - $(E) CC $< - $(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $< - -tests/testmain.o: tests/testmain.cc - $(E) CXX $< - $(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $< - -$(C_TESTS): % : %.c tests/testmain.o $$(LIBS) - $(E) CC $< - $(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -o $@ tests/testmain.o $< $(LIBS) - -$(CC_TESTS): % : %.cc tests/testmain.o $$(LIBS) - $(E) CXX $< - $(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CPPFLAGS) $(CXXFLAGS) -Wno-deprecated -o $@ tests/testmain.o $< $(LIBS) - -# Several of these tests don't actually test these libs, but use them -# incidentally to load a descriptor -LOAD_DESCRIPTOR_LIBS = lib/libupb.pb.a lib/libupb.descriptor.a - -# Specify which libs each test depends on. -tests/pb/test_varint: LIBS = lib/libupb.pb.a lib/libupb.a $(EXTRA_LIBS) -tests/test_def: LIBS = $(LOAD_DESCRIPTOR_LIBS) lib/libupb.a $(EXTRA_LIBS) -tests/test_handlers: LIBS = lib/libupb.descriptor.a lib/libupb.a $(EXTRA_LIBS) -tests/pb/test_decoder: LIBS = lib/libupb.pb.a lib/libupb.a $(EXTRA_LIBS) -tests/pb/test_encoder: LIBS = lib/libupb.pb.a lib/libupb.descriptor.a lib/libupb.a $(EXTRA_LIBS) -tests/test_cpp: LIBS = $(LOAD_DESCRIPTOR_LIBS) lib/libupb.a $(EXTRA_LIBS) -tests/test_table: LIBS = lib/libupb.a $(EXTRA_LIBS) -tests/json/test_json: LIBS = lib/libupb.a lib/libupb.json.a tests/json/test.upbdefs.o $(EXTRA_LIBS) - -tests/test.proto.pb: tests/test.proto - @# TODO: add .proto file parser to upb so this isn't necessary. - protoc tests/test.proto -otests/test.proto.pb - -VARIADIC_TESTS= \ - tests/t.test_vs_proto2.googlemessage1 \ - tests/t.test_vs_proto2.googlemessage2 \ - -ifeq ($(RUN_UNDER), valgrind) -RUN_UNDER=valgrind --leak-check=full --error-exitcode=1 --track-origins=yes -endif - -test: - @set -e # Abort on error. - @find tests -perm -u+x -type f | while read test; do \ - if [ -x ./$$test ] ; then \ - echo "RUN $$test"; \ - $(RUN_UNDER) ./$$test tests/test.proto.pb || exit 1; \ - fi \ - done; - @echo "All tests passed!" - -obj/conformance_protos: obj/conformance_protos.pb tools/upbc - cd obj && ../tools/upbc conformance_protos.pb && touch conformance_protos - -obj/conformance_protos.pb: third_party/protobuf/autogen.sh - protoc -Ithird_party/protobuf/conformance -Ithird_party/protobuf/src --include_imports \ - third_party/protobuf/conformance/conformance.proto \ - third_party/protobuf/src/google/protobuf/test_messages_proto3.proto \ - -o obj/conformance_protos.pb - -third_party/protouf/autogen.sh: .gitmodules - git submodule init && git submodule update - -tests/conformance_upb: tests/conformance_upb.c lib/libupb.a obj/conformance_protos - $(CC) -o tests/conformance_upb tests/conformance_upb.c -Iobj -I. $(CPPFLAGS) $(CFLAGS) obj/conformance.upb.c obj/google/protobuf/*.upb.c lib/libupb.a - - -# Lua extension ################################################################## - -ifeq ($(shell uname), Darwin) - LUA_LDFLAGS = -undefined dynamic_lookup -flat_namespace -else - LUA_LDFLAGS = -endif - -LUAEXTS = \ - upb/bindings/lua/upb_c.so \ - upb/bindings/lua/upb/pb_c.so \ - -LUATESTS = \ - tests/bindings/lua/test_upb.lua \ - tests/bindings/lua/test_upb.pb.lua \ - -.PHONY: clean_lua testlua lua - -testlua: lua - @set -e; \ - for test in $(LUATESTS) ; do \ - echo LUA $$test; \ - LUA_PATH="third_party/lunit/?.lua;upb/bindings/lua/?.lua" \ - LUA_CPATH=upb/bindings/lua/?.so \ - $(RUN_UNDER) lua $$test; \ - done - -clean: clean_lua -clean_lua: - @rm -f upb/bindings/lua/upb_c.so - @rm -f upb/bindings/lua/upb/pb_c.so - @rm -f upb/bindings/lua/upb/table_c.so - -lua: $(LUAEXTS) - -# Right now the core upb module depends on all of these. -# It's a TODO to factor this more cleanly in the code. -LUA_LIB_DEPS = \ - lib/libupb.pb_pic.a \ - lib/libupb.descriptor_pic.a \ - lib/libupb_pic.a \ - -upb/bindings/lua/upb_c.so: upb/bindings/lua/upb.c upb/bindings/lua/def.c upb/bindings/lua/msg.c $(LUA_LIB_DEPS) - $(E) 'CC upb/bindings/lua/{upb,def,msg}.c' - $(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $^ $(LUA_LDFLAGS) - -upb/bindings/lua/upb/table_c.so: upb/bindings/lua/upb/table.c lib/libupb_pic.a - $(E) CC upb/bindings/lua/upb/table.c - $(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $^ $(LUA_LDFLAGS) - -upb/bindings/lua/upb/pb_c.so: upb/bindings/lua/upb/pb.c $(LUA_LIB_DEPS) - $(E) CC upb/bindings/lua/upb/pb.c - $(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $^ $(LUA_LDFLAGS) - -# Amalgamated source (upb.c/upb.h) ############################################ - -AMALGAMATE_SRCS=$(upb_SRCS) $(upb_descriptor_SRCS) $(upb_pb_SRCS) $(upb_json_SRCS) - -amalgamate: upb.c upb.h - -upb.c upb.h: $(AMALGAMATE_SRCS) - $(E) AMALGAMATE $@ - $(Q) ./tools/amalgamate.py "" "" $^ - -amalgamated: upb.c upb.h - $(E) CC upb.c - $(Q) $(CC) -o upb.o -c upb.c $(WARNFLAGS) diff --git a/kokoro/ubuntu/build.sh b/kokoro/ubuntu/build.sh index bbf8841..70a8159 100644 --- a/kokoro/ubuntu/build.sh +++ b/kokoro/ubuntu/build.sh @@ -1,4 +1,11 @@ #!/bin/bash +# Install the latest version of Bazel. +use_bazel.sh latest + +# Log the bazel path and version. +which bazel +bazel version + cd $(dirname $0)/../.. bazel test :all diff --git a/tools/make_c_api.lua b/tools/make_c_api.lua deleted file mode 100644 index 9da4c1f..0000000 --- a/tools/make_c_api.lua +++ /dev/null @@ -1,575 +0,0 @@ ---[[ - - Code to generate a C API in: - foo.proto -> foo.upb.h - foo.upb.c - - This code is evolving very quickly and so there are lots of little things - that aren't perfect right now. As it settles a little more, the code - quality should improve. - ---]] - -local upb = require "upb" -local dump_cinit = require "dump_cinit" -local export = {} - -local typemap = { - [upb.TYPE_BOOL] = "bool", - [upb.TYPE_FLOAT] = "float", - [upb.TYPE_INT32] = "int32_t", - [upb.TYPE_UINT32] = "uint32_t", - [upb.TYPE_DOUBLE] = "double", - [upb.TYPE_INT64] = "int64_t", - [upb.TYPE_UINT64] = "uint64_t", - [upb.TYPE_STRING] = "upb_stringview", - [upb.TYPE_BYTES] = "upb_stringview", -} - -function strip_proto(filename) - return string.gsub(filename, '%.proto$','') -end - -local function join(...) - return table.concat({...}, ".") -end - -local function to_cident(...) - return string.gsub(join(...), "[%./]", "_") -end - -local function to_preproc(...) - return string.upper(to_cident(...)) -end - --- Strips away last path element, ie: --- foo.Bar.Baz -> foo.Bar -local function remove_name(name) - local package_end = 0 - for i=1,string.len(name) do - if string.byte(name, i) == string.byte(".", 1) then - package_end = i - 1 - end - end - return string.sub(name, 1, package_end) -end - -local function enum_value_symbol(enumdef, name) - return to_cident(remove_name(enumdef:full_name())) .. "_" .. name -end - -local function dump_enum_vals(enumdef, append) - local enum_vals = {} - - for k, v in enumdef:values() do - enum_vals[#enum_vals + 1] = {k, v} - end - - table.sort(enum_vals, function(a, b) return a[2] < b[2] end) - - -- protobuf convention is that enum values are scoped at the level of the - -- enum itself, to follow C++. Ie, if you have the enum: - -- message Foo { - -- enum E { - -- VAL1 = 1; - -- VAL2 = 2; - -- } - -- } - -- - -- The name of VAL1 is Foo.VAL1, not Foo.E.VAL1. - -- - -- This seems a bit sketchy, but people often name their enum values - -- accordingly, ie: - -- - -- enum Foo { - -- FOO_VAL1 = 1; - -- FOO_VAL2 = 2; - -- } - -- - -- So if we don't respect this also, we end up with constants that look like: - -- - -- GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_TYPE_TYPE_DOUBLE = 1 - -- - -- (notice the duplicated "TYPE"). - local cident = to_cident(remove_name(enumdef:full_name())) - for i, pair in ipairs(enum_vals) do - k, v = pair[1], pair[2] - append(' %s = %d', enum_value_symbol(enumdef, k), v) - if i == #enum_vals then - append('\n') - else - append(',\n') - end - end -end - -local function field_default(field) - if field:type() == upb.TYPE_MESSAGE then - return "NULL" - elseif field:type() == upb.TYPE_STRING or - field:type() == upb.TYPE_BYTES then - local default = field:default() or "" - return string.format('upb_stringview_make("%s", strlen("%s"))', field:default(), field:default()) - elseif field:type() == upb.TYPE_ENUM then - return enum_value_symbol(field:subdef(), field:default()) - else - return field:default(); - end -end - -local function ctype(field, const) - if const then - const = "const " - else - const = "" - end - - if field:label() == upb.LABEL_REPEATED then - return const .. "upb_array*" - elseif field:type() == upb.TYPE_MESSAGE then - if field:containing_type():file() == field:subdef():file() then - return const .. to_cident(field:subdef():full_name()) .. "*" - else - return const .. "struct " .. to_cident(field:subdef():full_name()) .. "*" - end - elseif field:type() == upb.TYPE_ENUM then - return to_cident(field:subdef():full_name()) - else - return typemap[field:type()] or "void*" - end -end - -local function emit_file_warning(filedef, append) - append('/* This file was generated by upbc (the upb compiler) from the input\n') - append(' * file:\n') - append(' *\n') - append(' * %s\n', filedef:name()) - append(' *\n') - append(' * Do not edit -- your changes will be discarded when the file is\n') - append(' * regenerated. */\n\n') -end - -local function field_layout_rank(field) - -- Order: - -- 1, 2, 3. primitive fields (8, 4, 1 byte) - -- 4. string fields - -- 5. submessage fields - -- 6. repeated fields - -- - -- This has the following nice properties: - -- - -- 1. padding alignment is (nearly) minimized. - -- 2. fields that might have defaults (1-4) are segregated - -- from fields that are always zero-initialized (5-7). - -- - -- We skip oneof fields, because they are emitted in a separate pass. - local rank - if field:containing_oneof() then - rank = 100 -- These go last (actually we skip them). - elseif field:label() == upb.LABEL_REPEATED then - rank = 6 - elseif field:type() == upb.TYPE_MESSAGE then - rank = 5 - elseif field:type() == upb.TYPE_STRING or field:type() == upb.TYPE_BYTES then - rank = 4 - elseif field:type() == upb.TYPE_BOOL then - rank = 3 - elseif field:type() == upb.TYPE_FLOAT or - field:type() == upb.TYPE_INT32 or - field:type() == upb.TYPE_UINT32 then - rank = 2 - else - rank = 1 - end - - -- Break ties with field number. - return (rank * 2^29) + field:number() -end - -local function sizeof(field) - if field:label() == upb.LABEL_REPEATED or - field:type() == upb.TYPE_MESSAGE then - return {4, 8} - elseif field:type() == upb.TYPE_STRING or field:type() == upb.TYPE_BYTES then - -- upb_stringview - return {8, 16} - elseif field:type() == upb.TYPE_BOOL then - return {1, 1} - elseif field:type() == upb.TYPE_FLOAT or - field:type() == upb.TYPE_INT32 or - field:type() == upb.TYPE_UINT32 then - return {4, 4} - else - return {8, 8} - end -end - -local function sizemax(size, max) - max[1] = math.max(max[1], size[1]) - max[2] = math.max(max[2], size[2]) -end - -local function alignup(val, align) - val[1] = math.ceil(val[1] / align[1]) * align[1] - val[2] = math.ceil(val[2] / align[2]) * align[2] -end - -local function copysize(size) - return {size[1], size[2]} -end - -local function place(offset, size, max) - alignup(offset, size) - local ret = copysize(offset) - - -- add size - offset[1] = offset[1] + size[1] - offset[2] = offset[2] + size[2] - - -- track max size - sizemax(size, max) - - return ret -end - -local function get_field_layout_order(msg) - local field_order = {} - - -- Sort fields by rank. - for field in msg:fields() do - table.insert(field_order, field) - end - table.sort(field_order, function(a, b) - return field_layout_rank(a) < field_layout_rank(b) - end) - - return field_order -end - -local function get_oneof_layout_order(msg) - local oneof_order = {} - - -- Sort oneofs by name. - for oneof in msg:oneofs() do - table.insert(oneof_order, oneof) - end - table.sort(oneof_order, function(a, b) - return a:name() < b:name() - end) - - return oneof_order -end - -local function has_hasbit(field) - if field:containing_type():file():syntax() == upb.SYNTAX_PROTO2 then - return field:label() ~= upb.LABEL_REPEATED and not field:containing_oneof() - else - return false - end -end - -local function get_message_layout(msg) - local hasbit_count = 0 - local hasbit_indexes = {} - local field_order = get_field_layout_order(msg) - local maxsize = {0, 0} - - -- Count hasbits. - for _, field in ipairs(field_order) do - if has_hasbit(field) then - hasbit_indexes[field] = hasbit_count - hasbit_count = hasbit_count + 1 - end - end - - -- Place hasbits at the beginning. - local offset = math.ceil(hasbit_count / 8) - offset = {offset, offset} -- 32, 64 bit - local offsets = {} - - -- Place non-oneof fields. - for _, field in ipairs(field_order) do - if not field:containing_oneof() then - offsets[field] = place(offset, sizeof(field), maxsize) - end - end - - -- Place oneof fields. - for oneof in msg:oneofs() do - local oneof_maxsize = {0, 0} - -- Calculate max size. - for field in oneof:fields() do - local size = sizeof(field) - sizemax(size, oneof_maxsize) - end - - -- Place discriminator enum and data. - local data = place(offset, oneof_maxsize, maxsize) - local case = place(offset, {4, 4}, maxsize) - offsets[oneof] = case - - -- Place oneof fields. - for oneof in msg:oneofs() do - for field in oneof:fields() do - offsets[field] = data - end - end - end - - -- Align overall size up to max size. - alignup(offset, maxsize) - local size = copysize(offset) - - return hasbit_indexes, offsets, size -end - -function get_sizeinit(size) - return string.format("UPB_SIZE(%s, %s)", size[1], size[2]) -end - -local function write_h_file(filedef, append) - emit_file_warning(filedef, append) - local basename_preproc = to_preproc(filedef:name()) - append('#ifndef %s_UPB_H_\n', basename_preproc) - append('#define %s_UPB_H_\n\n', basename_preproc) - - append('#include <string.h>\n\n') - append('#include "upb/msg.h"\n\n') - append('#include "upb/decode.h"\n') - append('#include "upb/encode.h"\n') - append('#include "upb/port_def.inc"\n') - - append('UPB_BEGIN_EXTERN_C\n\n') - - -- Forward-declare types defined in this file. - for msg in filedef:defs(upb.DEF_MSG) do - local msgname = to_cident(msg:full_name()) - append('struct %s;\n', msgname) - end - - for msg in filedef:defs(upb.DEF_MSG) do - local msgname = to_cident(msg:full_name()) - append('typedef struct %s %s;\n', msgname, msgname) - end - - -- Forward-declare types not in this file, but used as submessages. - for msg in filedef:defs(upb.DEF_MSG) do - for field in msg:fields() do - if field:type() == upb.TYPE_MESSAGE and - field:subdef():file() ~= filedef then - -- Forward declaration for message type declared in another file. - append('struct %s;\n', to_cident(field:subdef():full_name())) - end - end - end - - append('\n') - append("/* Enums */\n\n") - - for _, def in ipairs(sorted_defs(filedef:defs(upb.DEF_ENUM))) do - local cident = to_cident(def:full_name()) - append('typedef enum {\n') - dump_enum_vals(def, append) - append('} %s;\n\n', cident) - end - - for msg in filedef:defs(upb.DEF_MSG) do - local hasbit_indexes, offsets, size = get_message_layout(msg) - - append("/* %s */\n\n", msg:full_name()) - - local msgname = to_cident(msg:full_name()) - append('extern const upb_msglayout %s_msginit;\n', msgname) - append('UPB_INLINE %s *%s_new(upb_arena *arena) {\n', msgname, msgname) - append(' return (%s *)upb_msg_new(&%s_msginit, arena);\n', msgname, msgname) - append('}\n') - append('UPB_INLINE %s *%s_parsenew(upb_stringview buf, upb_arena *arena) {\n', - msgname, msgname) - append(' %s *ret = %s_new(arena);\n', msgname, msgname) - append(' return (ret && upb_decode(buf, ret, &%s_msginit)) ? ret : NULL;\n', msgname) - append('}\n') - append('UPB_INLINE char *%s_serialize(const %s *msg, upb_arena *arena, size_t *len) {\n', - msgname, msgname) - append(' return upb_encode(msg, &%s_msginit, arena, len);\n', msgname) - append('}\n') - append('\n') - - for oneof in msg:oneofs() do - local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name()) - local offset = offsets[oneof] - append('typedef enum {\n') - for field in oneof:fields() do - append(' %s = %d,\n', fullname .. "_" .. field:name(), field:number()) - end - append(' %s_NOT_SET = 0,\n', fullname) - append('} %s_oneofcases;\n', fullname) - append('UPB_INLINE %s_oneofcases %s_%s_case(const %s* msg) { ' .. - 'return UPB_FIELD_AT(msg, int, %s); }\n', - fullname, msgname, oneof:name(), msgname, get_sizeinit(offset)) - append('\n') - end - - for field in msg:fields() do - append('UPB_INLINE %s %s_%s(const %s *msg) {', - ctype(field, true), msgname, field:name(), msgname) - if field:containing_oneof() then - local data_offset = offsets[field] - local case_offset = offsets[field:containing_oneof()] - append(' return UPB_READ_ONEOF(msg, %s, %s, %s, %s, %s); }\n', - ctype(field, true), get_sizeinit(data_offset), - get_sizeinit(case_offset), field:number(), field_default(field)) - else - append(' return UPB_FIELD_AT(msg, %s, %s); }\n', - ctype(field, true), get_sizeinit(offsets[field])) - end - end - - append('\n') - - for field in msg:fields() do - append('UPB_INLINE void %s_set_%s(%s *msg, %s value) { ', - msgname, field:name(), msgname, ctype(field)) - if field:containing_oneof() then - local data_offset = offsets[field] - local case_offset = offsets[field:containing_oneof()] - append('UPB_WRITE_ONEOF(msg, %s, %s, value, %s, %s); }\n', - ctype(field), get_sizeinit(data_offset), get_sizeinit(case_offset), - field:number()) - else - append('UPB_FIELD_AT(msg, %s, %s) = value; }\n', - ctype(field), get_sizeinit(offsets[field])) - end - end - - append('\n\n') - end - - append('UPB_END_EXTERN_C\n') - - append('\n') - append('#include "upb/port_undef.inc"\n'); - append('\n') - - append('#endif /* %s_UPB_H_ */\n', basename_preproc) -end - -local function write_c_file(filedef, hfilename, append) - emit_file_warning(filedef, append) - - append('#include <stddef.h>\n') - append('#include "upb/msg.h"\n') - append('#include "%s"\n', hfilename) - - for dep in filedef:dependencies() do - local outbase = strip_proto(dep:name()) - append('#include "%s.upb.h"\n', outbase) - end - - append('\n') - append('#include "upb/port_def.inc"\n') - append('\n') - - for msg in filedef:defs(upb.DEF_MSG) do - local msgname = to_cident(msg:full_name()) - - local fields_array_ref = "NULL" - local submsgs_array_ref = "NULL" - local oneofs_array_ref = "NULL" - local field_count = 0 - local submsg_count = 0 - local submsg_set = {} - local submsg_indexes = {} - local hasbit_indexes, offsets, size = get_message_layout(msg) - local oneofs_layout_order = get_oneof_layout_order(msg) - local oneof_count = 0 - - -- Another sorted array in field number order. - local fields_number_order = {} - - for field in msg:fields() do - field_count = field_count + 1 - table.insert(fields_number_order, field) - if field:type() == upb.TYPE_MESSAGE then - submsg_count = submsg_count + 1 - submsg_set[field:subdef()] = true - end - end - - table.sort(fields_number_order, function(a, b) - return a:number() < b:number() - end) - - if submsg_count > 0 then - -- TODO(haberman): could save a little bit of space by only generating a - -- "submsgs" array for every strongly-connected component. - local submsgs_array_name = msgname .. "_submsgs" - submsgs_array_ref = "&" .. submsgs_array_name .. "[0]" - append('static const upb_msglayout *const %s[%s] = {\n', - submsgs_array_name, submsg_count) - - -- Create a deterministically-sorted array of submessage entries. - local submsg_array = {} - for k, v in pairs(submsg_set) do - table.insert(submsg_array, k) - end - table.sort(submsg_array, function(a, b) - return a:full_name() < b:full_name() - end) - - for i, submsg in ipairs(submsg_array) do - append(' &%s_msginit,\n', to_cident(submsg:full_name())) - submsg_indexes[submsg] = i - 1 - end - - append('};\n\n') - end - - if field_count > 0 then - local fields_array_name = msgname .. "__fields" - fields_array_ref = "&" .. fields_array_name .. "[0]" - append('static const upb_msglayout_field %s[%s] = {\n', - fields_array_name, field_count) - for _, field in ipairs(fields_number_order) do - local submsg_index = "0" - if field:type() == upb.TYPE_MESSAGE then - submsg_index = submsg_indexes[field:subdef()] - end - local presence = 0 - if has_hasbit(field) then - presence = hasbit_indexes[field] + 1 - elseif field:containing_oneof() then - local case_ofs = offsets[field:containing_oneof()] - presence = get_sizeinit({(-case_ofs[1]) - 1, (-case_ofs[2]) - 1}) - end - append(' {%s, %s, %s, %s, %s, %s},\n', - field:number(), - get_sizeinit(offsets[field]), - presence, - submsg_index, - field:descriptor_type(), - field:label()) - end - append('};\n\n') - end - - append('const upb_msglayout %s_msginit = {\n', msgname) - append(' %s,\n', submsgs_array_ref) - append(' %s,\n', fields_array_ref) - append(' %s, %s, %s,\n', - get_sizeinit(size), field_count, - 'false' -- TODO: extendable - ) - - append('};\n\n') - end - - append('#include "upb/port_undef.inc"\n') - append('\n') -end - -function export.write_gencode(filedef, hfilename, append_h, append_c) - write_h_file(filedef, append_h) - write_c_file(filedef, hfilename, append_c) -end - -return export diff --git a/tools/upbc.lua b/tools/upbc.lua index adea373..80d2886 100644 --- a/tools/upbc.lua +++ b/tools/upbc.lua @@ -9,7 +9,6 @@ --]] local dump_cinit = require "dump_cinit" -local make_c_api = require "make_c_api" local upb = require "upb" local generate_upbdefs = false @@ -77,40 +76,16 @@ for _, file in ipairs(files) do os.execute(string.format("mkdir -p `dirname %s`", outbase)) - if generate_upbdefs then - -- Legacy generated defs. - local hfile = open(hfilename) - local cfile = open(cfilename) + assert(generate_upbdefs) + -- Legacy generated defs. + local hfile = open(hfilename) + local cfile = open(cfilename) - local happend = dump_cinit.file_appender(hfile) - local cappend = dump_cinit.file_appender(cfile) + local happend = dump_cinit.file_appender(hfile) + local cappend = dump_cinit.file_appender(cfile) - dump_cinit.dump_defs(file, happend, cappend) + dump_cinit.dump_defs(file, happend, cappend) - hfile:close() - cfile:close() - else - -- Write C API. - hfilename = outbase .. ".upb.h" - cfilename = outbase .. ".upb.c" - - if os.getenv("UPBC_VERBOSE") then - print("upbc:") - print(string.format(" source file=%s", src)) - print(string.format(" output file base=%s", outbase)) - print(string.format(" hfilename=%s", hfilename)) - print(string.format(" cfilename=%s", cfilename)) - end - - local hfile = open(hfilename) - local cfile = open(cfilename) - - local happend = dump_cinit.file_appender(hfile) - local cappend = dump_cinit.file_appender(cfile) - - make_c_api.write_gencode(file, hfilename, happend, cappend) - - hfile:close() - cfile:close() - end + hfile:close() + cfile:close() end |