diff options
| -rw-r--r-- | .travis.yml | 2 | ||||
| -rw-r--r-- | Makefile | 49 | ||||
| -rw-r--r-- | README | 69 | ||||
| -rw-r--r-- | README.md | 148 | ||||
| -rw-r--r-- | tests/bindings/lua/test_upb.lua (renamed from tests/bindings/lua/upb.lua) | 0 | ||||
| -rw-r--r-- | tests/bindings/lua/test_upb.pb.lua (renamed from tests/bindings/lua/upb.pb.lua) | 0 | ||||
| -rwxr-xr-x | travis.sh | 26 | ||||
| -rw-r--r-- | upb/bindings/lua/upb.c | 5 | ||||
| -rw-r--r-- | upb/bindings/lua/upb/pb.c (renamed from upb/bindings/lua/upb.pb.c) | 0 |
9 files changed, 214 insertions, 85 deletions
diff --git a/.travis.yml b/.travis.yml index e366254..a65f56a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,4 +3,6 @@ script: ./travis.sh script env: - UPB_TRAVIS_BUILD=bare - UPB_TRAVIS_BUILD=barejit + - UPB_TRAVIS_BUILD=core32 - UPB_TRAVIS_BUILD=withprotobuf + - UPB_TRAVIS_BUILD=lua @@ -42,7 +42,7 @@ WITH_JIT=no CC=cc CXX=c++ CFLAGS=-std=c99 -CXXFLAGS=-Wno-unused-private-field $(USER_CXXFLAGS) +CXXFLAGS=-Wno-unused-private-field INCLUDE=-I. CPPFLAGS=$(INCLUDE) -DNDEBUG -Wall -Wextra -Wno-sign-compare $(USER_CPPFLAGS) LDLIBS=-lpthread upb/libupb.a @@ -87,7 +87,6 @@ clean_leave_profile: @rm -rf obj lib @rm -f benchmark/google_messages.proto.pb benchmark/google_messages.pb.* benchmarks/b.* benchmarks/*.pb* @rm -f $(TESTS) tests/testmain.o tests/t.* - @rm -f tests/test.proto.pb @rm -f upb/descriptor.pb @rm -rf tools/upbc deps @rm -rf upb/bindings/python/build @@ -159,7 +158,7 @@ upb_json_SRCS = \ # If the user doesn't specify an -O setting, we use -O3 for critical-path # code and -Os for the rest. -ifeq (, $(findstring -O, $(USER_CFLAGS))) +ifeq (, $(findstring -O, $(USER_CPPFLAGS))) OPT = -O3 lib/libupb.a : OPT = -Os lib/libupb.descriptor.a : OPT = -Os @@ -364,18 +363,28 @@ endif LUAEXTS = \ upb/bindings/lua/upb.so \ - upb/bindings/lua/upb.pb.so \ + upb/bindings/lua/upb/pb.so \ + +LUATESTS = \ + tests/bindings/lua/test_upb.lua \ + tests/bindings/lua/test_upb.pb.lua \ .PHONY: clean_lua testlua lua -testlua: - LUA_PATH=tests/bindings/lua/?.lua LUA_CPATH=upb/bindings/lua/?.so lua tests/bindings/lua/upb.lua +testlua: lua + @set -e # Abort on error. + @for test in $(LUATESTS) ; do \ + echo LUA $$test; \ + LUA_PATH="tests/bindings/lua/lunit/?.lua" \ + LUA_CPATH=upb/bindings/lua/?.so \ + lua $$test; \ + done clean: clean_lua clean_lua: @rm -f upb/bindings/lua/upb.lua.h @rm -f upb/bindings/lua/upb.so - @rm -f upb/bindings/lua/upb.pb.so + @rm -f upb/bindings/lua/upb/pb.so lua: $(LUAEXTS) @@ -383,13 +392,29 @@ upb/bindings/lua/upb.lua.h: $(E) XXD upb/bindings/lua/upb.lua $(Q) xxd -i < upb/bindings/lua/upb.lua > upb/bindings/lua/upb.lua.h -upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h lib/libupb.descriptor_pic.a lib/libupb_pic.a lib/libupb.pb_pic.a - $(E) CC upb/bindings/lua/upb.c - $(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< lib/libupb.pb_pic.a lib/libupb.descriptor_pic.a lib/libupb_pic.a $(LUA_LDFLAGS) +# Right now the core upb module depends on all of these. +# It's a TODO to factor this more cleanly in the code. +LUA_LIB_DEPS = \ + lib/libupb.pb_pic.a \ + lib/libupb.descriptor_pic.a \ + lib/libupb_pic.a \ -upb/bindings/lua/upb.pb.so: upb/bindings/lua/upb.pb.c lib/libupb_pic.a lib/libupb.pb_pic.a +upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h $(LUA_LIB_DEPS) + $(E) CC upb/bindings/lua/upb.c + $(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) $(LUA_LIB_DEPS) + +# TODO: the dependency between upb/pb.so and upb.so is expressed at the +# .so level, which means that the OS will try to load upb.so when upb/pb.so +# is loaded. This is what we want, but getting the paths right is tricky. +# Basically the dynamic linker needs to be able to find upb.so at: +# $(LD_LIBRARY_PATH)/upb/bindings/lua/upb.so +# So the user has to set both LD_LIBRARY_PATH and LUA_CPATH correctly. +# Another option would be to require the Lua program to always require +# "upb" before requiring eg. "upb.pb", and then the dependency would not +# be expressed at the .so level. +upb/bindings/lua/upb/pb.so: upb/bindings/lua/upb/pb.c upb/bindings/lua/upb.so $(E) CC upb/bindings/lua/upb.pb.c - $(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) + $(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< upb/bindings/lua/upb.so $(LUA_LDFLAGS) # Python extension ############################################################# @@ -1,69 +0,0 @@ - -upb - a small, low-level protocol buffer library - -BUILDING -======== - -To build (the core library is ANSI C99 and has no dependencies): -$ make - -Other useful targets: -$ make test -$ make benchmark -$ make lua (requires lua libraries to be installed) - -The tests and benchmarks have the following dependencies -(Ubuntu package names in parentheses): -- Google's protobuf compiler + libraries (protobuf-compiler, libprotobuf-dev) -- Lua binary and libraries (lua5.1, liblua5.1-dev) - -Issue tracking is on Google Code: - http://code.google.com/p/upb/issues/list - -A manual is forthcoming, for now see wiki docs at: - https://github.com/haberman/upb/wiki - -API -=== - -The public C/C++ API is defined by all of the .h files in upb/ except .int.h -files (which are internal-only). - -The .h files define both C and C++ APIs. Both languages have 100% complete and -first-class APIs. The C++ API is a wrapper around the C API, but all of the -wrapping is done in inline methods in .h files, so there is no overhead to this. - -For a more detailed description of the scheme we use to provide both C and C++ -APIs, see: https://github.com/haberman/upb/wiki/CAndCPlusPlusAPI - -All of the code that is under upb/ but *not* under upb/bindings/ forms the -namespace of upb's cross-language public API. For example, the code in -upb/descriptor would be exposed as follows: - - * in C/C++: - #include "upb/descriptor/X.h" - - * in Lua: - require "upb.descriptor" - - * in Python: - import upb.descriptor - - * etc. - -STABILITY / STATUS -================== - -API and ABI are both subject to change! Please do not distribute as a shared -library for this reason (for now at least). - -Parsing is becoming stable, but serialization is in very early stages. - -The Python bindings are completely broken and Lua only supports schemas for the -moment. - -CONTACT -======= - -Author: Josh Haberman (jhaberman@gmail.com, haberman@google.com) -See LICENSE for copyright information. diff --git a/README.md b/README.md new file mode 100644 index 0000000..953e4d3 --- /dev/null +++ b/README.md @@ -0,0 +1,148 @@ + +# Unleaded - small, fast parsers for the 21st century + +[](https://travis-ci.org/haberman/upb) + +Unleaded is a library of fast parsers and serializers. These +parsers/serializers are written in C and use every available +avenue (particularly JIT compilation) to achieve the fastest +possible speed. However they are also extremely lightweight +(less than 100k of object code) and low-overhead. + +The library started as a Protocol Buffers library (upb originally +meant μpb: Micro Protocol Buffers). It still uses +protobuf-like schemas as a core abstraction, but **it has expanded +beyond just Protocol Buffers** to JSON, and other formats are +planned. + +The library itself is written in C, but very idiomatic APIs +are provided for C++ and popular dynamic languages such as +Lua. See the rest of this README for more information about +these bindings. + +Some parts of Unleaded are mature (most notably parsing of +Protocol Buffers) but others are still immature or nonexistent. +The core library abstractions are rapidly converging (this +is saying a lot; it was a long road of about 5 years to make +this happen), which should make it possible to begin building +out the encoders and decoders in earnest. + +API and ABI are both subject to change! Please do not distribute +as a shared library for this reason (for now at least). + +## Building the core libraries + +The core libraries are pure C99 and have no dependencies. + + $ make + +This will create a separate C library for each core library +in `lib/`. They are built separately to help your binaries +slim, so you don't need to link in things you neither want +or need. + +Other useful targets: + + $ make tests + $ make test + +## How the library is organized + +Unleaded tries to stay very small, but also aims to support +lots of different formats. We reconcile these goals by +being *aggresively modular*. The source tree and the build +artifacts both reflect this organization: + +* **upb**: the core library of handlers and defs (schemas) +* **upb/pb**: encoders/decoders for Protocol Buffers +* **upb/json**: encoders/decoders for JSON +* **upb/descriptor**: building upb defs from protobuf desciptors + (ie. descriptor.proto) +* **upb/bindings/googlepb**: binding to the Google protobuf + library. +* **upb/bindings/lua**: binding to the Lua C API (Lua and LuaJIT). +* more to come! + +## C and C++ API + +The public C/C++ API is defined by all of the .h files in +`upb/` except `.int.h` files (which are internal-only). + +The `.h` files define both C and C++ APIs. Both languages +have 100% complete and first-class APIs. The C++ API is a +wrapper around the C API, but all of the wrapping is done in +inline methods in `.h` files, so there is no overhead to +this. + +For a more detailed description of the scheme we use to +provide both C and C++ APIs, see: +[https://github.com/haberman/upb/wiki/CAndCPlusPlusAPI](CAndCPlusPlusAPI). + +All of the code that is under `upb/` but *not* under +`upb/bindings/` forms the namespace of upb's cross-language +public API. For example, the code in upb/descriptor would +be exposed as follows: + + * **in C/C++:** `#include "upb/descriptor/X.h"` + * **in Lua:** `require "upb.descriptor"` + * **in Python:** `import upb.descriptor` + * etc. + +## Google protobuf bindings + +Unleaded supports integration with the +[https://github.com/google/protobuf](Google protobuf library). +These bindings let you: + +* convert protobuf schema objects (`Descriptor`, `FieldDescriptor`, etc). + to their Unleaded equivalents (`upb::MessageDef`, `upb::FieldDef`). +* use Unleaded parsers to populate protobuf generated classes. + Unleaded's parsers are much faster than protobuf's `DynamicMessage`. + If you are generating C++ with the protobuf compiler, then protobuf's + parsers are the same speed or a little faster than Unleaded in JIT + mode, but Unleaded will have smaller binaries because you don't + have to generate the code ahead of time. + +To build the Google protobuf integration you must have the protobuf +libraries already installed. Once they are installed run: + + $ make googlepb + +To test run: + + $ make googlepbtests + $ make test + +## Lua bindings + +Lua bindings provide Unleaded's functionality to Lua programs. +The bindings target Lua 5.1, Lua 5.2, LuaJIT, and (soon) Lua 5.3. + +Right now the Lua bindings support: + +* Building schema objects manually (eg. you can essentially write + .proto files natively in Lua). +* creating message objects. +* parsing Protocol Buffers into message objects. + +Other capabilities (parse/serialize JSON, serialize Protocol Buffers) +are coming. + +To build the Lua bindings, the Lua libraries must be installed. Once +they are installed, run: + + $ make lua + +Note that if the Lua headers are not in a standard place, you may +need to pass custom flags: + + $ make lua USER_CPPFLAGS=`pkg-config lua5.2 --cflags` + +To test the Lua bindings: + + $ make testlua + +## Contact + +Author: Josh Haberman ([jhaberman@gmail.com](jhaberman@gmail.com), +[haberman@google.com](haberman@google.com)) diff --git a/tests/bindings/lua/upb.lua b/tests/bindings/lua/test_upb.lua index f32a690..f32a690 100644 --- a/tests/bindings/lua/upb.lua +++ b/tests/bindings/lua/test_upb.lua diff --git a/tests/bindings/lua/upb.pb.lua b/tests/bindings/lua/test_upb.pb.lua index 02b443a..02b443a 100644 --- a/tests/bindings/lua/upb.pb.lua +++ b/tests/bindings/lua/test_upb.pb.lua @@ -5,7 +5,8 @@ bare_install() { : } bare_script() { - make -j12 tests && make test + make -j12 tests + make test } # Bare JIT build: no dependencies installed, but JIT enabled. @@ -13,7 +14,8 @@ barejit_install() { : } barejit_script() { - make -j12 tests WITH_JIT=yes && make test + make -j12 tests WITH_JIT=yes + make test } # Build with Google protobuf support and tests (with JIT). @@ -26,6 +28,26 @@ withprotobuf_script() { make test } +# A 32-bit build. Can only test the core because any dependencies +# need to be available as 32-bit libs also, which gets hairy fast. +# Can't enable the JIT because it only supports x64. +core32_install() { + : +} +core32_script() { + make -j12 tests USER_CPPFLAGS=-m32 + make test +} + +# A build of Lua and running of Lua tests. +lua_install() { + sudo apt-get update -qq + sudo apt-get install lua5.2 liblua5.2-dev +} +lua_script() { + make -j12 testlua USER_CPPFLAGS=`pkg-config lua5.2 --cflags` +} + set -e set -x eval ${UPB_TRAVIS_BUILD}_${1} diff --git a/upb/bindings/lua/upb.c b/upb/bindings/lua/upb.c index f938b26..f257430 100644 --- a/upb/bindings/lua/upb.c +++ b/upb/bindings/lua/upb.c @@ -1323,9 +1323,10 @@ static size_t lupb_sizeof(lua_State *L, const upb_fielddef *f) { case UPB_TYPE_STRING: case UPB_TYPE_BYTES: case UPB_TYPE_MESSAGE: - lupb_assert(L, false); - return 0; + break; } + lupb_assert(L, false); + return 0; } static int div_round_up(size_t n, size_t d) { diff --git a/upb/bindings/lua/upb.pb.c b/upb/bindings/lua/upb/pb.c index c9f1f47..c9f1f47 100644 --- a/upb/bindings/lua/upb.pb.c +++ b/upb/bindings/lua/upb/pb.c |