diff options
Diffstat (limited to 'upb/pb')
| -rw-r--r-- | upb/pb/compile_decoder.c | 217 | ||||
| -rw-r--r-- | upb/pb/compile_decoder_x64.c | 510 | ||||
| -rw-r--r-- | upb/pb/compile_decoder_x64.dasc | 1145 | ||||
| -rw-r--r-- | upb/pb/compile_decoder_x64.h | 1732 | ||||
| -rw-r--r-- | upb/pb/decoder.c | 57 | ||||
| -rw-r--r-- | upb/pb/decoder.h | 294 | ||||
| -rw-r--r-- | upb/pb/decoder.int.h | 61 | ||||
| -rw-r--r-- | upb/pb/encoder.c | 46 | ||||
| -rw-r--r-- | upb/pb/encoder.h | 78 | ||||
| -rw-r--r-- | upb/pb/glue.c | 54 | ||||
| -rw-r--r-- | upb/pb/glue.h | 72 | ||||
| -rw-r--r-- | upb/pb/textprinter.c | 25 | ||||
| -rw-r--r-- | upb/pb/textprinter.h | 81 | ||||
| -rw-r--r-- | upb/pb/varint.int.h | 15 |
14 files changed, 351 insertions, 4036 deletions
diff --git a/upb/pb/compile_decoder.c b/upb/pb/compile_decoder.c index cec7b2a..89f000a 100644 --- a/upb/pb/compile_decoder.c +++ b/upb/pb/compile_decoder.c @@ -4,11 +4,6 @@ ** Code to compile a upb::Handlers into bytecode for decoding a protobuf ** according to that specific schema and destination handlers. ** -** Compiling to bytecode is always the first step. If we are using the -** interpreted decoder we leave it as bytecode and interpret that. If we are -** using a JIT decoder we use a code generator to turn the bytecode into native -** code, LLVM IR, etc. -** ** Bytecode definition is in decoder.int.h. */ @@ -23,80 +18,22 @@ #define MAXLABEL 5 #define EMPTYLABEL -1 -/* mgroup *********************************************************************/ - -static void freegroup(upb_refcounted *r) { - mgroup *g = (mgroup*)r; - upb_inttable_uninit(&g->methods); -#ifdef UPB_USE_JIT_X64 - upb_pbdecoder_freejit(g); -#endif - upb_gfree(g->bytecode); - upb_gfree(g); -} - -static void visitgroup(const upb_refcounted *r, upb_refcounted_visit *visit, - void *closure) { - const mgroup *g = (const mgroup*)r; - upb_inttable_iter i; - upb_inttable_begin(&i, &g->methods); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i)); - visit(r, upb_pbdecodermethod_upcast(method), closure); - } -} - -mgroup *newgroup(const void *owner) { - mgroup *g = upb_gmalloc(sizeof(*g)); - static const struct upb_refcounted_vtbl vtbl = {visitgroup, freegroup}; - upb_refcounted_init(mgroup_upcast_mutable(g), &vtbl, owner); - upb_inttable_init(&g->methods, UPB_CTYPE_PTR); - g->bytecode = NULL; - g->bytecode_end = NULL; - return g; -} - - /* upb_pbdecodermethod ********************************************************/ -static void freemethod(upb_refcounted *r) { - upb_pbdecodermethod *method = (upb_pbdecodermethod*)r; - - if (method->dest_handlers_) { - upb_handlers_unref(method->dest_handlers_, method); - } - +static void freemethod(upb_pbdecodermethod *method) { upb_inttable_uninit(&method->dispatch); upb_gfree(method); } -static void visitmethod(const upb_refcounted *r, upb_refcounted_visit *visit, - void *closure) { - const upb_pbdecodermethod *m = (const upb_pbdecodermethod*)r; - visit(r, m->group, closure); -} - static upb_pbdecodermethod *newmethod(const upb_handlers *dest_handlers, mgroup *group) { - static const struct upb_refcounted_vtbl vtbl = {visitmethod, freemethod}; upb_pbdecodermethod *ret = upb_gmalloc(sizeof(*ret)); - upb_refcounted_init(upb_pbdecodermethod_upcast_mutable(ret), &vtbl, &ret); upb_byteshandler_init(&ret->input_handler_); - /* The method references the group and vice-versa, in a circular reference. */ - upb_ref2(ret, group); - upb_ref2(group, ret); - upb_inttable_insertptr(&group->methods, dest_handlers, upb_value_ptr(ret)); - upb_pbdecodermethod_unref(ret, &ret); - - ret->group = mgroup_upcast_mutable(group); + ret->group = group; ret->dest_handlers_ = dest_handlers; - ret->is_native_ = false; /* If we JIT, it will update this later. */ upb_inttable_init(&ret->dispatch, UPB_CTYPE_UINT64); - if (ret->dest_handlers_) { - upb_handlers_ref(ret->dest_handlers_, ret); - } return ret; } @@ -114,16 +51,28 @@ bool upb_pbdecodermethod_isnative(const upb_pbdecodermethod *m) { return m->is_native_; } -const upb_pbdecodermethod *upb_pbdecodermethod_new( - const upb_pbdecodermethodopts *opts, const void *owner) { - const upb_pbdecodermethod *ret; - upb_pbcodecache cache; - upb_pbcodecache_init(&cache); - ret = upb_pbcodecache_getdecodermethod(&cache, opts); - upb_pbdecodermethod_ref(ret, owner); - upb_pbcodecache_uninit(&cache); - return ret; +/* mgroup *********************************************************************/ + +static void freegroup(mgroup *g) { + upb_inttable_iter i; + + upb_inttable_begin(&i, &g->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + freemethod(upb_value_getptr(upb_inttable_iter_value(&i))); + } + + upb_inttable_uninit(&g->methods); + upb_gfree(g->bytecode); + upb_gfree(g); +} + +mgroup *newgroup() { + mgroup *g = upb_gmalloc(sizeof(*g)); + upb_inttable_init(&g->methods, UPB_CTYPE_PTR); + g->bytecode = NULL; + g->bytecode_end = NULL; + return g; } @@ -355,7 +304,7 @@ static void putop(compiler *c, int op, ...) { va_end(ap); } -#if defined(UPB_USE_JIT_X64) || defined(UPB_DUMP_BYTECODE) +#if defined(UPB_DUMP_BYTECODE) const char *upb_pbdecoder_getopname(unsigned int op) { #define QUOTE(x) #x @@ -558,7 +507,7 @@ static upb_pbdecodermethod *find_submethod(const compiler *c, static void putsel(compiler *c, opcode op, upb_selector_t sel, const upb_handlers *h) { - if (upb_handlers_gethandler(h, sel)) { + if (upb_handlers_gethandler(h, sel, NULL)) { putop(c, op, sel); } } @@ -574,9 +523,9 @@ static bool haslazyhandlers(const upb_handlers *h, const upb_fielddef *f) { if (!upb_fielddef_lazy(f)) return false; - return upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STARTSTR)) || - upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STRING)) || - upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_ENDSTR)); + return upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STARTSTR), NULL) || + upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_STRING), NULL) || + upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_ENDSTR), NULL); } @@ -814,10 +763,13 @@ static void find_methods(compiler *c, const upb_handlers *h) { upb_value v; upb_msg_field_iter i; const upb_msgdef *md; + upb_pbdecodermethod *method; if (upb_inttable_lookupptr(&c->group->methods, h, &v)) return; - newmethod(h, c->group); + + method = newmethod(h, c->group); + upb_inttable_insertptr(&c->group->methods, h, upb_value_ptr(method)); /* Find submethods. */ md = upb_handlers_msgdef(h); @@ -866,42 +818,15 @@ static void set_bytecode_handlers(mgroup *g) { } -/* JIT setup. *****************************************************************/ - -#ifdef UPB_USE_JIT_X64 - -static void sethandlers(mgroup *g, bool allowjit) { - g->jit_code = NULL; - if (allowjit) { - /* Compile byte-code into machine code, create handlers. */ - upb_pbdecoder_jit(g); - } else { - set_bytecode_handlers(g); - } -} - -#else /* UPB_USE_JIT_X64 */ - -static void sethandlers(mgroup *g, bool allowjit) { - /* No JIT compiled in; use bytecode handlers unconditionally. */ - UPB_UNUSED(allowjit); - set_bytecode_handlers(g); -} - -#endif /* UPB_USE_JIT_X64 */ - - /* TODO(haberman): allow this to be constructed for an arbitrary set of dest * handlers and other mgroups (but verify we have a transitive closure). */ -const mgroup *mgroup_new(const upb_handlers *dest, bool allowjit, bool lazy, - const void *owner) { +const mgroup *mgroup_new(const upb_handlers *dest, bool allowjit, bool lazy) { mgroup *g; compiler *c; UPB_UNUSED(allowjit); - UPB_ASSERT(upb_handlers_isfrozen(dest)); - g = newgroup(owner); + g = newgroup(); c = newcompiler(g, lazy); find_methods(c, dest); @@ -932,63 +857,71 @@ const mgroup *mgroup_new(const upb_handlers *dest, bool allowjit, bool lazy, } #endif - sethandlers(g, allowjit); + set_bytecode_handlers(g); return g; } /* upb_pbcodecache ************************************************************/ -void upb_pbcodecache_init(upb_pbcodecache *c) { - upb_inttable_init(&c->groups, UPB_CTYPE_CONSTPTR); - c->allow_jit_ = true; +upb_pbcodecache *upb_pbcodecache_new(upb_handlercache *dest) { + upb_pbcodecache *c = upb_gmalloc(sizeof(*c)); + + if (!c) return NULL; + + c->dest = dest; + c->allow_jit = true; + c->lazy = false; + + c->arena = upb_arena_new(); + if (!upb_inttable_init(&c->groups, UPB_CTYPE_CONSTPTR)) return NULL; + + return c; } -void upb_pbcodecache_uninit(upb_pbcodecache *c) { - upb_inttable_iter i; - upb_inttable_begin(&i, &c->groups); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - const mgroup *group = upb_value_getconstptr(upb_inttable_iter_value(&i)); - mgroup_unref(group, c); +void upb_pbcodecache_free(upb_pbcodecache *c) { + size_t i; + + for (i = 0; i < upb_inttable_count(&c->groups); i++) { + upb_value v; + bool ok = upb_inttable_lookup(&c->groups, i, &v); + UPB_ASSERT(ok); + freegroup((void*)upb_value_getconstptr(v)); } + upb_inttable_uninit(&c->groups); + upb_arena_free(c->arena); + upb_gfree(c); } bool upb_pbcodecache_allowjit(const upb_pbcodecache *c) { - return c->allow_jit_; + return c->allow_jit; } -bool upb_pbcodecache_setallowjit(upb_pbcodecache *c, bool allow) { - if (upb_inttable_count(&c->groups) > 0) - return false; - c->allow_jit_ = allow; - return true; +void upb_pbcodecache_setallowjit(upb_pbcodecache *c, bool allow) { + UPB_ASSERT(upb_inttable_count(&c->groups) == 0); + c->allow_jit = allow; +} + +void upb_pbdecodermethodopts_setlazy(upb_pbcodecache *c, bool lazy) { + UPB_ASSERT(upb_inttable_count(&c->groups) == 0); + c->lazy = lazy; } -const upb_pbdecodermethod *upb_pbcodecache_getdecodermethod( - upb_pbcodecache *c, const upb_pbdecodermethodopts *opts) { +const upb_pbdecodermethod *upb_pbcodecache_get(upb_pbcodecache *c, + const upb_msgdef *md) { upb_value v; bool ok; + const upb_handlers *h; + const mgroup *g; /* Right now we build a new DecoderMethod every time. * TODO(haberman): properly cache methods by their true key. */ - const mgroup *g = mgroup_new(opts->handlers, c->allow_jit_, opts->lazy, c); + h = upb_handlercache_get(c->dest, md); + g = mgroup_new(h, c->allow_jit, c->lazy); upb_inttable_push(&c->groups, upb_value_constptr(g)); - ok = upb_inttable_lookupptr(&g->methods, opts->handlers, &v); + ok = upb_inttable_lookupptr(&g->methods, h, &v); UPB_ASSERT(ok); return upb_value_getptr(v); } - - -/* upb_pbdecodermethodopts ****************************************************/ - -void upb_pbdecodermethodopts_init(upb_pbdecodermethodopts *opts, - const upb_handlers *h) { - opts->handlers = h; - opts->lazy = false; -} - -void upb_pbdecodermethodopts_setlazy(upb_pbdecodermethodopts *opts, bool lazy) { - opts->lazy = lazy; -} diff --git a/upb/pb/compile_decoder_x64.c b/upb/pb/compile_decoder_x64.c deleted file mode 100644 index fd541a4..0000000 --- a/upb/pb/compile_decoder_x64.c +++ /dev/null @@ -1,510 +0,0 @@ -/* -** Driver code for the x64 JIT compiler. -*/ - -/* Needed to ensure we get defines like MAP_ANON. */ -#define _GNU_SOURCE - -#include <dlfcn.h> -#include <stdio.h> -#include <sys/mman.h> -#include <unistd.h> -#include "upb/msg.h" -#include "upb/pb/decoder.h" -#include "upb/pb/decoder.int.h" -#include "upb/pb/varint.int.h" - -/* To debug the JIT: - * - * 1. Uncomment: - * #define UPB_JIT_LOAD_SO - * - * Note: this mode requires that we can shell out to gcc. - * - * 2. Run the test locally. This will load the JIT code by building a - * .so (/tmp/upb-jit-code.so) and using dlopen, so more of the tooling will - * work properly (like GDB). - * - * IF YOU ALSO WANT AUTOMATIC JIT DEBUG OUTPUT: - * - * 3. Run: upb/pb/make-gdb-script.rb > script.gdb. This reads - * /tmp/upb-jit-code.so as input and generates a GDB script that is specific - * to this jit code. - * - * 4. Run: gdb --command=script.gdb --args path/to/test - * This will drop you to a GDB prompt which you can now use normally. - * But when you run the test it will print a message to stdout every time - * the JIT executes assembly for a particular bytecode. Sample output: - * - * X.enterjit bytes=18 - * buf_ofs=1 data_rem=17 delim_rem=-2 X.0x6.OP_PARSE_DOUBLE - * buf_ofs=9 data_rem=9 delim_rem=-10 X.0x7.OP_CHECKDELIM - * buf_ofs=9 data_rem=9 delim_rem=-10 X.0x8.OP_TAG1 - * X.0x3.dispatch.DecoderTest - * X.parse_unknown - * X.0x3.dispatch.DecoderTest - * X.decode_unknown_tag_fallback - * X.exitjit - * - * This output should roughly correspond to the output that the bytecode - * interpreter emits when compiled with UPB_DUMP_BYTECODE (modulo some - * extra JIT-specific output). */ - -/* These defines are necessary for DynASM codegen. - * See dynasm/dasm_proto.h for more info. */ -#define Dst_DECL jitcompiler *jc -#define Dst_REF (jc->dynasm) -#define Dst (jc) - -/* In debug mode, make DynASM do internal checks (must be defined before any - * dasm header is included. */ -#ifndef NDEBUG -#define DASM_CHECKS -#endif - -#ifndef MAP_ANONYMOUS -#define MAP_ANONYMOUS MAP_ANON -#endif - -typedef struct { - mgroup *group; - uint32_t *pc; - - /* This pointer is allocated by dasm_init() and freed by dasm_free(). */ - struct dasm_State *dynasm; - - /* Maps some key (an arbitrary void*) to a pclabel. - * - * The pclabel represents a location in the generated code -- DynASM exposes - * a pclabel -> (machine code offset) lookup function. - * - * The key can be anything. There are two main kinds of keys: - * - bytecode location -- the void* points to the bytecode instruction - * itself. We can then use this to generate jumps to this instruction. - * - other object (like dispatch table). We use these to represent parts - * of the generated code that do not exactly correspond to a bytecode - * instruction. */ - upb_inttable jmptargets; - -#ifndef NDEBUG - /* Like jmptargets, but members are present in the table when they have had - * define_jmptarget() (as opposed to jmptarget) called. Used to verify that - * define_jmptarget() is called exactly once for every target. - * The value is ignored. */ - upb_inttable jmpdefined; - - /* For checking that two asmlabels aren't defined for the same byte. */ - int lastlabelofs; -#endif - -#ifdef UPB_JIT_LOAD_SO - /* For marking labels that should go into the generated code. - * Maps pclabel -> char* label (string is owned by the table). */ - upb_inttable asmlabels; -#endif - - /* The total number of pclabels currently defined. - * Note that this contains both jmptargets and asmlabels, which both use - * pclabels but for different purposes. */ - uint32_t pclabel_count; - - /* Used by DynASM to store globals. */ - void **globals; -} jitcompiler; - -/* Functions called by codegen. */ -static int jmptarget(jitcompiler *jc, const void *key); -static int define_jmptarget(jitcompiler *jc, const void *key); -static void asmlabel(jitcompiler *jc, const char *fmt, ...); -static int pcofs(jitcompiler* jc); -static int alloc_pclabel(jitcompiler *jc); - -#ifdef UPB_JIT_LOAD_SO -static char *upb_vasprintf(const char *fmt, va_list ap); -static char *upb_asprintf(const char *fmt, ...); -#endif - -#include "third_party/dynasm/dasm_proto.h" -#include "third_party/dynasm/dasm_x86.h" -#include "upb/pb/compile_decoder_x64.h" - -static jitcompiler *newjitcompiler(mgroup *group) { - jitcompiler *jc = malloc(sizeof(jitcompiler)); - jc->group = group; - jc->pclabel_count = 0; - upb_inttable_init(&jc->jmptargets, UPB_CTYPE_UINT32); -#ifndef NDEBUG - jc->lastlabelofs = -1; - upb_inttable_init(&jc->jmpdefined, UPB_CTYPE_BOOL); -#endif -#ifdef UPB_JIT_LOAD_SO - upb_inttable_init(&jc->asmlabels, UPB_CTYPE_PTR); -#endif - jc->globals = malloc(UPB_JIT_GLOBAL__MAX * sizeof(*jc->globals)); - - dasm_init(jc, 1); - dasm_setupglobal(jc, jc->globals, UPB_JIT_GLOBAL__MAX); - dasm_setup(jc, upb_jit_actionlist); - - return jc; -} - -static void freejitcompiler(jitcompiler *jc) { -#ifdef UPB_JIT_LOAD_SO - upb_inttable_iter i; - upb_inttable_begin(&i, &jc->asmlabels); - for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { - free(upb_value_getptr(upb_inttable_iter_value(&i))); - } - upb_inttable_uninit(&jc->asmlabels); -#endif -#ifndef NDEBUG - upb_inttable_uninit(&jc->jmpdefined); -#endif - upb_inttable_uninit(&jc->jmptargets); - dasm_free(jc); - free(jc->globals); - free(jc); -} - -#ifdef UPB_JIT_LOAD_SO - -/* Like sprintf except allocates the string, which is returned and owned by the - * caller. - * - * Like the GNU extension asprintf(), except we abort on error (since this is - * only for debugging). */ -static char *upb_vasprintf(const char *fmt, va_list args) { - /* Run once to get the length of the string. */ - va_list args_copy; - va_copy(args_copy, args); - int len = _upb_vsnprintf(NULL, 0, fmt, args_copy); - va_end(args_copy); - - char *ret = malloc(len + 1); /* + 1 for NULL terminator. */ - if (!ret) abort(); - int written = _upb_vsnprintf(ret, len + 1, fmt, args); - UPB_ASSERT(written == len); - - return ret; -} - -static char *upb_asprintf(const char *fmt, ...) { - va_list args; - va_start(args, fmt); - char *ret = upb_vasprintf(fmt, args); - va_end(args); - return ret; -} - -#endif - -static int alloc_pclabel(jitcompiler *jc) { - int newpc = jc->pclabel_count++; - dasm_growpc(jc, jc->pclabel_count); - return newpc; -} - -static bool try_getjmptarget(jitcompiler *jc, const void *key, int *pclabel) { - upb_value v; - if (upb_inttable_lookupptr(&jc->jmptargets, key, &v)) { - *pclabel = upb_value_getuint32(v); - return true; - } else { - return false; - } -} - -/* Gets the pclabel for this bytecode location's jmptarget. Requires that the - * jmptarget() has been previously defined. */ -static int getjmptarget(jitcompiler *jc, const void *key) { - int pclabel = 0; - bool ok; - - UPB_ASSERT_DEBUGVAR(upb_inttable_lookupptr(&jc->jmpdefined, key, NULL)); - ok = try_getjmptarget(jc, key, &pclabel); - UPB_ASSERT(ok); - return pclabel; -} - -/* Returns a pclabel that serves as a jmp target for the given bytecode pointer. - * This should only be called for code that is jumping to the target; code - * defining the target should use define_jmptarget(). - * - * Creates/allocates a pclabel for this target if one does not exist already. */ -static int jmptarget(jitcompiler *jc, const void *key) { - // Optimizer sometimes can't figure out that initializing this is unnecessary. - int pclabel = 0; - if (!try_getjmptarget(jc, key, &pclabel)) { - pclabel = alloc_pclabel(jc); - upb_inttable_insertptr(&jc->jmptargets, key, upb_value_uint32(pclabel)); - } - return pclabel; -} - -/* Defines a pclabel associated with the given bytecode location. - * Must be called exactly once by the code that is generating the code for this - * bytecode. - * - * Must be called exactly once before bytecode generation is complete (this is a - * sanity check to make sure the label is defined exactly once). */ -static int define_jmptarget(jitcompiler *jc, const void *key) { -#ifndef NDEBUG - upb_inttable_insertptr(&jc->jmpdefined, key, upb_value_bool(true)); -#endif - return jmptarget(jc, key); -} - -/* Returns a bytecode pc offset relative to the beginning of the group's - * code. */ -static int pcofs(jitcompiler *jc) { - return jc->pc - jc->group->bytecode; -} - -/* Returns a machine code offset corresponding to the given key. - * Requires that this key was defined with define_jmptarget. */ -static int machine_code_ofs(jitcompiler *jc, const void *key) { - int pclabel = getjmptarget(jc, key); - /* Despite its name, this function takes a pclabel and returns the - * corresponding machine code offset. */ - return dasm_getpclabel(jc, pclabel); -} - -/* Returns a machine code offset corresponding to the given method-relative - * bytecode offset. Note that the bytecode offset is relative to the given - * method, but the returned machine code offset is relative to the beginning of - * *all* the machine code. */ -static int machine_code_ofs2(jitcompiler *jc, const upb_pbdecodermethod *method, - int pcofs) { - void *bc_target = jc->group->bytecode + method->code_base.ofs + pcofs; - return machine_code_ofs(jc, bc_target); -} - -/* Given a pcofs relative to this method's base, returns a machine code offset - * relative to jmptarget(dispatch->array) (which is used in jitdispatch as the - * machine code base for dispatch table lookups). */ -uint32_t dispatchofs(jitcompiler *jc, const upb_pbdecodermethod *method, - int pcofs) { - int mc_base = machine_code_ofs(jc, method->dispatch.array); - int mc_target = machine_code_ofs2(jc, method, pcofs); - int ret; - - UPB_ASSERT(mc_base > 0); - UPB_ASSERT(mc_target > 0); - ret = mc_target - mc_base; - UPB_ASSERT(ret > 0); - return ret; -} - -/* Rewrites the dispatch tables into machine code offsets. */ -static void patchdispatch(jitcompiler *jc) { - upb_inttable_iter i; - upb_inttable_begin(&i, &jc->group->methods); - for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { - upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i)); - upb_inttable *dispatch = &method->dispatch; - upb_inttable_iter i2; - - method->is_native_ = true; - - /* Remove DISPATCH_ENDMSG -- only the bytecode interpreter needs it. - * And leaving it around will cause us to find field 0 improperly. */ - upb_inttable_remove(dispatch, DISPATCH_ENDMSG, NULL); - - upb_inttable_begin(&i2, dispatch); - for (; !upb_inttable_done(&i2); upb_inttable_next(&i2)) { - uintptr_t key = upb_inttable_iter_key(&i2); - uint64_t val = upb_value_getuint64(upb_inttable_iter_value(&i2)); - uint64_t newval; - bool ok; - if (key <= UPB_MAX_FIELDNUMBER) { - /* Primary slot. */ - uint64_t ofs; - uint8_t wt1; - uint8_t wt2; - upb_pbdecoder_unpackdispatch(val, &ofs, &wt1, &wt2); - - /* Update offset and repack. */ - ofs = dispatchofs(jc, method, ofs); - newval = upb_pbdecoder_packdispatch(ofs, wt1, wt2); - UPB_ASSERT((int64_t)newval > 0); - } else { - /* Secondary slot. Since we have 64 bits for the value, we use an - * absolute offset. */ - int mcofs = machine_code_ofs2(jc, method, val); - newval = (uint64_t)((char*)jc->group->jit_code + mcofs); - } - ok = upb_inttable_replace(dispatch, key, upb_value_uint64(newval)); - UPB_ASSERT(ok); - } - - /* Update entry point for this method to point at mc base instead of bc - * base. Set this only *after* we have patched the offsets - * (machine_code_ofs2() uses this). */ - method->code_base.ptr = (char*)jc->group->jit_code + machine_code_ofs(jc, method); - - { - upb_byteshandler *h = &method->input_handler_; - upb_byteshandler_setstartstr(h, upb_pbdecoder_startjit, NULL); - upb_byteshandler_setstring(h, jc->group->jit_code, method->code_base.ptr); - upb_byteshandler_setendstr(h, upb_pbdecoder_end, method); - } - } -} - -#ifdef UPB_JIT_LOAD_SO - -static void load_so(jitcompiler *jc) { - /* Dump to a .so file in /tmp and load that, so all the tooling works right - * (for example, debuggers and profilers will see symbol names for the JIT-ted - * code). This is the same goal of the GDB JIT code below, but the GDB JIT - * interface is only used/understood by GDB. Hopefully a standard will - * develop for registering JIT-ted code that all tools will recognize, - * rendering this obsolete. - * - * jc->asmlabels maps: - * pclabel -> char* label - * - * Use this to build mclabels, which maps: - * machine code offset -> char* label - * - * Then we can use mclabels to emit the labels as we iterate over the bytes we - * are outputting. */ - upb_inttable_iter i; - upb_inttable mclabels; - upb_inttable_init(&mclabels, UPB_CTYPE_PTR); - upb_inttable_begin(&i, &jc->asmlabels); - for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { - upb_inttable_insert(&mclabels, - dasm_getpclabel(jc, upb_inttable_iter_key(&i)), - upb_inttable_iter_value(&i)); - } - - /* We write a .s file in text format, as input to the assembler. - * Then we run gcc to turn it into a .so file. - * - * The last "XXXXXX" will be replaced with something randomly generated by - * mkstmemp(). We don't add ".s" to this filename because it makes the string - * processing for mkstemp() and system() more complicated. */ - char s_filename[] = "/tmp/upb-jit-codeXXXXXX"; - int fd = mkstemp(s_filename); - FILE *f; - if (fd >= 0 && (f = fdopen(fd, "wb")) != NULL) { - uint8_t *jit_code = (uint8_t*)jc->group->jit_code; - size_t linelen = 0; - size_t i; - fputs(" .text\n\n", f); - for (i = 0; i < jc->group->jit_size; i++) { - upb_value v; - if (upb_inttable_lookup(&mclabels, i, &v)) { - const char *label = upb_value_getptr(v); - /* "X." makes our JIT syms recognizable as such, which we build into - * other tooling. */ - fprintf(f, "\n\nX.%s:\n", label); - fprintf(f, " .globl X.%s", label); - linelen = 1000; - } - if (linelen >= 77) { - linelen = fprintf(f, "\n .byte %u", jit_code[i]); - } else { - linelen += fprintf(f, ",%u", jit_code[i]); - } - } - fputs("\n", f); - fclose(f); - } else { - fprintf(stderr, "Error opening tmp file for JIT debug output.\n"); - abort(); - } - - /* This is exploitable if you have an adversary on your machine who can write - * to this tmp directory. But this is just for debugging so we don't worry - * too much about that. It shouldn't be prone to races against concurrent - * (non-adversarial) upb JIT's because we used mkstemp(). */ - char *cmd = upb_asprintf("gcc -shared -o %s.so -x assembler %s", s_filename, - s_filename); - if (system(cmd) != 0) { - fprintf(stderr, "Error compiling %s\n", s_filename); - abort(); - } - free(cmd); - - char *so_filename = upb_asprintf("%s.so", s_filename); - - /* Some convenience symlinks. - * This is racy, but just for convenience. */ - int ret; - unlink("/tmp/upb-jit-code.so"); - unlink("/tmp/upb-jit-code.s"); - ret = symlink(s_filename, "/tmp/upb-jit-code.s"); - ret = symlink(so_filename, "/tmp/upb-jit-code.so"); - UPB_UNUSED(ret); // We don't care if this fails. - - jc->group->dl = dlopen(so_filename, RTLD_LAZY); - free(so_filename); - if (!jc->group->dl) { - fprintf(stderr, "Couldn't dlopen(): %s\n", dlerror()); - abort(); - } - - munmap(jc->group->jit_code, jc->group->jit_size); - jc->group->jit_code = dlsym(jc->group->dl, "X.enterjit"); - if (!jc->group->jit_code) { - fprintf(stderr, "Couldn't find enterjit sym\n"); - abort(); - } - - upb_inttable_uninit(&mclabels); -} - -#endif - -void upb_pbdecoder_jit(mgroup *group) { - jitcompiler *jc; - char *jit_code; - int dasm_status; - - group->debug_info = NULL; - group->dl = NULL; - - UPB_ASSERT(group->bytecode); - jc = newjitcompiler(group); - emit_static_asm(jc); - jitbytecode(jc); - - dasm_status = dasm_link(jc, &jc->group->jit_size); - if (dasm_status != DASM_S_OK) { - fprintf(stderr, "DynASM error; returned status: 0x%08x\n", dasm_status); - abort(); - } - - jit_code = mmap(NULL, jc->group->jit_size, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - dasm_encode(jc, jit_code); - mprotect(jit_code, jc->group->jit_size, PROT_EXEC | PROT_READ); - jc->group->jit_code = (upb_string_handlerfunc *)jit_code; - -#ifdef UPB_JIT_LOAD_SO - load_so(jc); -#endif - - patchdispatch(jc); - - freejitcompiler(jc); - - /* Now the bytecode is no longer needed. */ - free(group->bytecode); - group->bytecode = NULL; -} - -void upb_pbdecoder_freejit(mgroup *group) { - if (!group->jit_code) return; - if (group->dl) { -#ifdef UPB_JIT_LOAD_SO - dlclose(group->dl); -#endif - } else { - munmap((void*)group->jit_code, group->jit_size); - } - free(group->debug_info); -} diff --git a/upb/pb/compile_decoder_x64.dasc b/upb/pb/compile_decoder_x64.dasc deleted file mode 100644 index 7fcd006..0000000 --- a/upb/pb/compile_decoder_x64.dasc +++ /dev/null @@ -1,1145 +0,0 @@ -|// -|// upb - a minimalist implementation of protocol buffers. -|// -|// Copyright (c) 2011-2013 Google Inc. See LICENSE for details. -|// Author: Josh Haberman <jhaberman@gmail.com> -|// -|// JIT compiler for upb_pbdecoder on x86-64. Generates machine code from the -|// bytecode generated in compile_decoder.c. -| -|.arch x64 -|.actionlist upb_jit_actionlist -|.globals UPB_JIT_GLOBAL_ -|.globalnames upb_jit_globalnames -| -|// Calling conventions. Note -- this will need to be changed for -|// Windows, which uses a different calling convention! -|.define ARG1_64, rdi -|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi. -|.define ARG2_32, esi -|.define ARG2_64, rsi -|.define ARG3_8, dl -|.define ARG3_32, edx -|.define ARG3_64, rdx -|.define ARG4_64, rcx -|.define ARG5_64, r8 -|.define XMMARG1, xmm0 -| -|// Register allocation / type map. -|// ALL of the code in this file uses these register allocations. -|// When we "call" within this file, we do not use regular calling -|// conventions, but of course when calling to user callbacks we must. -|.define PTR, rbx // DECODER->ptr (unsynced) -|.define DATAEND, r12 // DECODER->data_end (unsynced) -|.define CLOSURE, r13 // FRAME->closure (unsynced) -|.type FRAME, upb_pbdecoder_frame, r14 // DECODER->top (unsynced) -|.type DECODER, upb_pbdecoder, r15 // DECODER (immutable) -|.define DELIMEND, rbp -| -| // Spills unsynced registers back to memory. -|.macro commit_regs -| mov DECODER->top, FRAME -| mov DECODER->ptr, PTR -| mov DECODER->data_end, DATAEND -| // We don't guarantee that delim_end is NULL when out of range like the -| // interpreter does. -| mov DECODER->delim_end, DELIMEND -| sub DELIMEND, DECODER->buf -| add DELIMEND, DECODER->bufstart_ofs -| mov FRAME->end_ofs, DELIMEND -| mov FRAME->sink.closure, CLOSURE -|.endmacro -| -| // Loads unsynced registers from memory back into registers. -|.macro load_regs -| mov FRAME, DECODER->top -| mov PTR, DECODER->ptr -| mov DATAEND, DECODER->data_end -| mov CLOSURE, FRAME->sink.closure -| mov DELIMEND, FRAME->end_ofs -| sub DELIMEND, DECODER->bufstart_ofs -| add DELIMEND, DECODER->buf -|.endmacro -| -| // Calls an external C function at address "addr". -|.macro callp, addr -| mov64 rax, (uintptr_t)addr -| -| // Stack must be 16-byte aligned (x86-64 ABI requires this). -| // -| // OPT: possibly remove this by statically ensuring correct alignment. -| // -| // OPT: use "call rel32" where possible. -| push r12 -| mov r12, rsp -| and rsp, 0xfffffffffffffff0UL // Align stack. -| call rax -| mov rsp, r12 -| pop r12 -|.endmacro -| -|.macro ld64, val -|| { -|| uintptr_t v = (uintptr_t)val; -|| if (v > 0xffffffff) { -| mov64 ARG2_64, v -|| } else if (v) { -| mov ARG2_32, v -|| } else { -| xor ARG2_32, ARG2_32 -|| } -|| } -|.endmacro -| -|.macro load_handler_data, h, arg -| ld64 upb_handlers_gethandlerdata(h, arg) -|.endmacro -| -|.macro chkeob, bytes, target -|| if (bytes == 1) { -| cmp PTR, DATAEND -| je target -|| } else { -| mov rcx, DATAEND -| sub rcx, PTR -| cmp rcx, bytes -| jb target -|| } -|.endmacro -| -|.macro chkneob, bytes, target -|| if (bytes == 1) { -| cmp PTR, DATAEND -| jne target -|| } else { -| mov rcx, DATAEND -| sub rcx, PTR -| cmp rcx, bytes -| jae target -|| } -|.endmacro - -|.macro sethas, reg, hasbit -|| if (hasbit >= 0) { -| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) -|| } -|.endmacro -| -| // Decodes 32-bit varint into rdx, inlining 1 byte. -|.macro dv32 -| chkeob 1, >7 -| movzx edx, byte [PTR] -| test dl, dl -| jns >8 -|7: -| call ->decodev32_fallback -|8: -| add PTR, 1 -|.endmacro - -#define DECODE_EOF -3 - -static upb_func *gethandler(const upb_handlers *h, upb_selector_t sel) { - return h ? upb_handlers_gethandler(h, sel) : NULL; -} - -/* Defines an "assembly label" for the current code generation offset. - * This label exists *purely* for debugging purposes: it is emitted into - * the .so, and printed as part of JIT debugging output when UPB_JIT_LOAD_SO is - * defined. - * - * We would define this in the .c file except that it conditionally defines a - * pclabel. */ -static void asmlabel(jitcompiler *jc, const char *fmt, ...) { -#ifndef NDEBUG - int ofs = jc->dynasm->section->ofs; - UPB_ASSERT(ofs != jc->lastlabelofs); - jc->lastlabelofs = ofs; -#endif - -#ifndef UPB_JIT_LOAD_SO - UPB_UNUSED(jc); - UPB_UNUSED(fmt); -#else - va_list args; - va_start(args, fmt); - char *str = upb_vasprintf(fmt, args); - va_end(args); - - int pclabel = alloc_pclabel(jc); - /* Normally we would prefer to allocate this inline with the codegen, - * ie. - * |=>asmlabel(...) - * But since we do this conditionally, only when UPB_JIT_LOAD_SO is defined, - * we do it here instead. */ - |=>pclabel: - upb_inttable_insert(&jc->asmlabels, pclabel, upb_value_ptr(str)); -#endif -} - -/* Should only be called when the associated handler is known to exist. */ -static bool alwaysok(const upb_handlers *h, upb_selector_t sel) { - upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; - bool ok = upb_handlers_getattr(h, sel, &attr); - bool ret; - - UPB_ASSERT(ok); - ret = upb_handlerattr_alwaysok(&attr); - upb_handlerattr_uninit(&attr); - return ret; -} - -/* Emit static assembly routines; code that does not vary based on the message - * schema. Since it's not input-dependent, we only need one single copy of it. - * For the moment we generate a single copy per generated handlers. Eventually - * we should generate this code at compile time and link it into the binary so - * we have one copy total. To do that we'll want to be sure that it is within - * 2GB of our JIT code, so that branches between the two are near (rel32). - * - * We'd put this assembly in a .s file directly, but DynASM's ability to - * calculate structure offsets automatically is too useful to pass up (it's way - * more convenient to write DECODER->sink than [rbx + 0x96], especially since - * the latter would have to be changed whenever the structure is updated). */ -static void emit_static_asm(jitcompiler *jc) { - | // Trampolines for entering/exiting the JIT. These are a bit tricky to - | // support full resuming; when we suspend we copy the JIT's portion of - | // the call stack into the upb_pbdecoder and restore it when we resume. - asmlabel(jc, "enterjit"); - |->enterjit: - |1: - | push rbp - | push r15 - | push r14 - | push r13 - | push r12 - | push rbx - | - | mov rbx, ARG2_64 // Preserve JIT method. - | - | mov DECODER, rdi - | callp upb_pbdecoder_resume // Same args as us; reuse regs. - | test eax, eax - | jns >1 - | mov DECODER->saved_rsp, rsp - | mov rax, rbx - | load_regs - | - | // Test whether we have a saved stack to resume. - | mov ARG3_64, DECODER->call_len - | test ARG3_64, ARG3_64 - | jnz >2 - | - | call rax - | - | mov rax, DECODER->size_param - | mov qword DECODER->call_len, 0 - |1: - | pop rbx - | pop r12 - | pop r13 - | pop r14 - | pop r15 - | pop rbp - | ret - | - |2: - | // Resume decoder. - | mov ARG2_64, DECODER->callstack - | sub rsp, ARG3_64 - | mov ARG1_64, rsp - | callp memcpy // Restore stack. - | ret // Return to resumed function (not ->enterjit caller). - | - | // Other code can call this to suspend the JIT. - | // To the calling code, it will appear that the function returns when - | // the JIT resumes, and more buffer space will be available. - | // Args: eax=the value that decode() should return. - asmlabel(jc, "exitjit"); - |->exitjit: - | // Save the stack into DECODER->callstack. - | mov ARG1_64, DECODER->callstack - | mov ARG2_64, rsp - | mov ARG3_64, DECODER->saved_rsp - | sub ARG3_64, rsp - | mov DECODER->call_len, ARG3_64 // Preserve len for next resume. - | mov ebx, eax // Preserve return value across memcpy. - | callp memcpy // Copy stack into decoder. - | mov eax, ebx // This will be our return value. - | - | // Must NOT do this before the memcpy(), otherwise memcpy() will - | // clobber the stack we are trying to save! - | mov rsp, DECODER->saved_rsp - | pop rbx - | pop r12 - | pop r13 - | pop r14 - | pop r15 - | pop rbp - | ret - | - | // Like suspend() in the C decoder, except that the function appears - | // (from the caller's perspective) not to return until the decoder is - | // resumed. - asmlabel(jc, "suspend"); - |->suspend: - | cmp DECODER->ptr, PTR - | je >1 - | mov DECODER->checkpoint, PTR - |1: - | commit_regs - | mov rdi, DECODER - | callp upb_pbdecoder_suspend - | jmp ->exitjit - | - asmlabel(jc, "pushlendelim"); - |->pushlendelim: - |1: - | mov FRAME->sink.closure, CLOSURE - | mov DECODER->checkpoint, PTR - | dv32 - | mov rcx, DELIMEND - | sub rcx, PTR - | sub rcx, rdx - | jb >4 // Len is greater than enclosing message. - | mov FRAME->end_ofs, rcx - | cmp FRAME, DECODER->limit - | je >3 // Stack overflow - | add FRAME, sizeof(upb_pbdecoder_frame) - | mov DELIMEND, PTR - | add DELIMEND, rdx - | mov dword FRAME->groupnum, 0 - | test rcx, rcx - | jz >2 - | mov DATAEND, DECODER->end - | cmp PTR, DELIMEND - | ja >2 - | cmp DELIMEND, DATAEND - | ja >2 - | mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND - |2: - | ret - |3: - | // Stack overflow error. - | mov PTR, DECODER->checkpoint // Rollback to before the delim len. - | // Prepare seterr args. - | mov ARG1_64, DECODER - | ld64 kPbDecoderStackOverflow - | callp upb_pbdecoder_seterr - | call ->suspend - | jmp <1 - |4: - | // Overextended len. - | mov PTR, DECODER->checkpoint // Rollback to before the delim len. - | // Prepare seterr args. - | mov ARG1_64, DECODER - | ld64 kPbDecoderSubmessageTooLong - | callp upb_pbdecoder_seterr - | call ->suspend - | jmp <1 - | - | // For getting a value that spans a buffer seam. Falls back to C. - |.macro getvalue_slow, func, bytes - | sub rsp, 8 // Need stack space for func to write value to. - |1: - | mov qword [rsp], 0 // For parsing routines that only parse 32 bits. - | mov ARG1_64, DECODER - | mov ARG2_64, rsp - | mov DECODER->checkpoint, PTR - | commit_regs - | callp func - | load_regs - | test eax, eax - | jns >2 - | // Success; return parsed data (in rdx AND xmm0). - | mov rdx, [rsp] - | movsd xmm0, qword [rsp] - | add rsp, 8 - | sub PTR, bytes // Bias our buffer pointer to rejoin the fast-path. - | mov DECODER->ptr, PTR - | ret - |2: - | call ->exitjit // Return eax from decode function. - | jmp <1 - |.endmacro - | - asmlabel(jc, "parse_unknown"); - | // Args: edx=fieldnum, cl=wire type - |->parse_unknown: - | // OPT: handle directly instead of kicking to C. - | // Check for ENDGROUP. - | mov ARG1_64, DECODER - | mov ARG2_32, edx - | movzx ARG3_32, cl - | commit_regs - | callp upb_pbdecoder_skipunknown - | load_regs - | cmp eax, DECODE_ENDGROUP - | jne >1 - | ret // Return eax=DECODE_ENDGROUP, not zero - |1: - | cmp eax, DECODE_OK - | je >1 - | call ->exitjit // Return eax from decode function. - |1: - | xor eax, eax - | ret - | - | // Fallback functions for parsing single values. These are used when the - | // buffer doesn't contain enough remaining data for the fast path. Each - | // primitive type (v32, v64, f32, f64) has two functions: decode & skip. - | // Decode functions return their value in rsi/esi. - | // - | // These functions leave PTR = value_end - fast_path_bytes, so that we can - | // re-join the fast path which will add fast_path_bytes after the callback - | // completes. We also set DECODER->ptr to this value which is a signal to - | // ->suspend that DECODER->checkpoint is up to date. - asmlabel(jc, "skip_decode_f32_fallback"); - |->skipf32_fallback: - |->decodef32_fallback: - | getvalue_slow upb_pbdecoder_decode_f32, 4 - | - asmlabel(jc, "skip_decode_f64_fallback"); - |->skipf64_fallback: - |->decodef64_fallback: - | getvalue_slow upb_pbdecoder_decode_f64, 8 - | - | // Called for varint >= 1 byte. - asmlabel(jc, "skip_decode_v32_fallback"); - |->skipv32_fallback: - |->skipv64_fallback: - | chkeob 16, >1 - | // With at least 16 bytes left, we can do a branch-less SSE version. - | movdqu xmm0, [PTR] - | pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0. - | not eax - | bsf eax, eax - | cmp al, 10 - | jae ->decode_varint_slow // Error (>10 byte varint). - | add PTR, rax // bsf result is 0-based, so PTR=end-1, as desired. - | ret - | - |1: - | // With fewer than 16 bytes, we have to read byte by byte. - | lea rcx, [PTR + 10] - | mov rax, PTR // Preserve PTR in case of fallback to slow path. - | cmp rcx, DATAEND - | cmova rcx, DATAEND // rcx = MIN(DATAEND, PTR + 10) - |2: - | cmp rax, rcx - | je ->decode_varint_slow - | test byte [rax], 0x80 - | jz >3 - | add rax, 1 - | jmp <2 - |3: - | mov PTR, rax // PTR = varint_end - 1, as desired - | ret - | - | // Returns tag in edx - asmlabel(jc, "decode_unknown_tag_fallback"); - |->decode_unknown_tag_fallback: - | sub rsp, 16 - |1: - | cmp PTR, DELIMEND - | jne >2 - | add rsp, 16 - | xor eax, eax - | ret - |2: - | // OPT: Have a medium-fast path before falling back to _slow. - | mov ARG1_64, DECODER - | mov ARG2_64, rsp - | commit_regs - | callp upb_pbdecoder_decode_varint_slow - | load_regs - | cmp eax, 0 - | jge >3 - | mov edx, [rsp] // Success; return parsed data. - | add rsp, 16 - | ret - |3: - | call ->exitjit // Return eax from decode function. - | jmp <1 - | - | // Called for varint >= 1 byte. - asmlabel(jc, "decode_v32_v64_fallback"); - |->decodev32_fallback: - |->decodev64_fallback: - | chkeob 10, ->decode_varint_slow - | // OPT: do something faster than just calling the C version. - | mov rdi, PTR - | callp upb_vdecode_fast - | test rax, rax - | je ->decode_varint_slow // Unterminated varint. - | mov PTR, rax - | sub PTR, 1 - | mov DECODER->ptr, PTR - | ret - | - asmlabel(jc, "decode_varint_slow"); - |->decode_varint_slow: - | // Slow path: end of buffer or error (varint length >= 10). - | getvalue_slow upb_pbdecoder_decode_varint_slow, 1 - | - | // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH}) - asmlabel(jc, "checktag_fallback"); - |->checktag_fallback: - | sub rsp, 8 - | mov [rsp], rsi // Preserve expected tag. - |1: - | mov ARG1_64, DECODER - | commit_regs - | mov DECODER->checkpoint, PTR - | callp upb_pbdecoder_checktag_slow - | load_regs - | cmp eax, 0 - | jge >2 - | add rsp, 8 - | ret - |2: - | call ->exitjit - | mov rsi, [rsp] - | cmp PTR, DELIMEND - | jne <1 - | mov eax, DECODE_EOF - | add rsp, 8 - | ret - | - | // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found). - | // Preserves: rcx, rdx - | // OPT: Could write this in assembly if it's a hotspot. - asmlabel(jc, "hashlookup"); - |->hashlookup: - | push rcx - | push rdx - | sub rsp, 16 - | mov rdi, rsi - | mov rsi, rdx - | mov rdx, rsp - | callp upb_inttable_lookup - | add rsp, 16 - | pop rdx - | pop rcx - | test al, al - | jz >2 // Unknown field. - | mov rax, [rsp-32] // Value from table. - | ret - |2: - | xor rax, rax - | not rax - | ret -} - -static void jitprimitive(jitcompiler *jc, opcode op, - const upb_handlers *h, upb_selector_t sel) { - typedef enum { V32, V64, F32, F64, X } valtype_t; - static valtype_t types[] = { - X, F64, F32, V64, V64, V32, F64, F32, V64, X, X, X, X, V32, V32, F32, F64, - V32, V64 }; - static char fastpath_bytes[] = { 1, 1, 4, 8 }; - const valtype_t vtype = types[op]; - const int fastbytes = fastpath_bytes[vtype]; - upb_func *handler = gethandler(h, sel); - upb_fieldtype_t ftype; - size_t offset; - int32_t hasbit; - - if (handler) { - |1: - | chkneob fastbytes, >3 - |2: - switch (vtype) { - case V32: - | call ->decodev32_fallback - break; - case V64: - | call ->decodev64_fallback - break; - case F32: - | call ->decodef32_fallback - break; - case F64: - | call ->decodef64_fallback - break; - case X: break; - } - | jmp >4 - - /* Fast path decode; for when check_bytes bytes are available. */ - |3: - switch (op) { - case OP_PARSE_SFIXED32: - case OP_PARSE_FIXED32: - | mov edx, dword [PTR] - break; - case OP_PARSE_SFIXED64: - case OP_PARSE_FIXED64: - | mov rdx, qword [PTR] - break; - case OP_PARSE_FLOAT: - | movss xmm0, dword [PTR] - break; - case OP_PARSE_DOUBLE: - | movsd xmm0, qword [PTR] - break; - default: - /* Inline one byte of varint decoding. */ - | movzx edx, byte [PTR] - | test dl, dl - | js <2 // Fallback to slow path for >1 byte varint. - break; - } - - /* Second-stage decode; used for both fast and slow paths */ - /* (only needed for a few types). */ - |4: - switch (op) { - case OP_PARSE_SINT32: - /* 32-bit zig-zag decode. */ - | mov eax, edx - | shr edx, 1 - | and eax, 1 - | neg eax - | xor edx, eax - break; - case OP_PARSE_SINT64: - /* 64-bit zig-zag decode. */ - | mov rax, rdx - | shr rdx, 1 - | and rax, 1 - | neg rax - | xor rdx, rax - break; - case OP_PARSE_BOOL: - | test rdx, rdx - | setne dl - break; - default: break; - } - - /* Call callback (or specialize if we can). */ - if (upb_msg_getscalarhandlerdata(h, sel, &ftype, &offset, &hasbit)) { - switch (ftype) { - case UPB_TYPE_INT64: - case UPB_TYPE_UINT64: - | mov [CLOSURE + offset], rdx - break; - case UPB_TYPE_INT32: - case UPB_TYPE_UINT32: - case UPB_TYPE_ENUM: - | mov [CLOSURE + offset], edx - break; - case UPB_TYPE_DOUBLE: - | movsd qword [CLOSURE + offset], XMMARG1 - break; - case UPB_TYPE_FLOAT: - | movss dword [CLOSURE + offset], XMMARG1 - break; - case UPB_TYPE_BOOL: - | mov [CLOSURE + offset], dl - break; - case UPB_TYPE_STRING: - case UPB_TYPE_BYTES: - case UPB_TYPE_MESSAGE: - UPB_ASSERT(false); break; - } - | sethas CLOSURE, hasbit - } else if (handler) { - | mov ARG1_64, CLOSURE - | load_handler_data h, sel - | callp handler - if (!alwaysok(h, sel)) { - | test al, al - | jnz >5 - | call ->suspend - | jmp <1 - |5: - } - } - - /* We do this last so that the checkpoint is not advanced past the user's - * data until the callback has returned success. */ - | add PTR, fastbytes - } else { - /* No handler registered for this value, just skip it. */ - | chkneob fastbytes, >3 - |2: - switch (vtype) { - case V32: - | call ->skipv32_fallback - break; - case V64: - | call ->skipv64_fallback - break; - case F32: - | call ->skipf32_fallback - break; - case F64: - | call ->skipf64_fallback - break; - case X: break; - } - - /* Fast-path skip. */ - |3: - if (vtype == V32 || vtype == V64) { - | test byte [PTR], 0x80 - | jnz <2 - } - | add PTR, fastbytes - } -} - -static void jitdispatch(jitcompiler *jc, - const upb_pbdecodermethod *method) { - /* Lots of room for tweaking/optimization here. */ - - const upb_inttable *dispatch = &method->dispatch; - bool has_hash_entries = (dispatch->t.count > 0); - - /* Whether any of the fields for this message can have two wire types which - * are both valid (packed & non-packed). - * - * OPT: populate this more precisely; not all messages with hash entries have - * this characteristic. */ - bool has_multi_wiretype = has_hash_entries; - - |=>define_jmptarget(jc, &method->dispatch): - |1: - /* Decode the field tag. */ - | mov aword DECODER->checkpoint, PTR - | chkeob 2, >6 - | movzx edx, byte [PTR] - | test dl, dl - | jns >7 // Jump if first byte has no continuation bit. - | movzx ecx, byte [PTR + 1] - | test cl, cl - | js >6 // Jump if second byte has continuation bit. - | // Confirmed two-byte varint. - | shl ecx, 7 - | and edx, 0x7f - | or edx, ecx - | add PTR, 2 - | jmp >8 - |6: - | call ->decode_unknown_tag_fallback - | test eax, eax // Hit DELIMEND? - | jnz >8 - | ret - |7: - | add PTR, 1 - |8: - | mov ecx, edx - | shr edx, 3 - | and cl, 7 - - /* See comment attached to upb_pbdecodermethod.dispatch for layout of the - * dispatch table. */ - |2: - | cmp edx, dispatch->array_size - if (has_hash_entries) { - | jae >7 - } else { - | jae >5 - } - | // OPT: Compact the lookup arr into 32-bit entries. - if ((uintptr_t)dispatch->array > 0x7fffffff) { - | mov64 rax, (uintptr_t)dispatch->array - | mov rax, qword [rax + rdx * 8] - } else { - | mov rax, qword [rdx * 8 + dispatch->array] - } - |3: - | // We take advantage of the fact that non-present entries are stored - | // as -1, which will result in wire types that will never match. - | cmp al, cl - if (has_multi_wiretype) { - | jne >6 - } else { - | jne >5 - } - | shr rax, 16 - | - | // Load the machine code address from the table entry. - | // The table entry is relative to the dispatch->array jmptarget - | // (patchdispatch() took care of this) which is the same as - | // local label "4". The "lea" is really just trying to do - | // lea rax, [>4 + rax] - | // - | // But we can't write that directly for some reason, so we use - | // rdx as a temporary. - | lea rdx, [>4] - |=>define_jmptarget(jc, dispatch->array): - |4: - | add rax, rdx - | ret - | - |5: - | // Field isn't in our table. - | - | // For pushing unknown fields to the unknown field handler. - | mov64 rax, (uintptr_t)method->dest_handlers_ - | mov FRAME->sink.handlers, rax - | - | call ->parse_unknown - | test eax, eax // ENDGROUP? - | jz <1 - | lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG. - | ret - - if (has_multi_wiretype) { - |6: - | // Primary wire type didn't match, check secondary wire type. - | cmp ah, cl - | jne <5 - | // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER. - | add rdx, UPB_MAX_FIELDNUMBER - | // This key will never be in the array part, so do a hash lookup. - UPB_ASSERT(has_hash_entries); - | ld64 dispatch - | jmp ->hashlookup // Tail call. - } - - if (has_hash_entries) { - |7: - | // Hash table lookup. - | ld64 dispatch - | call ->hashlookup - | jmp <3 - } -} - -static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, - const upb_pbdecodermethod *method) { - /* Internally we parse unknown fields; if this runs us into DELIMEND we jump - * to the corresponding DELIMEND target (either msg end or repeated field - * end), which we find from the OP_CHECKDELIM which must have necessarily - * preceded us. */ - uint32_t last_instruction = *(jc->pc - 2); - int last_arg = (int32_t)last_instruction >> 8; - uint32_t *delimend = (jc->pc - 1) + last_arg; - const size_t ptr_words = sizeof(void*) / sizeof(uint32_t); - - UPB_ASSERT((last_instruction & 0xff) == OP_CHECKDELIM); - - if (getop(*(jc->pc - 1)) == OP_TAGN) { - jc->pc += ptr_words; - } - - | chkneob n, >1 - - | // OPT: this is way too much fallback code to put here. - | // Reduce and/or move to a separate section to make better icache usage. - | ld64 tag - | call ->checktag_fallback - | cmp eax, DECODE_MISMATCH - | je >3 - | cmp eax, DECODE_EOF - | je =>jmptarget(jc, delimend) - | jmp >5 - - |1: - switch (n) { - case 1: - | cmp byte [PTR], tag - break; - case 2: - | cmp word [PTR], tag - break; - case 3: - | // OPT: Slightly more efficient code, but depends on an extra byte. - | // mov eax, dword [PTR] - | // shl eax, 8 - | // cmp eax, tag << 8 - | cmp word [PTR], (tag & 0xffff) - | jne >2 - | cmp byte [PTR + 2], (tag >> 16) - |2: - break; - case 4: - | cmp dword [PTR], tag - break; - case 5: - | cmp dword [PTR], (tag & 0xffffffff) - | jne >3 - | cmp byte [PTR + 4], (tag >> 32) - } - | je >4 - |3: - if (ofs == 0) { - | call =>jmptarget(jc, &method->dispatch) - | test rax, rax - | jz =>jmptarget(jc, delimend) - | jmp rax - } else { - | jmp =>jmptarget(jc, jc->pc + ofs) - } - |4: - | add PTR, n - |5: -} - -/* Compile the bytecode to x64. */ -static void jitbytecode(jitcompiler *jc) { - upb_pbdecodermethod *method = NULL; - const upb_handlers *h = NULL; - for (jc->pc = jc->group->bytecode; jc->pc < jc->group->bytecode_end; ) { - int32_t instr = *jc->pc; - opcode op = instr & 0xff; - uint32_t arg = instr >> 8; - int32_t longofs = arg; - - if (op != OP_SETDISPATCH) { - /* Skipped for SETDISPATCH because it defines its own asmlabel for the - * dispatch code it emits. */ - asmlabel(jc, "0x%lx.%s", pcofs(jc), upb_pbdecoder_getopname(op)); - - /* Skipped for SETDISPATCH because it should point at the function - * prologue, not the dispatch function that is emitted first. - * TODO: optimize this to only define pclabels that are actually used. */ - |=>define_jmptarget(jc, jc->pc): - } - - jc->pc++; - - switch (op) { - case OP_STARTMSG: { - upb_func *startmsg = gethandler(h, UPB_STARTMSG_SELECTOR); - if (startmsg) { - /* bool startmsg(void *closure, const void *hd) */ - |1: - | mov ARG1_64, CLOSURE - | load_handler_data h, UPB_STARTMSG_SELECTOR - | callp startmsg - if (!alwaysok(h, UPB_STARTMSG_SELECTOR)) { - | test al, al - | jnz >2 - | call ->suspend - | jmp <1 - |2: - } - } else { - | nop - } - break; - } - case OP_ENDMSG: { - upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR); - |9: - if (endmsg) { - /* bool endmsg(void *closure, const void *hd, upb_status *status) */ - | mov ARG1_64, CLOSURE - | load_handler_data h, UPB_ENDMSG_SELECTOR - | mov ARG3_64, DECODER->status - | callp endmsg - } - break; - } - case OP_SETDISPATCH: { - uint32_t *op_pc = jc->pc - 1; - const char *msgname; - upb_inttable *dispatch; - - /* Load info for new method. */ - memcpy(&dispatch, jc->pc, sizeof(void*)); - jc->pc += sizeof(void*) / sizeof(uint32_t); - /* The OP_SETDISPATCH bytecode contains a pointer that is - * &method->dispatch; we want to go backwards and recover method. */ - method = - (void*)((char*)dispatch - offsetof(upb_pbdecodermethod, dispatch)); - /* May be NULL, in which case no handlers for this message will be found. - * OPT: we should do better by completely skipping the message in this - * case instead of parsing it field by field. We should also do the skip - * in the containing message's code. */ - h = method->dest_handlers_; - msgname = upb_msgdef_fullname(upb_handlers_msgdef(h)); - - /* Emit dispatch code for new method. */ - asmlabel(jc, "0x%lx.dispatch.%s", pcofs(jc), msgname); - jitdispatch(jc, method); - - /* Emit function prologue for new method. */ - asmlabel(jc, "0x%lx.parse.%s", pcofs(jc), msgname); - |=>define_jmptarget(jc, op_pc): - |=>define_jmptarget(jc, method): - | sub rsp, 8 - - break; - } - case OP_PARSE_DOUBLE: - case OP_PARSE_FLOAT: - case OP_PARSE_INT64: - case OP_PARSE_UINT64: - case OP_PARSE_INT32: - case OP_PARSE_FIXED64: - case OP_PARSE_FIXED32: - case OP_PARSE_BOOL: - case OP_PARSE_UINT32: - case OP_PARSE_SFIXED32: - case OP_PARSE_SFIXED64: - case OP_PARSE_SINT32: - case OP_PARSE_SINT64: - jitprimitive(jc, op, h, arg); - break; - case OP_STARTSEQ: - case OP_STARTSUBMSG: - case OP_STARTSTR: { - upb_func *start = gethandler(h, arg); - if (start) { - /* void *startseq(void *closure, const void *hd) - * void *startsubmsg(void *closure, const void *hd) - * void *startstr(void *closure, const void *hd, size_t size_hint) */ - |1: - | mov ARG1_64, CLOSURE - | load_handler_data h, arg - if (op == OP_STARTSTR) { - | mov ARG3_64, DELIMEND - | sub ARG3_64, PTR - } - | callp start - if (!alwaysok(h, arg)) { - | test rax, rax - | jnz >2 - | call ->suspend - | jmp <1 - |2: - } - | mov CLOSURE, rax - } else { - /* TODO: nop is only required because of asmlabel(). */ - | nop - } - break; - } - case OP_ENDSEQ: - case OP_ENDSUBMSG: - case OP_ENDSTR: { - upb_func *end = gethandler(h, arg); - if (end) { - /* bool endseq(void *closure, const void *hd) - * bool endsubmsg(void *closure, const void *hd) - * bool endstr(void *closure, const void *hd) */ - |1: - | mov ARG1_64, CLOSURE - | load_handler_data h, arg - | callp end - if (!alwaysok(h, arg)) { - | test al, al - | jnz >2 - | call ->suspend - | jmp <1 - |2: - } - } else { - /* TODO: nop is only required because of asmlabel(). */ - | nop - } - break; - } - case OP_STRING: { - upb_func *str = gethandler(h, arg); - | cmp PTR, DELIMEND - | je >4 - |1: - | cmp PTR, DATAEND - | jne >2 - | call ->suspend - | jmp <1 - |2: - if (str) { - /* size_t str(void *closure, const void *hd, const char *str, - * size_t n) */ - | mov ARG1_64, CLOSURE - | load_handler_data h, arg - | mov ARG3_64, PTR - | mov ARG4_64, DATAEND - | sub ARG4_64, PTR - | mov ARG5_64, qword DECODER->handle - | callp str - | add PTR, rax - if (!alwaysok(h, arg)) { - | cmp PTR, DATAEND - | je >3 - | call ->strret_fallback - |3: - } - } else { - | mov PTR, DATAEND - } - | cmp PTR, DELIMEND - | jne <1 - |4: - break; - } - case OP_PUSHTAGDELIM: - | mov FRAME->sink.closure, CLOSURE - | // This shouldn't need to be read, because tag-delimited fields - | // shouldn't have an OP_SETDELIM after them. But for the moment - | // non-packed repeated fields do OP_SETDELIM so they can share more - | // code with the packed code-path. If this is changed later, this - | // store can be removed. - | mov qword FRAME->end_ofs, 0 - | cmp FRAME, DECODER->limit - | je ->err - | add FRAME, sizeof(upb_pbdecoder_frame) - | mov dword FRAME->groupnum, arg - break; - case OP_PUSHLENDELIM: - | call ->pushlendelim - break; - case OP_POP: - | sub FRAME, sizeof(upb_pbdecoder_frame) - | mov CLOSURE, FRAME->sink.closure - break; - case OP_SETDELIM: - /* OPT: experiment with testing vs old offset to optimize away. */ - | mov DATAEND, DECODER->end - | add DELIMEND, FRAME->end_ofs - | cmp DELIMEND, DECODER->buf - | jb >1 - | cmp DELIMEND, DATAEND - | ja >1 // OPT: try cmov. - | mov DATAEND, DELIMEND - |1: - break; - case OP_SETBIGGROUPNUM: - | mov dword FRAME->groupnum, *jc->pc++ - break; - case OP_CHECKDELIM: - | cmp DELIMEND, PTR - | je =>jmptarget(jc, jc->pc + longofs) - break; - case OP_CALL: - | call =>jmptarget(jc, jc->pc + longofs) - break; - case OP_BRANCH: - | jmp =>jmptarget(jc, jc->pc + longofs); - break; - case OP_RET: - |9: - | add rsp, 8 - | ret - break; - case OP_TAG1: - jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method); - break; - case OP_TAG2: - jittag(jc, (arg >> 8) & 0xffff, 2, (int8_t)arg, method); - break; - case OP_TAGN: { - uint64_t tag; - memcpy(&tag, jc->pc, 8); - jittag(jc, tag, arg >> 8, (int8_t)arg, method); - break; - } - case OP_DISPATCH: - | call =>jmptarget(jc, &method->dispatch) - break; - case OP_HALT: - UPB_ASSERT(false); - } - } - - asmlabel(jc, "eof"); - | nop -} diff --git a/upb/pb/compile_decoder_x64.h b/upb/pb/compile_decoder_x64.h deleted file mode 100644 index 2c07063..0000000 --- a/upb/pb/compile_decoder_x64.h +++ /dev/null @@ -1,1732 +0,0 @@ -/* -** This file has been pre-processed with DynASM. -** http://luajit.org/dynasm.html -** DynASM version 1.3.0, DynASM x64 version 1.3.0 -** DO NOT EDIT! The original file is in "upb/pb/compile_decoder_x64.dasc". -*/ - -#if DASM_VERSION != 10300 -#error "Version mismatch between DynASM and included encoding engine" -#endif - -# 1 "upb/pb/compile_decoder_x64.dasc" -/*|// */ -/*|// upb - a minimalist implementation of protocol buffers. */ -/*|// */ -/*|// Copyright (c) 2011-2013 Google Inc. See LICENSE for details. */ -/*|// Author: Josh Haberman <jhaberman@gmail.com> */ -/*|// */ -/*|// JIT compiler for upb_pbdecoder on x86-64. Generates machine code from the */ -/*|// bytecode generated in compile_decoder.c. */ -/*| */ -/*|.arch x64 */ -/*|.actionlist upb_jit_actionlist */ -static const unsigned char upb_jit_actionlist[2467] = { - 249,255,248,10,248,1,85,65,87,65,86,65,85,65,84,83,72,137,252,243,73,137, - 252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137, - 228,65,92,133,192,15,137,244,247,73,137,167,233,72,137,216,77,139,183,233, - 73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233, - 73,3,175,233,73,139,151,233,72,133,210,15,133,244,248,252,255,208,73,139, - 135,233,73,199,135,233,0,0,0,0,248,1,255,91,65,92,65,93,65,94,65,95,93,195, - 248,2,73,139,183,233,72,41,212,72,137,231,72,184,237,237,65,84,73,137,228, - 72,129,228,239,252,255,208,76,137,228,65,92,195,255,248,11,73,139,191,233, - 72,137,230,73,139,151,233,72,41,226,73,137,151,233,137,195,72,184,237,237, - 65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,137,216,73,139, - 167,233,91,65,92,65,93,65,94,65,95,93,195,255,248,12,73,57,159,233,15,132, - 244,247,73,137,159,233,248,1,77,137,183,233,73,137,159,233,77,137,167,233, - 73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,76, - 137,252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76, - 137,228,65,92,252,233,244,11,255,248,13,248,1,77,137,174,233,73,137,159,233, - 255,76,57,227,15,132,244,253,255,76,137,225,72,41,217,72,131,252,249,1,15, - 130,244,253,255,15,182,19,132,210,15,137,244,254,248,7,232,244,14,248,8,72, - 131,195,1,72,137,252,233,72,41,217,72,41,209,15,130,244,250,73,137,142,233, - 77,59,183,233,15,132,244,249,73,129,198,239,72,137,221,72,1,213,65,199,134, - 233,0,0,0,0,72,133,201,15,132,244,248,77,139,167,233,72,57,252,235,15,135, - 244,248,76,57,229,15,135,244,248,255,73,137,252,236,248,2,195,248,3,73,139, - 159,233,76,137,252,255,255,72,190,237,237,255,190,237,255,49,252,246,255, - 72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65, - 92,232,244,12,252,233,244,1,248,4,73,139,159,233,76,137,252,255,255,72,184, - 237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,232, - 244,12,252,233,244,1,255,248,15,76,137,252,255,137,214,15,182,209,77,137, - 183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175, - 233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228,72,129, - 228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139, - 167,233,77,139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,129,252, - 248,239,255,15,133,244,247,195,248,1,129,252,248,239,15,132,244,247,232,244, - 11,248,1,49,192,195,255,248,16,248,17,72,131,252,236,8,248,1,72,199,4,36, - 0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137,183,233,73,137,159, - 233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233, - 77,137,174,233,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208, - 76,137,228,65,92,77,139,183,233,73,139,159,233,77,139,167,233,77,139,174, - 233,73,139,174,233,255,73,43,175,233,73,3,175,233,133,192,15,137,244,248, - 72,139,20,36,252,242,15,16,4,36,72,131,196,8,72,131,252,235,4,73,137,159, - 233,195,248,2,232,244,11,252,233,244,1,255,248,18,248,19,72,131,252,236,8, - 248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137, - 183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175, - 233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228,72,129, - 228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139, - 167,233,77,139,174,233,73,139,174,233,255,73,43,175,233,73,3,175,233,133, - 192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72,131,196,8,72,131,252, - 235,8,73,137,159,233,195,248,2,232,244,11,252,233,244,1,255,248,20,248,21, - 255,76,57,227,15,132,244,247,255,76,137,225,72,41,217,72,131,252,249,16,15, - 130,244,247,255,252,243,15,111,3,102,15,215,192,252,247,208,15,188,192,60, - 10,15,131,244,22,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73, - 15,71,204,248,2,72,57,200,15,132,244,22,252,246,0,128,15,132,244,249,72,131, - 192,1,252,233,244,2,248,3,72,137,195,195,255,248,23,72,131,252,236,16,248, - 1,72,57,252,235,15,133,244,248,72,131,196,16,49,192,195,248,2,76,137,252, - 255,72,137,230,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175,233, - 73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,237,65, - 84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233, - 73,139,159,233,77,139,167,233,77,139,174,233,255,73,139,174,233,73,43,175, - 233,73,3,175,233,131,252,248,0,15,141,244,249,139,20,36,72,131,196,16,195, - 248,3,232,244,11,252,233,244,1,255,248,14,248,24,255,76,57,227,15,132,244, - 22,255,76,137,225,72,41,217,72,131,252,249,10,15,130,244,22,255,72,137,223, - 72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65, - 92,72,133,192,15,132,244,22,72,137,195,72,131,252,235,1,73,137,159,233,195, - 255,248,22,72,131,252,236,8,248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137, - 230,73,137,159,233,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175, - 233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,237, - 65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233, - 73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233, - 255,73,3,175,233,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72, - 131,196,8,72,131,252,235,1,73,137,159,233,195,248,2,232,244,11,252,233,244, - 1,255,248,25,72,131,252,236,8,72,137,52,36,248,1,76,137,252,255,77,137,183, - 233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,233, - 73,137,174,233,77,137,174,233,73,137,159,233,72,184,237,237,65,84,73,137, - 228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159, - 233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,255,73,3,175, - 233,131,252,248,0,15,141,244,248,72,131,196,8,195,248,2,232,244,11,72,139, - 52,36,72,57,252,235,15,133,244,1,184,237,72,131,196,8,195,255,248,26,81,82, - 72,131,252,236,16,72,137,252,247,72,137,214,72,137,226,72,184,237,237,65, - 84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,72,131,196,16,90, - 89,132,192,15,132,244,248,72,139,68,36,224,195,248,2,72,49,192,72,252,247, - 208,195,255,76,57,227,15,133,244,249,255,76,137,225,72,41,217,72,129,252, - 249,239,15,131,244,249,255,248,2,255,232,244,14,255,232,244,24,255,232,244, - 17,255,232,244,19,255,252,233,244,250,255,248,3,255,139,19,255,72,139,19, - 255,252,243,15,16,3,255,252,242,15,16,3,255,15,182,19,132,210,15,136,244, - 2,255,248,4,255,137,208,209,252,234,131,224,1,252,247,216,49,194,255,72,137, - 208,72,209,252,234,72,131,224,1,72,252,247,216,72,49,194,255,72,133,210,15, - 149,210,255,73,137,149,233,255,65,137,149,233,255,252,242,65,15,17,133,233, - 255,252,243,65,15,17,133,233,255,65,136,149,233,255,65,128,141,233,235,255, - 76,137,252,239,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255, - 208,76,137,228,65,92,255,132,192,15,133,244,251,232,244,12,252,233,244,1, - 248,5,255,72,129,195,239,255,232,244,20,255,232,244,21,255,232,244,16,255, - 232,244,18,255,252,246,3,128,15,133,244,2,255,249,248,1,255,76,57,227,15, - 132,244,252,255,76,137,225,72,41,217,72,131,252,249,2,15,130,244,252,255, - 15,182,19,132,210,15,137,244,253,15,182,139,233,132,201,15,136,244,252,193, - 225,7,131,226,127,9,202,72,131,195,2,252,233,244,254,248,6,232,244,23,133, - 192,15,133,244,254,195,248,7,72,131,195,1,248,8,137,209,193,252,234,3,128, - 225,7,255,248,2,129,252,250,239,255,15,131,244,253,255,15,131,244,251,255, - 72,184,237,237,72,139,4,208,255,72,139,4,213,237,255,248,3,56,200,255,15, - 133,244,252,255,15,133,244,251,255,72,193,232,16,72,141,21,244,250,249,248, - 4,72,1,208,195,248,5,72,184,237,237,73,137,134,233,232,244,15,133,192,15, - 132,244,1,72,141,5,244,255,195,255,248,6,56,204,15,133,244,5,72,129,194,239, - 255,252,233,244,26,255,248,7,255,232,244,26,252,233,244,3,255,76,57,227,15, - 133,244,247,255,76,137,225,72,41,217,72,129,252,249,239,15,131,244,247,255, - 232,244,25,129,252,248,239,15,132,244,249,129,252,248,239,15,132,245,252, - 233,244,251,255,128,59,235,255,102,129,59,238,255,102,129,59,238,15,133,244, - 248,128,187,233,235,248,2,255,129,59,239,255,129,59,239,15,133,244,249,128, - 187,233,235,255,15,132,244,250,248,3,255,232,245,72,133,192,15,132,245,252, - 255,224,255,252,233,245,255,248,4,72,129,195,239,248,5,255,248,1,76,137,252, - 239,255,132,192,15,133,244,248,232,244,12,252,233,244,1,248,2,255,144,255, - 248,9,255,73,139,151,233,72,184,237,237,65,84,73,137,228,72,129,228,239,252, - 255,208,76,137,228,65,92,255,249,249,72,131,252,236,8,255,72,137,252,234, - 72,41,218,255,72,133,192,15,133,244,248,232,244,12,252,233,244,1,248,2,255, - 73,137,197,255,72,57,252,235,15,132,244,250,248,1,76,57,227,15,133,244,248, - 232,244,12,252,233,244,1,248,2,255,72,137,218,76,137,225,72,41,217,77,139, - 135,233,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137, - 228,65,92,72,1,195,255,76,57,227,15,132,244,249,232,244,27,248,3,255,76,137, - 227,255,72,57,252,235,15,133,244,1,248,4,255,77,137,174,233,73,199,134,233, - 0,0,0,0,77,59,183,233,15,132,244,28,73,129,198,239,65,199,134,233,237,255, - 232,244,13,255,73,129,252,238,239,77,139,174,233,255,77,139,167,233,73,3, - 174,233,73,59,175,233,15,130,244,247,76,57,229,15,135,244,247,73,137,252, - 236,248,1,255,72,57,221,15,132,245,255,232,245,255,248,9,72,131,196,8,195, - 255 -}; - -# 12 "upb/pb/compile_decoder_x64.dasc" -/*|.globals UPB_JIT_GLOBAL_ */ -enum { - UPB_JIT_GLOBAL_enterjit, - UPB_JIT_GLOBAL_exitjit, - UPB_JIT_GLOBAL_suspend, - UPB_JIT_GLOBAL_pushlendelim, - UPB_JIT_GLOBAL_decodev32_fallback, - UPB_JIT_GLOBAL_parse_unknown, - UPB_JIT_GLOBAL_skipf32_fallback, - UPB_JIT_GLOBAL_decodef32_fallback, - UPB_JIT_GLOBAL_skipf64_fallback, - UPB_JIT_GLOBAL_decodef64_fallback, - UPB_JIT_GLOBAL_skipv32_fallback, - UPB_JIT_GLOBAL_skipv64_fallback, - UPB_JIT_GLOBAL_decode_varint_slow, - UPB_JIT_GLOBAL_decode_unknown_tag_fallback, - UPB_JIT_GLOBAL_decodev64_fallback, - UPB_JIT_GLOBAL_checktag_fallback, - UPB_JIT_GLOBAL_hashlookup, - UPB_JIT_GLOBAL_strret_fallback, - UPB_JIT_GLOBAL_err, - UPB_JIT_GLOBAL__MAX -}; -# 13 "upb/pb/compile_decoder_x64.dasc" -/*|.globalnames upb_jit_globalnames */ -static const char *const upb_jit_globalnames[] = { - "enterjit", - "exitjit", - "suspend", - "pushlendelim", - "decodev32_fallback", - "parse_unknown", - "skipf32_fallback", - "decodef32_fallback", - "skipf64_fallback", - "decodef64_fallback", - "skipv32_fallback", - "skipv64_fallback", - "decode_varint_slow", - "decode_unknown_tag_fallback", - "decodev64_fallback", - "checktag_fallback", - "hashlookup", - "strret_fallback", - "err", - (const char *)0 -}; -# 14 "upb/pb/compile_decoder_x64.dasc" -/*| */ -/*|// Calling conventions. Note -- this will need to be changed for */ -/*|// Windows, which uses a different calling convention! */ -/*|.define ARG1_64, rdi */ -/*|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi. */ -/*|.define ARG2_32, esi */ -/*|.define ARG2_64, rsi */ -/*|.define ARG3_8, dl */ -/*|.define ARG3_32, edx */ -/*|.define ARG3_64, rdx */ -/*|.define ARG4_64, rcx */ -/*|.define ARG5_64, r8 */ -/*|.define XMMARG1, xmm0 */ -/*| */ -/*|// Register allocation / type map. */ -/*|// ALL of the code in this file uses these register allocations. */ -/*|// When we "call" within this file, we do not use regular calling */ -/*|// conventions, but of course when calling to user callbacks we must. */ -/*|.define PTR, rbx // DECODER->ptr (unsynced) */ -/*|.define DATAEND, r12 // DECODER->data_end (unsynced) */ -/*|.define CLOSURE, r13 // FRAME->closure (unsynced) */ -/*|.type FRAME, upb_pbdecoder_frame, r14 // DECODER->top (unsynced) */ -#define Dt1(_V) (int)(ptrdiff_t)&(((upb_pbdecoder_frame *)0)_V) -# 36 "upb/pb/compile_decoder_x64.dasc" -/*|.type DECODER, upb_pbdecoder, r15 // DECODER (immutable) */ -#define Dt2(_V) (int)(ptrdiff_t)&(((upb_pbdecoder *)0)_V) -# 37 "upb/pb/compile_decoder_x64.dasc" -/*|.define DELIMEND, rbp */ -/*| */ -/*| // Spills unsynced registers back to memory. */ -/*|.macro commit_regs */ -/*| mov DECODER->top, FRAME */ -/*| mov DECODER->ptr, PTR */ -/*| mov DECODER->data_end, DATAEND */ -/*| // We don't guarantee that delim_end is NULL when out of range like the */ -/*| // interpreter does. */ -/*| mov DECODER->delim_end, DELIMEND */ -/*| sub DELIMEND, DECODER->buf */ -/*| add DELIMEND, DECODER->bufstart_ofs */ -/*| mov FRAME->end_ofs, DELIMEND */ -/*| mov FRAME->sink.closure, CLOSURE */ -/*|.endmacro */ -/*| */ -/*| // Loads unsynced registers from memory back into registers. */ -/*|.macro load_regs */ -/*| mov FRAME, DECODER->top */ -/*| mov PTR, DECODER->ptr */ -/*| mov DATAEND, DECODER->data_end */ -/*| mov CLOSURE, FRAME->sink.closure */ -/*| mov DELIMEND, FRAME->end_ofs */ -/*| sub DELIMEND, DECODER->bufstart_ofs */ -/*| add DELIMEND, DECODER->buf */ -/*|.endmacro */ -/*| */ -/*| // Calls an external C function at address "addr". */ -/*|.macro callp, addr */ -/*| mov64 rax, (uintptr_t)addr */ -/*| */ -/*| // Stack must be 16-byte aligned (x86-64 ABI requires this). */ -/*| // */ -/*| // OPT: possibly remove this by statically ensuring correct alignment. */ -/*| // */ -/*| // OPT: use "call rel32" where possible. */ -/*| push r12 */ -/*| mov r12, rsp */ -/*| and rsp, 0xfffffffffffffff0UL // Align stack. */ -/*| call rax */ -/*| mov rsp, r12 */ -/*| pop r12 */ -/*|.endmacro */ -/*| */ -/*|.macro ld64, val */ -/*|| { */ -/*|| uintptr_t v = (uintptr_t)val; */ -/*|| if (v > 0xffffffff) { */ -/*| mov64 ARG2_64, v */ -/*|| } else if (v) { */ -/*| mov ARG2_32, v */ -/*|| } else { */ -/*| xor ARG2_32, ARG2_32 */ -/*|| } */ -/*|| } */ -/*|.endmacro */ -/*| */ -/*|.macro load_handler_data, h, arg */ -/*| ld64 upb_handlers_gethandlerdata(h, arg) */ -/*|.endmacro */ -/*| */ -/*|.macro chkeob, bytes, target */ -/*|| if (bytes == 1) { */ -/*| cmp PTR, DATAEND */ -/*| je target */ -/*|| } else { */ -/*| mov rcx, DATAEND */ -/*| sub rcx, PTR */ -/*| cmp rcx, bytes */ -/*| jb target */ -/*|| } */ -/*|.endmacro */ -/*| */ -/*|.macro chkneob, bytes, target */ -/*|| if (bytes == 1) { */ -/*| cmp PTR, DATAEND */ -/*| jne target */ -/*|| } else { */ -/*| mov rcx, DATAEND */ -/*| sub rcx, PTR */ -/*| cmp rcx, bytes */ -/*| jae target */ -/*|| } */ -/*|.endmacro */ - -/*|.macro sethas, reg, hasbit */ -/*|| if (hasbit >= 0) { */ -/*| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) */ -/*|| } */ -/*|.endmacro */ -/*| */ -/*| // Decodes 32-bit varint into rdx, inlining 1 byte. */ -/*|.macro dv32 */ -/*| chkeob 1, >7 */ -/*| movzx edx, byte [PTR] */ -/*| test dl, dl */ -/*| jns >8 */ -/*|7: */ -/*| call ->decodev32_fallback */ -/*|8: */ -/*| add PTR, 1 */ -/*|.endmacro */ - -#define DECODE_EOF -3 - -static upb_func *gethandler(const upb_handlers *h, upb_selector_t sel) { - return h ? upb_handlers_gethandler(h, sel) : NULL; -} - -/* Defines an "assembly label" for the current code generation offset. - * This label exists *purely* for debugging purposes: it is emitted into - * the .so, and printed as part of JIT debugging output when UPB_JIT_LOAD_SO is - * defined. - * - * We would define this in the .c file except that it conditionally defines a - * pclabel. */ -static void asmlabel(jitcompiler *jc, const char *fmt, ...) { -#ifndef NDEBUG - int ofs = jc->dynasm->section->ofs; - UPB_ASSERT(ofs != jc->lastlabelofs); - jc->lastlabelofs = ofs; -#endif - -#ifndef UPB_JIT_LOAD_SO - UPB_UNUSED(jc); - UPB_UNUSED(fmt); -#else - va_list args; - va_start(args, fmt); - char *str = upb_vasprintf(fmt, args); - va_end(args); - - int pclabel = alloc_pclabel(jc); - /* Normally we would prefer to allocate this inline with the codegen, - * ie. - * |=>asmlabel(...) - * But since we do this conditionally, only when UPB_JIT_LOAD_SO is defined, - * we do it here instead. */ - /*|=>pclabel: */ - dasm_put(Dst, 0, pclabel); -# 176 "upb/pb/compile_decoder_x64.dasc" - upb_inttable_insert(&jc->asmlabels, pclabel, upb_value_ptr(str)); -#endif -} - -/* Should only be called when the associated handler is known to exist. */ -static bool alwaysok(const upb_handlers *h, upb_selector_t sel) { - upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; - bool ok = upb_handlers_getattr(h, sel, &attr); - bool ret; - - UPB_ASSERT(ok); - ret = upb_handlerattr_alwaysok(&attr); - upb_handlerattr_uninit(&attr); - return ret; -} - -/* Emit static assembly routines; code that does not vary based on the message - * schema. Since it's not input-dependent, we only need one single copy of it. - * For the moment we generate a single copy per generated handlers. Eventually - * we should generate this code at compile time and link it into the binary so - * we have one copy total. To do that we'll want to be sure that it is within - * 2GB of our JIT code, so that branches between the two are near (rel32). - * - * We'd put this assembly in a .s file directly, but DynASM's ability to - * calculate structure offsets automatically is too useful to pass up (it's way - * more convenient to write DECODER->sink than [rbx + 0x96], especially since - * the latter would have to be changed whenever the structure is updated). */ -static void emit_static_asm(jitcompiler *jc) { - /*| // Trampolines for entering/exiting the JIT. These are a bit tricky to */ - /*| // support full resuming; when we suspend we copy the JIT's portion of */ - /*| // the call stack into the upb_pbdecoder and restore it when we resume. */ - asmlabel(jc, "enterjit"); - /*|->enterjit: */ - /*|1: */ - /*| push rbp */ - /*| push r15 */ - /*| push r14 */ - /*| push r13 */ - /*| push r12 */ - /*| push rbx */ - /*| */ - /*| mov rbx, ARG2_64 // Preserve JIT method. */ - /*| */ - /*| mov DECODER, rdi */ - /*| callp upb_pbdecoder_resume // Same args as us; reuse regs. */ - /*| test eax, eax */ - /*| jns >1 */ - /*| mov DECODER->saved_rsp, rsp */ - /*| mov rax, rbx */ - /*| load_regs */ - /*| */ - /*| // Test whether we have a saved stack to resume. */ - /*| mov ARG3_64, DECODER->call_len */ - /*| test ARG3_64, ARG3_64 */ - /*| jnz >2 */ - /*| */ - /*| call rax */ - /*| */ - /*| mov rax, DECODER->size_param */ - /*| mov qword DECODER->call_len, 0 */ - /*|1: */ - /*| pop rbx */ - dasm_put(Dst, 2, (unsigned int)((uintptr_t)upb_pbdecoder_resume), (unsigned int)(((uintptr_t)upb_pbdecoder_resume)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->call_len), Dt2(->size_param), Dt2(->call_len)); -# 238 "upb/pb/compile_decoder_x64.dasc" - /*| pop r12 */ - /*| pop r13 */ - /*| pop r14 */ - /*| pop r15 */ - /*| pop rbp */ - /*| ret */ - /*| */ - /*|2: */ - /*| // Resume decoder. */ - /*| mov ARG2_64, DECODER->callstack */ - /*| sub rsp, ARG3_64 */ - /*| mov ARG1_64, rsp */ - /*| callp memcpy // Restore stack. */ - /*| ret // Return to resumed function (not ->enterjit caller). */ - /*| */ - /*| // Other code can call this to suspend the JIT. */ - /*| // To the calling code, it will appear that the function returns when */ - /*| // the JIT resumes, and more buffer space will be available. */ - /*| // Args: eax=the value that decode() should return. */ - dasm_put(Dst, 115, Dt2(->callstack), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL); -# 257 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "exitjit"); - /*|->exitjit: */ - /*| // Save the stack into DECODER->callstack. */ - /*| mov ARG1_64, DECODER->callstack */ - /*| mov ARG2_64, rsp */ - /*| mov ARG3_64, DECODER->saved_rsp */ - /*| sub ARG3_64, rsp */ - /*| mov DECODER->call_len, ARG3_64 // Preserve len for next resume. */ - /*| mov ebx, eax // Preserve return value across memcpy. */ - /*| callp memcpy // Copy stack into decoder. */ - /*| mov eax, ebx // This will be our return value. */ - /*| */ - /*| // Must NOT do this before the memcpy(), otherwise memcpy() will */ - /*| // clobber the stack we are trying to save! */ - /*| mov rsp, DECODER->saved_rsp */ - /*| pop rbx */ - /*| pop r12 */ - /*| pop r13 */ - /*| pop r14 */ - /*| pop r15 */ - /*| pop rbp */ - /*| ret */ - /*| */ - /*| // Like suspend() in the C decoder, except that the function appears */ - /*| // (from the caller's perspective) not to return until the decoder is */ - /*| // resumed. */ - dasm_put(Dst, 161, Dt2(->callstack), Dt2(->saved_rsp), Dt2(->call_len), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp)); -# 283 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "suspend"); - /*|->suspend: */ - /*| cmp DECODER->ptr, PTR */ - /*| je >1 */ - /*| mov DECODER->checkpoint, PTR */ - /*|1: */ - /*| commit_regs */ - /*| mov rdi, DECODER */ - /*| callp upb_pbdecoder_suspend */ - /*| jmp ->exitjit */ - /*| */ - dasm_put(Dst, 222, Dt2(->ptr), Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_suspend), (unsigned int)(((uintptr_t)upb_pbdecoder_suspend)>>32), 0xfffffffffffffff0UL); -# 294 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "pushlendelim"); - /*|->pushlendelim: */ - /*|1: */ - /*| mov FRAME->sink.closure, CLOSURE */ - /*| mov DECODER->checkpoint, PTR */ - /*| dv32 */ - dasm_put(Dst, 300, Dt1(->sink.closure), Dt2(->checkpoint)); - if (1 == 1) { - dasm_put(Dst, 313); - } else { - dasm_put(Dst, 321); - } -# 300 "upb/pb/compile_decoder_x64.dasc" - /*| mov rcx, DELIMEND */ - /*| sub rcx, PTR */ - /*| sub rcx, rdx */ - /*| jb >4 // Len is greater than enclosing message. */ - /*| mov FRAME->end_ofs, rcx */ - /*| cmp FRAME, DECODER->limit */ - /*| je >3 // Stack overflow */ - /*| add FRAME, sizeof(upb_pbdecoder_frame) */ - /*| mov DELIMEND, PTR */ - /*| add DELIMEND, rdx */ - /*| mov dword FRAME->groupnum, 0 */ - /*| test rcx, rcx */ - /*| jz >2 */ - /*| mov DATAEND, DECODER->end */ - /*| cmp PTR, DELIMEND */ - /*| ja >2 */ - /*| cmp DELIMEND, DATAEND */ - /*| ja >2 */ - /*| mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND */ - dasm_put(Dst, 337, Dt1(->end_ofs), Dt2(->limit), sizeof(upb_pbdecoder_frame), Dt1(->groupnum), Dt2(->end)); -# 319 "upb/pb/compile_decoder_x64.dasc" - /*|2: */ - /*| ret */ - /*|3: */ - /*| // Stack overflow error. */ - /*| mov PTR, DECODER->checkpoint // Rollback to before the delim len. */ - /*| // Prepare seterr args. */ - /*| mov ARG1_64, DECODER */ - /*| ld64 kPbDecoderStackOverflow */ - dasm_put(Dst, 428, Dt2(->checkpoint)); - { - uintptr_t v = (uintptr_t)kPbDecoderStackOverflow; - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 327 "upb/pb/compile_decoder_x64.dasc" - /*| callp upb_pbdecoder_seterr */ - /*| call ->suspend */ - /*| jmp <1 */ - /*|4: */ - /*| // Overextended len. */ - /*| mov PTR, DECODER->checkpoint // Rollback to before the delim len. */ - /*| // Prepare seterr args. */ - /*| mov ARG1_64, DECODER */ - /*| ld64 kPbDecoderSubmessageTooLong */ - dasm_put(Dst, 458, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32), 0xfffffffffffffff0UL, Dt2(->checkpoint)); - { - uintptr_t v = (uintptr_t)kPbDecoderSubmessageTooLong; - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 336 "upb/pb/compile_decoder_x64.dasc" - /*| callp upb_pbdecoder_seterr */ - /*| call ->suspend */ - /*| jmp <1 */ - /*| */ - /*| // For getting a value that spans a buffer seam. Falls back to C. */ - /*|.macro getvalue_slow, func, bytes */ - /*| sub rsp, 8 // Need stack space for func to write value to. */ - /*|1: */ - /*| mov qword [rsp], 0 // For parsing routines that only parse 32 bits. */ - /*| mov ARG1_64, DECODER */ - /*| mov ARG2_64, rsp */ - /*| mov DECODER->checkpoint, PTR */ - /*| commit_regs */ - /*| callp func */ - /*| load_regs */ - /*| test eax, eax */ - /*| jns >2 */ - /*| // Success; return parsed data (in rdx AND xmm0). */ - /*| mov rdx, [rsp] */ - /*| movsd xmm0, qword [rsp] */ - /*| add rsp, 8 */ - /*| sub PTR, bytes // Bias our buffer pointer to rejoin the fast-path. */ - /*| mov DECODER->ptr, PTR */ - /*| ret */ - /*|2: */ - /*| call ->exitjit // Return eax from decode function. */ - /*| jmp <1 */ - /*|.endmacro */ - /*| */ - dasm_put(Dst, 497, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32), 0xfffffffffffffff0UL); -# 365 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "parse_unknown"); - /*| // Args: edx=fieldnum, cl=wire type */ - /*|->parse_unknown: */ - /*| // OPT: handle directly instead of kicking to C. */ - /*| // Check for ENDGROUP. */ - /*| mov ARG1_64, DECODER */ - /*| mov ARG2_32, edx */ - /*| movzx ARG3_32, cl */ - /*| commit_regs */ - /*| callp upb_pbdecoder_skipunknown */ - /*| load_regs */ - /*| cmp eax, DECODE_ENDGROUP */ - /*| jne >1 */ - dasm_put(Dst, 526, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP); -# 378 "upb/pb/compile_decoder_x64.dasc" - /*| ret // Return eax=DECODE_ENDGROUP, not zero */ - /*|1: */ - /*| cmp eax, DECODE_OK */ - /*| je >1 */ - /*| call ->exitjit // Return eax from decode function. */ - /*|1: */ - /*| xor eax, eax */ - /*| ret */ - /*| */ - /*| // Fallback functions for parsing single values. These are used when the */ - /*| // buffer doesn't contain enough remaining data for the fast path. Each */ - /*| // primitive type (v32, v64, f32, f64) has two functions: decode & skip. */ - /*| // Decode functions return their value in rsi/esi. */ - /*| // */ - /*| // These functions leave PTR = value_end - fast_path_bytes, so that we can */ - /*| // re-join the fast path which will add fast_path_bytes after the callback */ - /*| // completes. We also set DECODER->ptr to this value which is a signal to */ - /*| // ->suspend that DECODER->checkpoint is up to date. */ - dasm_put(Dst, 623, DECODE_OK); -# 396 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "skip_decode_f32_fallback"); - /*|->skipf32_fallback: */ - /*|->decodef32_fallback: */ - /*| getvalue_slow upb_pbdecoder_decode_f32, 4 */ - dasm_put(Dst, 647, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs)); -# 400 "upb/pb/compile_decoder_x64.dasc" - /*| */ - dasm_put(Dst, 751, Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->ptr)); -# 401 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "skip_decode_f64_fallback"); - /*|->skipf64_fallback: */ - /*|->decodef64_fallback: */ - /*| getvalue_slow upb_pbdecoder_decode_f64, 8 */ - dasm_put(Dst, 799, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs)); -# 405 "upb/pb/compile_decoder_x64.dasc" - /*| */ - /*| // Called for varint >= 1 byte. */ - dasm_put(Dst, 903, Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->ptr)); -# 407 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "skip_decode_v32_fallback"); - /*|->skipv32_fallback: */ - /*|->skipv64_fallback: */ - /*| chkeob 16, >1 */ - dasm_put(Dst, 951); - if (16 == 1) { - dasm_put(Dst, 956); - } else { - dasm_put(Dst, 964); - } -# 411 "upb/pb/compile_decoder_x64.dasc" - /*| // With at least 16 bytes left, we can do a branch-less SSE version. */ - /*| movdqu xmm0, [PTR] */ - /*| pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0. */ - /*| not eax */ - /*| bsf eax, eax */ - /*| cmp al, 10 */ - /*| jae ->decode_varint_slow // Error (>10 byte varint). */ - /*| add PTR, rax // bsf result is 0-based, so PTR=end-1, as desired. */ - /*| ret */ - /*| */ - /*|1: */ - /*| // With fewer than 16 bytes, we have to read byte by byte. */ - /*| lea rcx, [PTR + 10] */ - /*| mov rax, PTR // Preserve PTR in case of fallback to slow path. */ - /*| cmp rcx, DATAEND */ - /*| cmova rcx, DATAEND // rcx = MIN(DATAEND, PTR + 10) */ - /*|2: */ - /*| cmp rax, rcx */ - /*| je ->decode_varint_slow */ - /*| test byte [rax], 0x80 */ - /*| jz >3 */ - /*| add rax, 1 */ - /*| jmp <2 */ - /*|3: */ - /*| mov PTR, rax // PTR = varint_end - 1, as desired */ - /*| ret */ - /*| */ - /*| // Returns tag in edx */ - dasm_put(Dst, 980, 10); -# 439 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "decode_unknown_tag_fallback"); - /*|->decode_unknown_tag_fallback: */ - /*| sub rsp, 16 */ - /*|1: */ - /*| cmp PTR, DELIMEND */ - /*| jne >2 */ - /*| add rsp, 16 */ - /*| xor eax, eax */ - /*| ret */ - /*|2: */ - /*| // OPT: Have a medium-fast path before falling back to _slow. */ - /*| mov ARG1_64, DECODER */ - /*| mov ARG2_64, rsp */ - /*| commit_regs */ - /*| callp upb_pbdecoder_decode_varint_slow */ - /*| load_regs */ - dasm_put(Dst, 1053, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure)); -# 455 "upb/pb/compile_decoder_x64.dasc" - /*| cmp eax, 0 */ - /*| jge >3 */ - /*| mov edx, [rsp] // Success; return parsed data. */ - /*| add rsp, 16 */ - /*| ret */ - /*|3: */ - /*| call ->exitjit // Return eax from decode function. */ - /*| jmp <1 */ - /*| */ - /*| // Called for varint >= 1 byte. */ - dasm_put(Dst, 1156, Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); -# 465 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "decode_v32_v64_fallback"); - /*|->decodev32_fallback: */ - /*|->decodev64_fallback: */ - /*| chkeob 10, ->decode_varint_slow */ - dasm_put(Dst, 1194); - if (10 == 1) { - dasm_put(Dst, 1199); - } else { - dasm_put(Dst, 1207); - } -# 469 "upb/pb/compile_decoder_x64.dasc" - /*| // OPT: do something faster than just calling the C version. */ - /*| mov rdi, PTR */ - /*| callp upb_vdecode_fast */ - /*| test rax, rax */ - /*| je ->decode_varint_slow // Unterminated varint. */ - /*| mov PTR, rax */ - /*| sub PTR, 1 */ - /*| mov DECODER->ptr, PTR */ - /*| ret */ - /*| */ - dasm_put(Dst, 1223, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32), 0xfffffffffffffff0UL, Dt2(->ptr)); -# 479 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "decode_varint_slow"); - /*|->decode_varint_slow: */ - /*| // Slow path: end of buffer or error (varint length >= 10). */ - /*| getvalue_slow upb_pbdecoder_decode_varint_slow, 1 */ - dasm_put(Dst, 1268, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs)); -# 483 "upb/pb/compile_decoder_x64.dasc" - /*| */ - /*| // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH}) */ - dasm_put(Dst, 1374, Dt2(->buf), Dt2(->ptr)); -# 485 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "checktag_fallback"); - /*|->checktag_fallback: */ - /*| sub rsp, 8 */ - /*| mov [rsp], rsi // Preserve expected tag. */ - /*|1: */ - /*| mov ARG1_64, DECODER */ - /*| commit_regs */ - /*| mov DECODER->checkpoint, PTR */ - /*| callp upb_pbdecoder_checktag_slow */ - /*| load_regs */ - dasm_put(Dst, 1418, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint), (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs)); -# 495 "upb/pb/compile_decoder_x64.dasc" - /*| cmp eax, 0 */ - /*| jge >2 */ - /*| add rsp, 8 */ - /*| ret */ - /*|2: */ - /*| call ->exitjit */ - /*| mov rsi, [rsp] */ - /*| cmp PTR, DELIMEND */ - /*| jne <1 */ - /*| mov eax, DECODE_EOF */ - /*| add rsp, 8 */ - /*| ret */ - /*| */ - /*| // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found). */ - /*| // Preserves: rcx, rdx */ - /*| // OPT: Could write this in assembly if it's a hotspot. */ - dasm_put(Dst, 1517, Dt2(->buf), DECODE_EOF); -# 511 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "hashlookup"); - /*|->hashlookup: */ - /*| push rcx */ - /*| push rdx */ - /*| sub rsp, 16 */ - /*| mov rdi, rsi */ - /*| mov rsi, rdx */ - /*| mov rdx, rsp */ - /*| callp upb_inttable_lookup */ - /*| add rsp, 16 */ - /*| pop rdx */ - /*| pop rcx */ - /*| test al, al */ - /*| jz >2 // Unknown field. */ - /*| mov rax, [rsp-32] // Value from table. */ - /*| ret */ - /*|2: */ - /*| xor rax, rax */ - /*| not rax */ - /*| ret */ - dasm_put(Dst, 1559, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32), 0xfffffffffffffff0UL); -# 531 "upb/pb/compile_decoder_x64.dasc" -} - -static void jitprimitive(jitcompiler *jc, opcode op, - const upb_handlers *h, upb_selector_t sel) { - typedef enum { V32, V64, F32, F64, X } valtype_t; - static valtype_t types[] = { - X, F64, F32, V64, V64, V32, F64, F32, V64, X, X, X, X, V32, V32, F32, F64, - V32, V64 }; - static char fastpath_bytes[] = { 1, 1, 4, 8 }; - const valtype_t vtype = types[op]; - const int fastbytes = fastpath_bytes[vtype]; - upb_func *handler = gethandler(h, sel); - upb_fieldtype_t ftype; - size_t offset; - int32_t hasbit; - - if (handler) { - /*|1: */ - /*| chkneob fastbytes, >3 */ - dasm_put(Dst, 112); - if (fastbytes == 1) { - dasm_put(Dst, 1628); - } else { - dasm_put(Dst, 1636, fastbytes); - } -# 550 "upb/pb/compile_decoder_x64.dasc" - /*|2: */ - dasm_put(Dst, 1652); -# 551 "upb/pb/compile_decoder_x64.dasc" - switch (vtype) { - case V32: - /*| call ->decodev32_fallback */ - dasm_put(Dst, 1655); -# 554 "upb/pb/compile_decoder_x64.dasc" - break; - case V64: - /*| call ->decodev64_fallback */ - dasm_put(Dst, 1659); -# 557 "upb/pb/compile_decoder_x64.dasc" - break; - case F32: - /*| call ->decodef32_fallback */ - dasm_put(Dst, 1663); -# 560 "upb/pb/compile_decoder_x64.dasc" - break; - case F64: - /*| call ->decodef64_fallback */ - dasm_put(Dst, 1667); -# 563 "upb/pb/compile_decoder_x64.dasc" - break; - case X: break; - } - /*| jmp >4 */ - dasm_put(Dst, 1671); -# 567 "upb/pb/compile_decoder_x64.dasc" - - /* Fast path decode; for when check_bytes bytes are available. */ - /*|3: */ - dasm_put(Dst, 1676); -# 570 "upb/pb/compile_decoder_x64.dasc" - switch (op) { - case OP_PARSE_SFIXED32: - case OP_PARSE_FIXED32: - /*| mov edx, dword [PTR] */ - dasm_put(Dst, 1679); -# 574 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_PARSE_SFIXED64: - case OP_PARSE_FIXED64: - /*| mov rdx, qword [PTR] */ - dasm_put(Dst, 1682); -# 578 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_PARSE_FLOAT: - /*| movss xmm0, dword [PTR] */ - dasm_put(Dst, 1686); -# 581 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_PARSE_DOUBLE: - /*| movsd xmm0, qword [PTR] */ - dasm_put(Dst, 1692); -# 584 "upb/pb/compile_decoder_x64.dasc" - break; - default: - /* Inline one byte of varint decoding. */ - /*| movzx edx, byte [PTR] */ - /*| test dl, dl */ - /*| js <2 // Fallback to slow path for >1 byte varint. */ - dasm_put(Dst, 1698); -# 590 "upb/pb/compile_decoder_x64.dasc" - break; - } - - /* Second-stage decode; used for both fast and slow paths */ - /* (only needed for a few types). */ - /*|4: */ - dasm_put(Dst, 1708); -# 596 "upb/pb/compile_decoder_x64.dasc" - switch (op) { - case OP_PARSE_SINT32: - /* 32-bit zig-zag decode. */ - /*| mov eax, edx */ - /*| shr edx, 1 */ - /*| and eax, 1 */ - /*| neg eax */ - /*| xor edx, eax */ - dasm_put(Dst, 1711); -# 604 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_PARSE_SINT64: - /* 64-bit zig-zag decode. */ - /*| mov rax, rdx */ - /*| shr rdx, 1 */ - /*| and rax, 1 */ - /*| neg rax */ - /*| xor rdx, rax */ - dasm_put(Dst, 1725); -# 612 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_PARSE_BOOL: - /*| test rdx, rdx */ - /*| setne dl */ - dasm_put(Dst, 1744); -# 616 "upb/pb/compile_decoder_x64.dasc" - break; - default: break; - } - - /* Call callback (or specialize if we can). */ - if (upb_msg_getscalarhandlerdata(h, sel, &ftype, &offset, &hasbit)) { - switch (ftype) { - case UPB_TYPE_INT64: - case UPB_TYPE_UINT64: - /*| mov [CLOSURE + offset], rdx */ - dasm_put(Dst, 1751, offset); -# 626 "upb/pb/compile_decoder_x64.dasc" - break; - case UPB_TYPE_INT32: - case UPB_TYPE_UINT32: - case UPB_TYPE_ENUM: - /*| mov [CLOSURE + offset], edx */ - dasm_put(Dst, 1756, offset); -# 631 "upb/pb/compile_decoder_x64.dasc" - break; - case UPB_TYPE_DOUBLE: - /*| movsd qword [CLOSURE + offset], XMMARG1 */ - dasm_put(Dst, 1761, offset); -# 634 "upb/pb/compile_decoder_x64.dasc" - break; - case UPB_TYPE_FLOAT: - /*| movss dword [CLOSURE + offset], XMMARG1 */ - dasm_put(Dst, 1769, offset); -# 637 "upb/pb/compile_decoder_x64.dasc" - break; - case UPB_TYPE_BOOL: - /*| mov [CLOSURE + offset], dl */ - dasm_put(Dst, 1777, offset); -# 640 "upb/pb/compile_decoder_x64.dasc" - break; - case UPB_TYPE_STRING: - case UPB_TYPE_BYTES: - case UPB_TYPE_MESSAGE: - UPB_ASSERT(false); break; - } - /*| sethas CLOSURE, hasbit */ - if (hasbit >= 0) { - dasm_put(Dst, 1782, ((uint32_t)hasbit / 8), (1 << ((uint32_t)hasbit % 8))); - } -# 647 "upb/pb/compile_decoder_x64.dasc" - } else if (handler) { - /*| mov ARG1_64, CLOSURE */ - /*| load_handler_data h, sel */ - dasm_put(Dst, 1788); - { - uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, sel); - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 650 "upb/pb/compile_decoder_x64.dasc" - /*| callp handler */ - dasm_put(Dst, 1793, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32), 0xfffffffffffffff0UL); -# 651 "upb/pb/compile_decoder_x64.dasc" - if (!alwaysok(h, sel)) { - /*| test al, al */ - /*| jnz >5 */ - /*| call ->suspend */ - /*| jmp <1 */ - /*|5: */ - dasm_put(Dst, 1815); -# 657 "upb/pb/compile_decoder_x64.dasc" - } - } - - /* We do this last so that the checkpoint is not advanced past the user's - * data until the callback has returned success. */ - /*| add PTR, fastbytes */ - dasm_put(Dst, 1831, fastbytes); -# 663 "upb/pb/compile_decoder_x64.dasc" - } else { - /* No handler registered for this value, just skip it. */ - /*| chkneob fastbytes, >3 */ - if (fastbytes == 1) { - dasm_put(Dst, 1628); - } else { - dasm_put(Dst, 1636, fastbytes); - } -# 666 "upb/pb/compile_decoder_x64.dasc" - /*|2: */ - dasm_put(Dst, 1652); -# 667 "upb/pb/compile_decoder_x64.dasc" - switch (vtype) { - case V32: - /*| call ->skipv32_fallback */ - dasm_put(Dst, 1836); -# 670 "upb/pb/compile_decoder_x64.dasc" - break; - case V64: - /*| call ->skipv64_fallback */ - dasm_put(Dst, 1840); -# 673 "upb/pb/compile_decoder_x64.dasc" - break; - case F32: - /*| call ->skipf32_fallback */ - dasm_put(Dst, 1844); -# 676 "upb/pb/compile_decoder_x64.dasc" - break; - case F64: - /*| call ->skipf64_fallback */ - dasm_put(Dst, 1848); -# 679 "upb/pb/compile_decoder_x64.dasc" - break; - case X: break; - } - - /* Fast-path skip. */ - /*|3: */ - dasm_put(Dst, 1676); -# 685 "upb/pb/compile_decoder_x64.dasc" - if (vtype == V32 || vtype == V64) { - /*| test byte [PTR], 0x80 */ - /*| jnz <2 */ - dasm_put(Dst, 1852); -# 688 "upb/pb/compile_decoder_x64.dasc" - } - /*| add PTR, fastbytes */ - dasm_put(Dst, 1831, fastbytes); -# 690 "upb/pb/compile_decoder_x64.dasc" - } -} - -static void jitdispatch(jitcompiler *jc, - const upb_pbdecodermethod *method) { - /* Lots of room for tweaking/optimization here. */ - - const upb_inttable *dispatch = &method->dispatch; - bool has_hash_entries = (dispatch->t.count > 0); - - /* Whether any of the fields for this message can have two wire types which - * are both valid (packed & non-packed). - * - * OPT: populate this more precisely; not all messages with hash entries have - * this characteristic. */ - bool has_multi_wiretype = has_hash_entries; - - /*|=>define_jmptarget(jc, &method->dispatch): */ - /*|1: */ - dasm_put(Dst, 1861, define_jmptarget(jc, &method->dispatch)); -# 709 "upb/pb/compile_decoder_x64.dasc" - /* Decode the field tag. */ - /*| mov aword DECODER->checkpoint, PTR */ - /*| chkeob 2, >6 */ - dasm_put(Dst, 308, Dt2(->checkpoint)); - if (2 == 1) { - dasm_put(Dst, 1865); - } else { - dasm_put(Dst, 1873); - } -# 712 "upb/pb/compile_decoder_x64.dasc" - /*| movzx edx, byte [PTR] */ - /*| test dl, dl */ - /*| jns >7 // Jump if first byte has no continuation bit. */ - /*| movzx ecx, byte [PTR + 1] */ - /*| test cl, cl */ - /*| js >6 // Jump if second byte has continuation bit. */ - /*| // Confirmed two-byte varint. */ - /*| shl ecx, 7 */ - /*| and edx, 0x7f */ - /*| or edx, ecx */ - /*| add PTR, 2 */ - /*| jmp >8 */ - /*|6: */ - /*| call ->decode_unknown_tag_fallback */ - /*| test eax, eax // Hit DELIMEND? */ - /*| jnz >8 */ - /*| ret */ - /*|7: */ - /*| add PTR, 1 */ - /*|8: */ - /*| mov ecx, edx */ - /*| shr edx, 3 */ - /*| and cl, 7 */ - dasm_put(Dst, 1889, 1); -# 735 "upb/pb/compile_decoder_x64.dasc" - - /* See comment attached to upb_pbdecodermethod.dispatch for layout of the - * dispatch table. */ - /*|2: */ - /*| cmp edx, dispatch->array_size */ - dasm_put(Dst, 1954, dispatch->array_size); -# 740 "upb/pb/compile_decoder_x64.dasc" - if (has_hash_entries) { - /*| jae >7 */ - dasm_put(Dst, 1961); -# 742 "upb/pb/compile_decoder_x64.dasc" - } else { - /*| jae >5 */ - dasm_put(Dst, 1966); -# 744 "upb/pb/compile_decoder_x64.dasc" - } - /*| // OPT: Compact the lookup arr into 32-bit entries. */ - if ((uintptr_t)dispatch->array > 0x7fffffff) { - /*| mov64 rax, (uintptr_t)dispatch->array */ - /*| mov rax, qword [rax + rdx * 8] */ - dasm_put(Dst, 1971, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32)); -# 749 "upb/pb/compile_decoder_x64.dasc" - } else { - /*| mov rax, qword [rdx * 8 + dispatch->array] */ - dasm_put(Dst, 1980, dispatch->array); -# 751 "upb/pb/compile_decoder_x64.dasc" - } - /*|3: */ - /*| // We take advantage of the fact that non-present entries are stored */ - /*| // as -1, which will result in wire types that will never match. */ - /*| cmp al, cl */ - dasm_put(Dst, 1986); -# 756 "upb/pb/compile_decoder_x64.dasc" - if (has_multi_wiretype) { - /*| jne >6 */ - dasm_put(Dst, 1991); -# 758 "upb/pb/compile_decoder_x64.dasc" - } else { - /*| jne >5 */ - dasm_put(Dst, 1996); -# 760 "upb/pb/compile_decoder_x64.dasc" - } - /*| shr rax, 16 */ - /*| */ - /*| // Load the machine code address from the table entry. */ - /*| // The table entry is relative to the dispatch->array jmptarget */ - /*| // (patchdispatch() took care of this) which is the same as */ - /*| // local label "4". The "lea" is really just trying to do */ - /*| // lea rax, [>4 + rax] */ - /*| // */ - /*| // But we can't write that directly for some reason, so we use */ - /*| // rdx as a temporary. */ - /*| lea rdx, [>4] */ - /*|=>define_jmptarget(jc, dispatch->array): */ - /*|4: */ - /*| add rax, rdx */ - /*| ret */ - /*| */ - /*|5: */ - /*| // Field isn't in our table. */ - /*| */ - /*| // For pushing unknown fields to the unknown field handler. */ - /*| mov64 rax, (uintptr_t)method->dest_handlers_ */ - /*| mov FRAME->sink.handlers, rax */ - /*| */ - /*| call ->parse_unknown */ - /*| test eax, eax // ENDGROUP? */ - /*| jz <1 */ - /*| lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG. */ - /*| ret */ - dasm_put(Dst, 2001, define_jmptarget(jc, dispatch->array), (unsigned int)((uintptr_t)method->dest_handlers_), (unsigned int)(((uintptr_t)method->dest_handlers_)>>32), Dt1(->sink.handlers)); -# 789 "upb/pb/compile_decoder_x64.dasc" - - if (has_multi_wiretype) { - /*|6: */ - /*| // Primary wire type didn't match, check secondary wire type. */ - /*| cmp ah, cl */ - /*| jne <5 */ - /*| // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER. */ - /*| add rdx, UPB_MAX_FIELDNUMBER */ - /*| // This key will never be in the array part, so do a hash lookup. */ - dasm_put(Dst, 2043, UPB_MAX_FIELDNUMBER); -# 798 "upb/pb/compile_decoder_x64.dasc" - UPB_ASSERT(has_hash_entries); - /*| ld64 dispatch */ - { - uintptr_t v = (uintptr_t)dispatch; - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 800 "upb/pb/compile_decoder_x64.dasc" - /*| jmp ->hashlookup // Tail call. */ - dasm_put(Dst, 2056); -# 801 "upb/pb/compile_decoder_x64.dasc" - } - - if (has_hash_entries) { - /*|7: */ - /*| // Hash table lookup. */ - /*| ld64 dispatch */ - dasm_put(Dst, 2061); - { - uintptr_t v = (uintptr_t)dispatch; - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 807 "upb/pb/compile_decoder_x64.dasc" - /*| call ->hashlookup */ - /*| jmp <3 */ - dasm_put(Dst, 2064); -# 809 "upb/pb/compile_decoder_x64.dasc" - } -} - -static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, - const upb_pbdecodermethod *method) { - /* Internally we parse unknown fields; if this runs us into DELIMEND we jump - * to the corresponding DELIMEND target (either msg end or repeated field - * end), which we find from the OP_CHECKDELIM which must have necessarily - * preceded us. */ - uint32_t last_instruction = *(jc->pc - 2); - int last_arg = (int32_t)last_instruction >> 8; - uint32_t *delimend = (jc->pc - 1) + last_arg; - const size_t ptr_words = sizeof(void*) / sizeof(uint32_t); - - UPB_ASSERT((last_instruction & 0xff) == OP_CHECKDELIM); - - if (getop(*(jc->pc - 1)) == OP_TAGN) { - jc->pc += ptr_words; - } - - /*| chkneob n, >1 */ - if (n == 1) { - dasm_put(Dst, 2072); - } else { - dasm_put(Dst, 2080, n); - } -# 830 "upb/pb/compile_decoder_x64.dasc" - - /*| // OPT: this is way too much fallback code to put here. */ - /*| // Reduce and/or move to a separate section to make better icache usage. */ - /*| ld64 tag */ - { - uintptr_t v = (uintptr_t)tag; - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 834 "upb/pb/compile_decoder_x64.dasc" - /*| call ->checktag_fallback */ - /*| cmp eax, DECODE_MISMATCH */ - /*| je >3 */ - /*| cmp eax, DECODE_EOF */ - /*| je =>jmptarget(jc, delimend) */ - /*| jmp >5 */ - dasm_put(Dst, 2096, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend)); -# 840 "upb/pb/compile_decoder_x64.dasc" - - /*|1: */ - dasm_put(Dst, 112); -# 842 "upb/pb/compile_decoder_x64.dasc" - switch (n) { - case 1: - /*| cmp byte [PTR], tag */ - dasm_put(Dst, 2119, tag); -# 845 "upb/pb/compile_decoder_x64.dasc" - break; - case 2: - /*| cmp word [PTR], tag */ - dasm_put(Dst, 2123, tag); -# 848 "upb/pb/compile_decoder_x64.dasc" - break; - case 3: - /*| // OPT: Slightly more efficient code, but depends on an extra byte. */ - /*| // mov eax, dword [PTR] */ - /*| // shl eax, 8 */ - /*| // cmp eax, tag << 8 */ - /*| cmp word [PTR], (tag & 0xffff) */ - /*| jne >2 */ - /*| cmp byte [PTR + 2], (tag >> 16) */ - /*|2: */ - dasm_put(Dst, 2128, (tag & 0xffff), 2, (tag >> 16)); -# 858 "upb/pb/compile_decoder_x64.dasc" - break; - case 4: - /*| cmp dword [PTR], tag */ - dasm_put(Dst, 2143, tag); -# 861 "upb/pb/compile_decoder_x64.dasc" - break; - case 5: - /*| cmp dword [PTR], (tag & 0xffffffff) */ - /*| jne >3 */ - /*| cmp byte [PTR + 4], (tag >> 32) */ - dasm_put(Dst, 2147, (tag & 0xffffffff), 4, (tag >> 32)); -# 866 "upb/pb/compile_decoder_x64.dasc" - } - /*| je >4 */ - /*|3: */ - dasm_put(Dst, 2159); -# 869 "upb/pb/compile_decoder_x64.dasc" - if (ofs == 0) { - /*| call =>jmptarget(jc, &method->dispatch) */ - /*| test rax, rax */ - /*| jz =>jmptarget(jc, delimend) */ - /*| jmp rax */ - dasm_put(Dst, 2166, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend)); -# 874 "upb/pb/compile_decoder_x64.dasc" - } else { - /*| jmp =>jmptarget(jc, jc->pc + ofs) */ - dasm_put(Dst, 2178, jmptarget(jc, jc->pc + ofs)); -# 876 "upb/pb/compile_decoder_x64.dasc" - } - /*|4: */ - /*| add PTR, n */ - /*|5: */ - dasm_put(Dst, 2182, n); -# 880 "upb/pb/compile_decoder_x64.dasc" -} - -/* Compile the bytecode to x64. */ -static void jitbytecode(jitcompiler *jc) { - upb_pbdecodermethod *method = NULL; - const upb_handlers *h = NULL; - for (jc->pc = jc->group->bytecode; jc->pc < jc->group->bytecode_end; ) { - int32_t instr = *jc->pc; - opcode op = instr & 0xff; - uint32_t arg = instr >> 8; - int32_t longofs = arg; - - if (op != OP_SETDISPATCH) { - /* Skipped for SETDISPATCH because it defines its own asmlabel for the - * dispatch code it emits. */ - asmlabel(jc, "0x%lx.%s", pcofs(jc), upb_pbdecoder_getopname(op)); - - /* Skipped for SETDISPATCH because it should point at the function - * prologue, not the dispatch function that is emitted first. - * TODO: optimize this to only define pclabels that are actually used. */ - /*|=>define_jmptarget(jc, jc->pc): */ - dasm_put(Dst, 0, define_jmptarget(jc, jc->pc)); -# 901 "upb/pb/compile_decoder_x64.dasc" - } - - jc->pc++; - - switch (op) { - case OP_STARTMSG: { - upb_func *startmsg = gethandler(h, UPB_STARTMSG_SELECTOR); - if (startmsg) { - /* bool startmsg(void *closure, const void *hd) */ - /*|1: */ - /*| mov ARG1_64, CLOSURE */ - /*| load_handler_data h, UPB_STARTMSG_SELECTOR */ - dasm_put(Dst, 2191); - { - uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_STARTMSG_SELECTOR); - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 913 "upb/pb/compile_decoder_x64.dasc" - /*| callp startmsg */ - dasm_put(Dst, 1793, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32), 0xfffffffffffffff0UL); -# 914 "upb/pb/compile_decoder_x64.dasc" - if (!alwaysok(h, UPB_STARTMSG_SELECTOR)) { - /*| test al, al */ - /*| jnz >2 */ - /*| call ->suspend */ - /*| jmp <1 */ - /*|2: */ - dasm_put(Dst, 2198); -# 920 "upb/pb/compile_decoder_x64.dasc" - } - } else { - /*| nop */ - dasm_put(Dst, 2214); -# 923 "upb/pb/compile_decoder_x64.dasc" - } - break; - } - case OP_ENDMSG: { - upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR); - /*|9: */ - dasm_put(Dst, 2216); -# 929 "upb/pb/compile_decoder_x64.dasc" - if (endmsg) { - /* bool endmsg(void *closure, const void *hd, upb_status *status) */ - /*| mov ARG1_64, CLOSURE */ - /*| load_handler_data h, UPB_ENDMSG_SELECTOR */ - dasm_put(Dst, 1788); - { - uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_ENDMSG_SELECTOR); - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 933 "upb/pb/compile_decoder_x64.dasc" - /*| mov ARG3_64, DECODER->status */ - /*| callp endmsg */ - dasm_put(Dst, 2219, Dt2(->status), (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32), 0xfffffffffffffff0UL); -# 935 "upb/pb/compile_decoder_x64.dasc" - } - break; - } - case OP_SETDISPATCH: { - uint32_t *op_pc = jc->pc - 1; - const char *msgname; - upb_inttable *dispatch; - - /* Load info for new method. */ - memcpy(&dispatch, jc->pc, sizeof(void*)); - jc->pc += sizeof(void*) / sizeof(uint32_t); - /* The OP_SETDISPATCH bytecode contains a pointer that is - * &method->dispatch; we want to go backwards and recover method. */ - method = - (void*)((char*)dispatch - offsetof(upb_pbdecodermethod, dispatch)); - /* May be NULL, in which case no handlers for this message will be found. - * OPT: we should do better by completely skipping the message in this - * case instead of parsing it field by field. We should also do the skip - * in the containing message's code. */ - h = method->dest_handlers_; - msgname = upb_msgdef_fullname(upb_handlers_msgdef(h)); - - /* Emit dispatch code for new method. */ - asmlabel(jc, "0x%lx.dispatch.%s", pcofs(jc), msgname); - jitdispatch(jc, method); - - /* Emit function prologue for new method. */ - asmlabel(jc, "0x%lx.parse.%s", pcofs(jc), msgname); - /*|=>define_jmptarget(jc, op_pc): */ - /*|=>define_jmptarget(jc, method): */ - /*| sub rsp, 8 */ - dasm_put(Dst, 2245, define_jmptarget(jc, op_pc), define_jmptarget(jc, method)); -# 966 "upb/pb/compile_decoder_x64.dasc" - - break; - } - case OP_PARSE_DOUBLE: - case OP_PARSE_FLOAT: - case OP_PARSE_INT64: - case OP_PARSE_UINT64: - case OP_PARSE_INT32: - case OP_PARSE_FIXED64: - case OP_PARSE_FIXED32: - case OP_PARSE_BOOL: - case OP_PARSE_UINT32: - case OP_PARSE_SFIXED32: - case OP_PARSE_SFIXED64: - case OP_PARSE_SINT32: - case OP_PARSE_SINT64: - jitprimitive(jc, op, h, arg); - break; - case OP_STARTSEQ: - case OP_STARTSUBMSG: - case OP_STARTSTR: { - upb_func *start = gethandler(h, arg); - if (start) { - /* void *startseq(void *closure, const void *hd) - * void *startsubmsg(void *closure, const void *hd) - * void *startstr(void *closure, const void *hd, size_t size_hint) */ - /*|1: */ - /*| mov ARG1_64, CLOSURE */ - /*| load_handler_data h, arg */ - dasm_put(Dst, 2191); - { - uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 995 "upb/pb/compile_decoder_x64.dasc" - if (op == OP_STARTSTR) { - /*| mov ARG3_64, DELIMEND */ - /*| sub ARG3_64, PTR */ - dasm_put(Dst, 2253); -# 998 "upb/pb/compile_decoder_x64.dasc" - } - /*| callp start */ - dasm_put(Dst, 1793, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32), 0xfffffffffffffff0UL); -# 1000 "upb/pb/compile_decoder_x64.dasc" - if (!alwaysok(h, arg)) { - /*| test rax, rax */ - /*| jnz >2 */ - /*| call ->suspend */ - /*| jmp <1 */ - /*|2: */ - dasm_put(Dst, 2261); -# 1006 "upb/pb/compile_decoder_x64.dasc" - } - /*| mov CLOSURE, rax */ - dasm_put(Dst, 2278); -# 1008 "upb/pb/compile_decoder_x64.dasc" - } else { - /* TODO: nop is only required because of asmlabel(). */ - /*| nop */ - dasm_put(Dst, 2214); -# 1011 "upb/pb/compile_decoder_x64.dasc" - } - break; - } - case OP_ENDSEQ: - case OP_ENDSUBMSG: - case OP_ENDSTR: { - upb_func *end = gethandler(h, arg); - if (end) { - /* bool endseq(void *closure, const void *hd) - * bool endsubmsg(void *closure, const void *hd) - * bool endstr(void *closure, const void *hd) */ - /*|1: */ - /*| mov ARG1_64, CLOSURE */ - /*| load_handler_data h, arg */ - dasm_put(Dst, 2191); - { - uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 1025 "upb/pb/compile_decoder_x64.dasc" - /*| callp end */ - dasm_put(Dst, 1793, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32), 0xfffffffffffffff0UL); -# 1026 "upb/pb/compile_decoder_x64.dasc" - if (!alwaysok(h, arg)) { - /*| test al, al */ - /*| jnz >2 */ - /*| call ->suspend */ - /*| jmp <1 */ - /*|2: */ - dasm_put(Dst, 2198); -# 1032 "upb/pb/compile_decoder_x64.dasc" - } - } else { - /* TODO: nop is only required because of asmlabel(). */ - /*| nop */ - dasm_put(Dst, 2214); -# 1036 "upb/pb/compile_decoder_x64.dasc" - } - break; - } - case OP_STRING: { - upb_func *str = gethandler(h, arg); - /*| cmp PTR, DELIMEND */ - /*| je >4 */ - /*|1: */ - /*| cmp PTR, DATAEND */ - /*| jne >2 */ - /*| call ->suspend */ - /*| jmp <1 */ - /*|2: */ - dasm_put(Dst, 2282); -# 1049 "upb/pb/compile_decoder_x64.dasc" - if (str) { - /* size_t str(void *closure, const void *hd, const char *str, - * size_t n) */ - /*| mov ARG1_64, CLOSURE */ - /*| load_handler_data h, arg */ - dasm_put(Dst, 1788); - { - uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); - if (v > 0xffffffff) { - dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); - } else if (v) { - dasm_put(Dst, 451, v); - } else { - dasm_put(Dst, 454); - } - } -# 1054 "upb/pb/compile_decoder_x64.dasc" - /*| mov ARG3_64, PTR */ - /*| mov ARG4_64, DATAEND */ - /*| sub ARG4_64, PTR */ - /*| mov ARG5_64, qword DECODER->handle */ - /*| callp str */ - /*| add PTR, rax */ - dasm_put(Dst, 2309, Dt2(->handle), (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32), 0xfffffffffffffff0UL); -# 1060 "upb/pb/compile_decoder_x64.dasc" - if (!alwaysok(h, arg)) { - /*| cmp PTR, DATAEND */ - /*| je >3 */ - /*| call ->strret_fallback */ - /*|3: */ - dasm_put(Dst, 2347); -# 1065 "upb/pb/compile_decoder_x64.dasc" - } - } else { - /*| mov PTR, DATAEND */ - dasm_put(Dst, 2360); -# 1068 "upb/pb/compile_decoder_x64.dasc" - } - /*| cmp PTR, DELIMEND */ - /*| jne <1 */ - /*|4: */ - dasm_put(Dst, 2364); -# 1072 "upb/pb/compile_decoder_x64.dasc" - break; - } - case OP_PUSHTAGDELIM: - /*| mov FRAME->sink.closure, CLOSURE */ - /*| // This shouldn't need to be read, because tag-delimited fields */ - /*| // shouldn't have an OP_SETDELIM after them. But for the moment */ - /*| // non-packed repeated fields do OP_SETDELIM so they can share more */ - /*| // code with the packed code-path. If this is changed later, this */ - /*| // store can be removed. */ - /*| mov qword FRAME->end_ofs, 0 */ - /*| cmp FRAME, DECODER->limit */ - /*| je ->err */ - /*| add FRAME, sizeof(upb_pbdecoder_frame) */ - /*| mov dword FRAME->groupnum, arg */ - dasm_put(Dst, 2375, Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->limit), sizeof(upb_pbdecoder_frame), Dt1(->groupnum), arg); -# 1086 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_PUSHLENDELIM: - /*| call ->pushlendelim */ - dasm_put(Dst, 2405); -# 1089 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_POP: - /*| sub FRAME, sizeof(upb_pbdecoder_frame) */ - /*| mov CLOSURE, FRAME->sink.closure */ - dasm_put(Dst, 2409, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure)); -# 1093 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_SETDELIM: - /* OPT: experiment with testing vs old offset to optimize away. */ - /*| mov DATAEND, DECODER->end */ - /*| add DELIMEND, FRAME->end_ofs */ - /*| cmp DELIMEND, DECODER->buf */ - /*| jb >1 */ - /*| cmp DELIMEND, DATAEND */ - /*| ja >1 // OPT: try cmov. */ - /*| mov DATAEND, DELIMEND */ - /*|1: */ - dasm_put(Dst, 2419, Dt2(->end), Dt1(->end_ofs), Dt2(->buf)); -# 1104 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_SETBIGGROUPNUM: - /*| mov dword FRAME->groupnum, *jc->pc++ */ - dasm_put(Dst, 2399, Dt1(->groupnum), *jc->pc++); -# 1107 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_CHECKDELIM: - /*| cmp DELIMEND, PTR */ - /*| je =>jmptarget(jc, jc->pc + longofs) */ - dasm_put(Dst, 2449, jmptarget(jc, jc->pc + longofs)); -# 1111 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_CALL: - /*| call =>jmptarget(jc, jc->pc + longofs) */ - dasm_put(Dst, 2456, jmptarget(jc, jc->pc + longofs)); -# 1114 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_BRANCH: - /*| jmp =>jmptarget(jc, jc->pc + longofs); */ - dasm_put(Dst, 2178, jmptarget(jc, jc->pc + longofs)); -# 1117 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_RET: - /*|9: */ - /*| add rsp, 8 */ - /*| ret */ - dasm_put(Dst, 2459); -# 1122 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_TAG1: - jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method); - break; - case OP_TAG2: - jittag(jc, (arg >> 8) & 0xffff, 2, (int8_t)arg, method); - break; - case OP_TAGN: { - uint64_t tag; - memcpy(&tag, jc->pc, 8); - jittag(jc, tag, arg >> 8, (int8_t)arg, method); - break; - } - case OP_DISPATCH: - /*| call =>jmptarget(jc, &method->dispatch) */ - dasm_put(Dst, 2456, jmptarget(jc, &method->dispatch)); -# 1137 "upb/pb/compile_decoder_x64.dasc" - break; - case OP_HALT: - UPB_ASSERT(false); - } - } - - asmlabel(jc, "eof"); - /*| nop */ - dasm_put(Dst, 2214); -# 1145 "upb/pb/compile_decoder_x64.dasc" -} diff --git a/upb/pb/decoder.c b/upb/pb/decoder.c index 0cae05b..2384312 100644 --- a/upb/pb/decoder.c +++ b/upb/pb/decoder.c @@ -99,9 +99,7 @@ static bool in_residual_buf(const upb_pbdecoder *d, const char *p); * benchmarks. */ static void seterr(upb_pbdecoder *d, const char *msg) { - upb_status status = UPB_STATUS_INIT; - upb_status_seterrmsg(&status, msg); - upb_env_reporterror(d->env, &status); + upb_status_seterrmsg(d->status, msg); } void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg) { @@ -596,7 +594,7 @@ have_tag: if (d->top->groupnum >= 0) { /* TODO: More code needed for handling unknown groups. */ - upb_sink_putunknown(&d->top->sink, d->checkpoint, d->ptr - d->checkpoint); + upb_sink_putunknown(d->top->sink, d->checkpoint, d->ptr - d->checkpoint); return DECODE_OK; } @@ -690,7 +688,7 @@ size_t run_decoder_vm(upb_pbdecoder *d, const mgroup *group, VMCASE(OP_PARSE_ ## type, { \ ctype val; \ CHECK_RETURN(decode_ ## wt(d, &val)); \ - upb_sink_put ## name(&d->top->sink, arg, (convfunc)(val)); \ + upb_sink_put ## name(d->top->sink, arg, (convfunc)(val)); \ }) while(1) { @@ -742,36 +740,36 @@ size_t run_decoder_vm(upb_pbdecoder *d, const mgroup *group, d->pc += sizeof(void*) / sizeof(uint32_t); ) VMCASE(OP_STARTMSG, - CHECK_SUSPEND(upb_sink_startmsg(&d->top->sink)); + CHECK_SUSPEND(upb_sink_startmsg(d->top->sink)); ) VMCASE(OP_ENDMSG, - CHECK_SUSPEND(upb_sink_endmsg(&d->top->sink, d->status)); + CHECK_SUSPEND(upb_sink_endmsg(d->top->sink, d->status)); ) VMCASE(OP_STARTSEQ, upb_pbdecoder_frame *outer = outer_frame(d); - CHECK_SUSPEND(upb_sink_startseq(&outer->sink, arg, &d->top->sink)); + CHECK_SUSPEND(upb_sink_startseq(outer->sink, arg, &d->top->sink)); ) VMCASE(OP_ENDSEQ, - CHECK_SUSPEND(upb_sink_endseq(&d->top->sink, arg)); + CHECK_SUSPEND(upb_sink_endseq(d->top->sink, arg)); ) VMCASE(OP_STARTSUBMSG, upb_pbdecoder_frame *outer = outer_frame(d); - CHECK_SUSPEND(upb_sink_startsubmsg(&outer->sink, arg, &d->top->sink)); + CHECK_SUSPEND(upb_sink_startsubmsg(outer->sink, arg, &d->top->sink)); ) VMCASE(OP_ENDSUBMSG, - CHECK_SUSPEND(upb_sink_endsubmsg(&d->top->sink, arg)); + CHECK_SUSPEND(upb_sink_endsubmsg(d->top->sink, arg)); ) VMCASE(OP_STARTSTR, uint32_t len = delim_remaining(d); upb_pbdecoder_frame *outer = outer_frame(d); - CHECK_SUSPEND(upb_sink_startstr(&outer->sink, arg, len, &d->top->sink)); + CHECK_SUSPEND(upb_sink_startstr(outer->sink, arg, len, &d->top->sink)); if (len == 0) { d->pc++; /* Skip OP_STRING. */ } ) VMCASE(OP_STRING, uint32_t len = curbufleft(d); - size_t n = upb_sink_putstring(&d->top->sink, arg, d->ptr, len, handle); + size_t n = upb_sink_putstring(d->top->sink, arg, d->ptr, len, handle); if (n > len) { if (n > delim_remaining(d)) { seterr(d, "Tried to skip past end of string."); @@ -792,7 +790,7 @@ size_t run_decoder_vm(upb_pbdecoder *d, const mgroup *group, } ) VMCASE(OP_ENDSTR, - CHECK_SUSPEND(upb_sink_endstr(&d->top->sink, arg)); + CHECK_SUSPEND(upb_sink_endstr(d->top->sink, arg)); ) VMCASE(OP_PUSHTAGDELIM, CHECK_SUSPEND(pushtagdelim(d, arg)); @@ -992,40 +990,39 @@ void upb_pbdecoder_reset(upb_pbdecoder *d) { d->residual_end = d->residual; } -upb_pbdecoder *upb_pbdecoder_create(upb_env *e, const upb_pbdecodermethod *m, - upb_sink *sink) { +upb_pbdecoder *upb_pbdecoder_create(upb_arena *a, const upb_pbdecodermethod *m, + upb_sink sink, upb_status *status) { const size_t default_max_nesting = 64; #ifndef NDEBUG - size_t size_before = upb_env_bytesallocated(e); + size_t size_before = upb_arena_bytesallocated(a); #endif - upb_pbdecoder *d = upb_env_malloc(e, sizeof(upb_pbdecoder)); + upb_pbdecoder *d = upb_arena_malloc(a, sizeof(upb_pbdecoder)); if (!d) return NULL; d->method_ = m; - d->callstack = upb_env_malloc(e, callstacksize(d, default_max_nesting)); - d->stack = upb_env_malloc(e, stacksize(d, default_max_nesting)); + d->callstack = upb_arena_malloc(a, callstacksize(d, default_max_nesting)); + d->stack = upb_arena_malloc(a, stacksize(d, default_max_nesting)); if (!d->stack || !d->callstack) { return NULL; } - d->env = e; + d->arena = a; d->limit = d->stack + default_max_nesting - 1; d->stack_size = default_max_nesting; - d->status = NULL; + d->status = status; upb_pbdecoder_reset(d); upb_bytessink_reset(&d->input_, &m->input_handler_, d); - UPB_ASSERT(sink); if (d->method_->dest_handlers_) { - if (sink->handlers != d->method_->dest_handlers_) + if (sink.handlers != d->method_->dest_handlers_) return NULL; } - upb_sink_reset(&d->top->sink, sink->handlers, sink->closure); + d->top->sink = sink; /* If this fails, increase the value in decoder.h. */ - UPB_ASSERT_DEBUGVAR(upb_env_bytesallocated(e) - size_before <= + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(a) - size_before <= UPB_PB_DECODER_SIZE); return d; } @@ -1038,8 +1035,8 @@ const upb_pbdecodermethod *upb_pbdecoder_method(const upb_pbdecoder *d) { return d->method_; } -upb_bytessink *upb_pbdecoder_input(upb_pbdecoder *d) { - return &d->input_; +upb_bytessink upb_pbdecoder_input(upb_pbdecoder *d) { + return d->input_; } size_t upb_pbdecoder_maxnesting(const upb_pbdecoder *d) { @@ -1058,7 +1055,7 @@ bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max) { /* Need to reallocate stack and callstack to accommodate. */ size_t old_size = stacksize(d, d->stack_size); size_t new_size = stacksize(d, max); - void *p = upb_env_realloc(d->env, d->stack, old_size, new_size); + void *p = upb_arena_realloc(d->arena, d->stack, old_size, new_size); if (!p) { return false; } @@ -1066,7 +1063,7 @@ bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max) { old_size = callstacksize(d, d->stack_size); new_size = callstacksize(d, max); - p = upb_env_realloc(d->env, d->callstack, old_size, new_size); + p = upb_arena_realloc(d->arena, d->callstack, old_size, new_size); if (!p) { return false; } diff --git a/upb/pb/decoder.h b/upb/pb/decoder.h index 7c1877a..709db49 100644 --- a/upb/pb/decoder.h +++ b/upb/pb/decoder.h @@ -21,20 +21,13 @@ namespace upb { namespace pb { class CodeCache; -class Decoder; -class DecoderMethod; +class DecoderPtr; +class DecoderMethodPtr; class DecoderMethodOptions; } /* namespace pb */ } /* namespace upb */ #endif -UPB_DECLARE_TYPE(upb::pb::CodeCache, upb_pbcodecache) -UPB_DECLARE_TYPE(upb::pb::Decoder, upb_pbdecoder) -UPB_DECLARE_TYPE(upb::pb::DecoderMethodOptions, upb_pbdecodermethodopts) - -UPB_DECLARE_DERIVED_TYPE(upb::pb::DecoderMethod, upb::RefCounted, - upb_pbdecodermethod, upb_refcounted) - /* The maximum number of bytes we are required to buffer internally between * calls to the decoder. The value is 14: a 5 byte unknown tag plus ten-byte * varint, less one because we are buffering an incomplete value. @@ -42,83 +35,111 @@ UPB_DECLARE_DERIVED_TYPE(upb::pb::DecoderMethod, upb::RefCounted, * Should only be used by unit tests. */ #define UPB_DECODER_MAX_RESIDUAL_BYTES 14 -#ifdef __cplusplus +/* upb_pbdecodermethod ********************************************************/ -/* The parameters one uses to construct a DecoderMethod. - * TODO(haberman): move allowjit here? Seems more convenient for users. - * TODO(haberman): move this to be heap allocated for ABI stability. */ -class upb::pb::DecoderMethodOptions { - public: - /* Parameter represents the destination handlers that this method will push - * to. */ - explicit DecoderMethodOptions(const Handlers* dest_handlers); +struct upb_pbdecodermethod; +typedef struct upb_pbdecodermethod upb_pbdecodermethod; - /* Should the decoder push submessages to lazy handlers for fields that have - * them? The caller should set this iff the lazy handlers expect data that is - * in protobuf binary format and the caller wishes to lazy parse it. */ - void set_lazy(bool lazy); -#else -struct upb_pbdecodermethodopts { +#ifdef __cplusplus +extern "C" { #endif - const upb_handlers *handlers; - bool lazy; -}; + +const upb_handlers *upb_pbdecodermethod_desthandlers( + const upb_pbdecodermethod *m); +const upb_byteshandler *upb_pbdecodermethod_inputhandler( + const upb_pbdecodermethod *m); +bool upb_pbdecodermethod_isnative(const upb_pbdecodermethod *m); #ifdef __cplusplus +} /* extern "C" */ /* Represents the code to parse a protobuf according to a destination * Handlers. */ -class upb::pb::DecoderMethod { +class upb::pb::DecoderMethodPtr { public: - /* Include base methods from upb::ReferenceCounted. */ - UPB_REFCOUNTED_CPPMETHODS + DecoderMethodPtr() : ptr_(nullptr) {} + DecoderMethodPtr(const upb_pbdecodermethod* ptr) : ptr_(ptr) {} + + const upb_pbdecodermethod* ptr() { return ptr_; } /* The destination handlers that are statically bound to this method. * This method is only capable of outputting to a sink that uses these * handlers. */ - const Handlers* dest_handlers() const; + const Handlers *dest_handlers() const { + return upb_pbdecodermethod_desthandlers(ptr_); + } /* The input handlers for this decoder method. */ - const BytesHandler* input_handler() const; + const BytesHandler* input_handler() const { + return upb_pbdecodermethod_inputhandler(ptr_); + } /* Whether this method is native. */ - bool is_native() const; - - /* Convenience method for generating a DecoderMethod without explicitly - * creating a CodeCache. */ - static reffed_ptr<const DecoderMethod> New(const DecoderMethodOptions& opts); + bool is_native() const { + return upb_pbdecodermethod_isnative(ptr_); + } private: - UPB_DISALLOW_POD_OPS(DecoderMethod, upb::pb::DecoderMethod) + const upb_pbdecodermethod* ptr_; }; #endif +/* upb_pbdecoder **************************************************************/ + /* Preallocation hint: decoder won't allocate more bytes than this when first * constructed. This hint may be an overestimate for some build configurations. * But if the decoder library is upgraded without recompiling the application, * it may be an underestimate. */ #define UPB_PB_DECODER_SIZE 4416 +struct upb_pbdecoder; +typedef struct upb_pbdecoder upb_pbdecoder; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_pbdecoder *upb_pbdecoder_create(upb_arena *arena, + const upb_pbdecodermethod *method, + upb_sink output, upb_status *status); +const upb_pbdecodermethod *upb_pbdecoder_method(const upb_pbdecoder *d); +upb_bytessink upb_pbdecoder_input(upb_pbdecoder *d); +uint64_t upb_pbdecoder_bytesparsed(const upb_pbdecoder *d); +size_t upb_pbdecoder_maxnesting(const upb_pbdecoder *d); +bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max); +void upb_pbdecoder_reset(upb_pbdecoder *d); + #ifdef __cplusplus +} /* extern "C" */ /* A Decoder receives binary protobuf data on its input sink and pushes the * decoded data to its output sink. */ -class upb::pb::Decoder { +class upb::pb::DecoderPtr { public: + DecoderPtr() : ptr_(nullptr) {} + DecoderPtr(upb_pbdecoder* ptr) : ptr_(ptr) {} + + upb_pbdecoder* ptr() { return ptr_; } + /* Constructs a decoder instance for the given method, which must outlive this * decoder. Any errors during parsing will be set on the given status, which * must also outlive this decoder. * * The sink must match the given method. */ - static Decoder* Create(Environment* env, const DecoderMethod* method, - Sink* output); + static DecoderPtr Create(Arena *arena, DecoderMethodPtr method, + upb::Sink output, Status *status) { + return DecoderPtr(upb_pbdecoder_create(arena->ptr(), method.ptr(), + output.sink(), status->ptr())); + } /* Returns the DecoderMethod this decoder is parsing from. */ - const DecoderMethod* method() const; + const DecoderMethodPtr method() const { + return DecoderMethodPtr(upb_pbdecoder_method(ptr_)); + } /* The sink on which this decoder receives input. */ - BytesSink* input(); + BytesSink input() { return BytesSink(upb_pbdecoder_input(ptr())); } /* Returns number of bytes successfully parsed. * @@ -127,7 +148,7 @@ class upb::pb::Decoder { * * This value may not be up-to-date when called from inside a parsing * callback. */ - uint64_t BytesParsed() const; + uint64_t BytesParsed() { return upb_pbdecoder_bytesparsed(ptr()); } /* Gets/sets the parsing nexting limit. If the total number of nested * submessages and repeated fields hits this limit, parsing will fail. This @@ -136,31 +157,55 @@ class upb::pb::Decoder { * * Setting the limit will fail if the parser is currently suspended at a depth * greater than this, or if memory allocation of the stack fails. */ - size_t max_nesting() const; - bool set_max_nesting(size_t max); + size_t max_nesting() { return upb_pbdecoder_maxnesting(ptr()); } + bool set_max_nesting(size_t max) { return upb_pbdecoder_maxnesting(ptr()); } - void Reset(); + void Reset() { upb_pbdecoder_reset(ptr()); } static const size_t kSize = UPB_PB_DECODER_SIZE; private: - UPB_DISALLOW_POD_OPS(Decoder, upb::pb::Decoder) + upb_pbdecoder *ptr_; }; #endif /* __cplusplus */ +/* upb_pbcodecache ************************************************************/ + +/* Lazily builds and caches decoder methods that will push data to the given + * handlers. The destination handlercache must outlive this object. */ + +struct upb_pbcodecache; +typedef struct upb_pbcodecache upb_pbcodecache; + +#ifdef __cplusplus +extern "C" { +#endif + +upb_pbcodecache *upb_pbcodecache_new(upb_handlercache *dest); +void upb_pbcodecache_free(upb_pbcodecache *c); +bool upb_pbcodecache_allowjit(const upb_pbcodecache *c); +void upb_pbcodecache_setallowjit(upb_pbcodecache *c, bool allow); +void upb_pbcodecache_setlazy(upb_pbcodecache *c, bool lazy); +const upb_pbdecodermethod *upb_pbcodecache_get(upb_pbcodecache *c, + const upb_msgdef *md); + #ifdef __cplusplus +} /* extern "C" */ /* A class for caching protobuf processing code, whether bytecode for the * interpreted decoder or machine code for the JIT. * - * This class is not thread-safe. - * - * TODO(haberman): move this to be heap allocated for ABI stability. */ + * This class is not thread-safe. */ class upb::pb::CodeCache { public: - CodeCache(); - ~CodeCache(); + CodeCache(upb::HandlerCache *dest) + : ptr_(upb_pbcodecache_new(dest->ptr()), upb_pbcodecache_free) {} + CodeCache(CodeCache&&) = default; + CodeCache& operator=(CodeCache&&) = default; + + upb_pbcodecache* ptr() { return ptr_.get(); } + const upb_pbcodecache* ptr() const { return ptr_.get(); } /* Whether the cache is allowed to generate machine code. Defaults to true. * There is no real reason to turn it off except for testing or if you are @@ -169,144 +214,27 @@ class upb::pb::CodeCache { * Note that allow_jit = true does not *guarantee* that the code will be JIT * compiled. If this platform is not supported or the JIT was not compiled * in, the code may still be interpreted. */ - bool allow_jit() const; + bool allow_jit() const { return upb_pbcodecache_allowjit(ptr()); } /* This may only be called when the object is first constructed, and prior to - * any code generation, otherwise returns false and does nothing. */ - bool set_allow_jit(bool allow); + * any code generation. */ + void set_allow_jit(bool allow) { upb_pbcodecache_setallowjit(ptr(), allow); } - /* Returns a DecoderMethod that can push data to the given handlers. - * If a suitable method already exists, it will be returned from the cache. - * - * Specifying the destination handlers here allows the DecoderMethod to be - * statically bound to the destination handlers if possible, which can allow - * more efficient decoding. However the returned method may or may not - * actually be statically bound. But in all cases, the returned method can - * push data to the given handlers. */ - const DecoderMethod *GetDecoderMethod(const DecoderMethodOptions& opts); + /* Should the decoder push submessages to lazy handlers for fields that have + * them? The caller should set this iff the lazy handlers expect data that is + * in protobuf binary format and the caller wishes to lazy parse it. */ + void set_lazy(bool lazy) { upb_pbcodecache_setlazy(ptr(), lazy); } - /* If/when someone needs to explicitly create a dynamically-bound - * DecoderMethod*, we can add a method to get it here. */ + /* Returns a DecoderMethod that can push data to the given handlers. + * If a suitable method already exists, it will be returned from the cache. */ + const DecoderMethodPtr Get(MessageDefPtr md) { + return DecoderMethodPtr(upb_pbcodecache_get(ptr(), md.ptr())); + } private: - UPB_DISALLOW_COPY_AND_ASSIGN(CodeCache) -#else -struct upb_pbcodecache { -#endif - bool allow_jit_; - - /* Array of mgroups. */ - upb_inttable groups; + std::unique_ptr<upb_pbcodecache, decltype(&upb_pbcodecache_free)> ptr_; }; -UPB_BEGIN_EXTERN_C - -upb_pbdecoder *upb_pbdecoder_create(upb_env *e, - const upb_pbdecodermethod *method, - upb_sink *output); -const upb_pbdecodermethod *upb_pbdecoder_method(const upb_pbdecoder *d); -upb_bytessink *upb_pbdecoder_input(upb_pbdecoder *d); -uint64_t upb_pbdecoder_bytesparsed(const upb_pbdecoder *d); -size_t upb_pbdecoder_maxnesting(const upb_pbdecoder *d); -bool upb_pbdecoder_setmaxnesting(upb_pbdecoder *d, size_t max); -void upb_pbdecoder_reset(upb_pbdecoder *d); - -void upb_pbdecodermethodopts_init(upb_pbdecodermethodopts *opts, - const upb_handlers *h); -void upb_pbdecodermethodopts_setlazy(upb_pbdecodermethodopts *opts, bool lazy); - - -/* Include refcounted methods like upb_pbdecodermethod_ref(). */ -UPB_REFCOUNTED_CMETHODS(upb_pbdecodermethod, upb_pbdecodermethod_upcast) - -const upb_handlers *upb_pbdecodermethod_desthandlers( - const upb_pbdecodermethod *m); -const upb_byteshandler *upb_pbdecodermethod_inputhandler( - const upb_pbdecodermethod *m); -bool upb_pbdecodermethod_isnative(const upb_pbdecodermethod *m); -const upb_pbdecodermethod *upb_pbdecodermethod_new( - const upb_pbdecodermethodopts *opts, const void *owner); - -void upb_pbcodecache_init(upb_pbcodecache *c); -void upb_pbcodecache_uninit(upb_pbcodecache *c); -bool upb_pbcodecache_allowjit(const upb_pbcodecache *c); -bool upb_pbcodecache_setallowjit(upb_pbcodecache *c, bool allow); -const upb_pbdecodermethod *upb_pbcodecache_getdecodermethod( - upb_pbcodecache *c, const upb_pbdecodermethodopts *opts); - -UPB_END_EXTERN_C - -#ifdef __cplusplus - -namespace upb { - -namespace pb { - -/* static */ -inline Decoder* Decoder::Create(Environment* env, const DecoderMethod* m, - Sink* sink) { - return upb_pbdecoder_create(env, m, sink); -} -inline const DecoderMethod* Decoder::method() const { - return upb_pbdecoder_method(this); -} -inline BytesSink* Decoder::input() { - return upb_pbdecoder_input(this); -} -inline uint64_t Decoder::BytesParsed() const { - return upb_pbdecoder_bytesparsed(this); -} -inline size_t Decoder::max_nesting() const { - return upb_pbdecoder_maxnesting(this); -} -inline bool Decoder::set_max_nesting(size_t max) { - return upb_pbdecoder_setmaxnesting(this, max); -} -inline void Decoder::Reset() { upb_pbdecoder_reset(this); } - -inline DecoderMethodOptions::DecoderMethodOptions(const Handlers* h) { - upb_pbdecodermethodopts_init(this, h); -} -inline void DecoderMethodOptions::set_lazy(bool lazy) { - upb_pbdecodermethodopts_setlazy(this, lazy); -} - -inline const Handlers* DecoderMethod::dest_handlers() const { - return upb_pbdecodermethod_desthandlers(this); -} -inline const BytesHandler* DecoderMethod::input_handler() const { - return upb_pbdecodermethod_inputhandler(this); -} -inline bool DecoderMethod::is_native() const { - return upb_pbdecodermethod_isnative(this); -} -/* static */ -inline reffed_ptr<const DecoderMethod> DecoderMethod::New( - const DecoderMethodOptions &opts) { - const upb_pbdecodermethod *m = upb_pbdecodermethod_new(&opts, &m); - return reffed_ptr<const DecoderMethod>(m, &m); -} - -inline CodeCache::CodeCache() { - upb_pbcodecache_init(this); -} -inline CodeCache::~CodeCache() { - upb_pbcodecache_uninit(this); -} -inline bool CodeCache::allow_jit() const { - return upb_pbcodecache_allowjit(this); -} -inline bool CodeCache::set_allow_jit(bool allow) { - return upb_pbcodecache_setallowjit(this, allow); -} -inline const DecoderMethod *CodeCache::GetDecoderMethod( - const DecoderMethodOptions& opts) { - return upb_pbcodecache_getdecodermethod(this, &opts); -} - -} /* namespace pb */ -} /* namespace upb */ - #endif /* __cplusplus */ #endif /* UPB_DECODER_H_ */ diff --git a/upb/pb/decoder.int.h b/upb/pb/decoder.int.h index c9ad551..42fd7f8 100644 --- a/upb/pb/decoder.int.h +++ b/upb/pb/decoder.int.h @@ -11,11 +11,6 @@ #include "upb/sink.h" #include "upb/table.int.h" -#ifndef __cplusplus - -UPB_DECLARE_DERIVED_TYPE(upb::pb::MessageGroup, upb::RefCounted, - mgroup, upb_refcounted) - /* Opcode definitions. The canonical meaning of each opcode is its * implementation in the interpreter (the JIT is written to match this). * @@ -77,30 +72,25 @@ typedef enum { UPB_INLINE opcode getop(uint32_t instr) { return (opcode)(instr & 0xff); } +struct upb_pbcodecache { + upb_arena *arena; + upb_handlercache *dest; + bool allow_jit; + bool lazy; + + /* Array of mgroups. */ + upb_inttable groups; +}; + /* Method group; represents a set of decoder methods that had their code - * emitted together, and must therefore be freed together. Immutable once - * created. It is possible we may want to expose this to users at some point. - * - * Overall ownership of Decoder objects looks like this: - * - * +----------+ - * | | <---> DecoderMethod - * | method | - * CodeCache ---> | group | <---> DecoderMethod - * | | - * | (mgroup) | <---> DecoderMethod - * +----------+ - */ -struct mgroup { - upb_refcounted base; - - /* Maps upb_msgdef/upb_handlers -> upb_pbdecodermethod. We own refs on the - * methods. */ + * emitted together. Immutable once created. */ +typedef struct { + /* Maps upb_msgdef/upb_handlers -> upb_pbdecodermethod. Owned by us. + * + * Ideally this would be on pbcodecache (if we were actually caching code). + * Right now we don't actually cache anything, which is wasteful. */ upb_inttable methods; - /* When we add the ability to link to previously existing mgroups, we'll - * need an array of mgroups we reference here, and own refs on them. */ - /* The bytecode for our methods, if any exists. Owned by us. */ uint32_t *bytecode; uint32_t *bytecode_end; @@ -113,7 +103,7 @@ struct mgroup { char *debug_info; void *dl; #endif -}; +} mgroup; /* The maximum that any submessages can be nested. Matches proto2's limit. * This specifies the size of the decoder's statically-sized array and therefore @@ -153,8 +143,6 @@ typedef struct { } upb_pbdecoder_frame; struct upb_pbdecodermethod { - upb_refcounted base; - /* While compiling, the base is relative in "ofs", after compiling it is * absolute in "ptr". */ union { @@ -162,14 +150,8 @@ struct upb_pbdecodermethod { void *ptr; /* Pointer to bytecode or machine code for this method. */ } code_base; - /* The decoder method group to which this method belongs. We own a ref. - * Owning a ref on the entire group is more coarse-grained than is strictly - * necessary; all we truly require is that methods we directly reference - * outlive us, while the group could contain many other messages we don't - * require. But the group represents the messages that were - * allocated+compiled together, so it makes the most sense to free them - * together also. */ - const upb_refcounted *group; + /* The decoder method group to which this method belongs. */ + const mgroup *group; /* Whether this method is native code or bytecode. */ bool is_native_; @@ -187,7 +169,7 @@ struct upb_pbdecodermethod { }; struct upb_pbdecoder { - upb_env *env; + upb_arena *arena; /* Our input sink. */ upb_bytessink input_; @@ -270,7 +252,6 @@ const char *upb_pbdecoder_getopname(unsigned int op); /* JIT codegen entry point. */ void upb_pbdecoder_jit(mgroup *group); void upb_pbdecoder_freejit(mgroup *group); -UPB_REFCOUNTED_CMETHODS(mgroup, mgroup_upcast) /* A special label that means "do field dispatch for this message and branch to * wherever that takes you." */ @@ -322,6 +303,4 @@ UPB_INLINE void upb_pbdecoder_unpackdispatch(uint64_t dispatch, uint64_t *ofs, #define CHECK_RETURN(x) { int32_t ret = x; if (ret >= 0) return ret; } -#endif /* __cplusplus */ - #endif /* UPB_DECODER_INT_H_ */ diff --git a/upb/pb/encoder.c b/upb/pb/encoder.c index 839ede0..5f74010 100644 --- a/upb/pb/encoder.c +++ b/upb/pb/encoder.c @@ -91,11 +91,11 @@ typedef struct { } upb_pb_encoder_segment; struct upb_pb_encoder { - upb_env *env; + upb_arena *arena; /* Our input and output. */ upb_sink input_; - upb_bytessink *output_; + upb_bytessink output_; /* The "subclosure" -- used as the inner closure as part of the bytessink * protocol. */ @@ -150,7 +150,7 @@ static bool reserve(upb_pb_encoder *e, size_t bytes) { new_size *= 2; } - new_buf = upb_env_realloc(e->env, e->buf, old_size, new_size); + new_buf = upb_arena_realloc(e->arena, e->buf, old_size, new_size); if (new_buf == NULL) { return false; @@ -230,7 +230,7 @@ static bool start_delim(upb_pb_encoder *e) { (e->seglimit - e->segbuf) * sizeof(upb_pb_encoder_segment); size_t new_size = old_size * 2; upb_pb_encoder_segment *new_buf = - upb_env_realloc(e->env, e->segbuf, old_size, new_size); + upb_arena_realloc(e->arena, e->segbuf, old_size, new_size); if (new_buf == NULL) { return false; @@ -304,8 +304,7 @@ static void new_tag(upb_handlers *h, const upb_fielddef *f, upb_wiretype_t wt, tag_t *tag = upb_gmalloc(sizeof(tag_t)); tag->bytes = upb_vencode64((n << 3) | wt, tag->tag); - upb_handlerattr_init(attr); - upb_handlerattr_sethandlerdata(attr, tag); + attr->handler_data = tag; upb_handlers_addcleanup(h, tag, upb_gfree); } @@ -434,6 +433,7 @@ T(sint64, int64_t, upb_zzenc_64, encode_varint) /* code to build the handlers *************************************************/ +#include <stdio.h> static void newhandlers_callback(const void *closure, upb_handlers *h) { const upb_msgdef *m; upb_msg_field_iter i; @@ -451,7 +451,7 @@ static void newhandlers_callback(const void *closure, upb_handlers *h) { const upb_fielddef *f = upb_msg_iter_field(&i); bool packed = upb_fielddef_isseq(f) && upb_fielddef_isprimitive(f) && upb_fielddef_packed(f); - upb_handlerattr attr; + upb_handlerattr attr = UPB_HANDLERATTR_INIT; upb_wiretype_t wt = packed ? UPB_WIRE_TYPE_DELIMITED : upb_pb_native_wire_types[upb_fielddef_descriptortype(f)]; @@ -500,20 +500,17 @@ static void newhandlers_callback(const void *closure, upb_handlers *h) { break; case UPB_DESCRIPTOR_TYPE_GROUP: { /* Endgroup takes a different tag (wire_type = END_GROUP). */ - upb_handlerattr attr2; + upb_handlerattr attr2 = UPB_HANDLERATTR_INIT; new_tag(h, f, UPB_WIRE_TYPE_END_GROUP, &attr2); upb_handlers_setstartsubmsg(h, f, encode_startgroup, &attr); upb_handlers_setendsubmsg(h, f, encode_endgroup, &attr2); - upb_handlerattr_uninit(&attr2); break; } } #undef T - - upb_handlerattr_uninit(&attr); } } @@ -526,27 +523,26 @@ void upb_pb_encoder_reset(upb_pb_encoder *e) { /* public API *****************************************************************/ -const upb_handlers *upb_pb_encoder_newhandlers(const upb_msgdef *m, - const void *owner) { - return upb_handlers_newfrozen(m, owner, newhandlers_callback, NULL); +upb_handlercache *upb_pb_encoder_newcache() { + return upb_handlercache_new(newhandlers_callback, NULL); } -upb_pb_encoder *upb_pb_encoder_create(upb_env *env, const upb_handlers *h, - upb_bytessink *output) { +upb_pb_encoder *upb_pb_encoder_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output) { const size_t initial_bufsize = 256; const size_t initial_segbufsize = 16; /* TODO(haberman): make this configurable. */ const size_t stack_size = 64; #ifndef NDEBUG - const size_t size_before = upb_env_bytesallocated(env); + const size_t size_before = upb_arena_bytesallocated(arena); #endif - upb_pb_encoder *e = upb_env_malloc(env, sizeof(upb_pb_encoder)); + upb_pb_encoder *e = upb_arena_malloc(arena, sizeof(upb_pb_encoder)); if (!e) return NULL; - e->buf = upb_env_malloc(env, initial_bufsize); - e->segbuf = upb_env_malloc(env, initial_segbufsize * sizeof(*e->segbuf)); - e->stack = upb_env_malloc(env, stack_size * sizeof(*e->stack)); + e->buf = upb_arena_malloc(arena, initial_bufsize); + e->segbuf = upb_arena_malloc(arena, initial_segbufsize * sizeof(*e->segbuf)); + e->stack = upb_arena_malloc(arena, stack_size * sizeof(*e->stack)); if (!e->buf || !e->segbuf || !e->stack) { return NULL; @@ -559,15 +555,15 @@ upb_pb_encoder *upb_pb_encoder_create(upb_env *env, const upb_handlers *h, upb_pb_encoder_reset(e); upb_sink_reset(&e->input_, h, e); - e->env = env; + e->arena = arena; e->output_ = output; - e->subc = output->closure; + e->subc = output.closure; e->ptr = e->buf; /* If this fails, increase the value in encoder.h. */ - UPB_ASSERT_DEBUGVAR(upb_env_bytesallocated(env) - size_before <= + UPB_ASSERT_DEBUGVAR(upb_arena_bytesallocated(arena) - size_before <= UPB_PB_ENCODER_SIZE); return e; } -upb_sink *upb_pb_encoder_input(upb_pb_encoder *e) { return &e->input_; } +upb_sink upb_pb_encoder_input(upb_pb_encoder *e) { return e->input_; } diff --git a/upb/pb/encoder.h b/upb/pb/encoder.h index 41b7e7b..1113c3a 100644 --- a/upb/pb/encoder.h +++ b/upb/pb/encoder.h @@ -17,75 +17,67 @@ #ifdef __cplusplus namespace upb { namespace pb { -class Encoder; +class EncoderPtr; } /* namespace pb */ } /* namespace upb */ #endif -UPB_DECLARE_TYPE(upb::pb::Encoder, upb_pb_encoder) - #define UPB_PBENCODER_MAX_NESTING 100 -/* upb::pb::Encoder ***********************************************************/ +/* upb_pb_encoder *************************************************************/ /* Preallocation hint: decoder won't allocate more bytes than this when first * constructed. This hint may be an overestimate for some build configurations. * But if the decoder library is upgraded without recompiling the application, * it may be an underestimate. */ -#define UPB_PB_ENCODER_SIZE 768 +#define UPB_PB_ENCODER_SIZE 784 + +struct upb_pb_encoder; +typedef struct upb_pb_encoder upb_pb_encoder; #ifdef __cplusplus +extern "C" { +#endif -class upb::pb::Encoder { +upb_sink upb_pb_encoder_input(upb_pb_encoder *p); +upb_pb_encoder* upb_pb_encoder_create(upb_arena* a, const upb_handlers* h, + upb_bytessink output); + +/* Lazily builds and caches handlers that will push encoded data to a bytessink. + * Any msgdef objects used with this object must outlive it. */ +upb_handlercache *upb_pb_encoder_newcache(); + +#ifdef __cplusplus +} /* extern "C" { */ + +class upb::pb::EncoderPtr { public: + EncoderPtr(upb_pb_encoder* ptr) : ptr_(ptr) {} + + upb_pb_encoder* ptr() { return ptr_; } + /* Creates a new encoder in the given environment. The Handlers must have * come from NewHandlers() below. */ - static Encoder* Create(Environment* env, const Handlers* handlers, - BytesSink* output); + static EncoderPtr Create(Arena* arena, const Handlers* handlers, + BytesSink output) { + return EncoderPtr( + upb_pb_encoder_create(arena->ptr(), handlers, output.sink())); + } /* The input to the encoder. */ - Sink* input(); + upb::Sink input() { return upb_pb_encoder_input(ptr()); } /* Creates a new set of handlers for this MessageDef. */ - static reffed_ptr<const Handlers> NewHandlers(const MessageDef* msg); + static HandlerCache NewCache() { + return HandlerCache(upb_pb_encoder_newcache()); + } static const size_t kSize = UPB_PB_ENCODER_SIZE; private: - UPB_DISALLOW_POD_OPS(Encoder, upb::pb::Encoder) + upb_pb_encoder* ptr_; }; -#endif - -UPB_BEGIN_EXTERN_C - -const upb_handlers *upb_pb_encoder_newhandlers(const upb_msgdef *m, - const void *owner); -upb_sink *upb_pb_encoder_input(upb_pb_encoder *p); -upb_pb_encoder* upb_pb_encoder_create(upb_env* e, const upb_handlers* h, - upb_bytessink* output); - -UPB_END_EXTERN_C - -#ifdef __cplusplus - -namespace upb { -namespace pb { -inline Encoder* Encoder::Create(Environment* env, const Handlers* handlers, - BytesSink* output) { - return upb_pb_encoder_create(env, handlers, output); -} -inline Sink* Encoder::input() { - return upb_pb_encoder_input(this); -} -inline reffed_ptr<const Handlers> Encoder::NewHandlers( - const upb::MessageDef *md) { - const Handlers* h = upb_pb_encoder_newhandlers(md, &h); - return reffed_ptr<const Handlers>(h, &h); -} -} /* namespace pb */ -} /* namespace upb */ - -#endif +#endif /* __cplusplus */ #endif /* UPB_ENCODER_H_ */ diff --git a/upb/pb/glue.c b/upb/pb/glue.c deleted file mode 100644 index fb2b769..0000000 --- a/upb/pb/glue.c +++ /dev/null @@ -1,54 +0,0 @@ - -#include "upb/pb/glue.h" - -#include "upb/descriptor/reader.h" -#include "upb/pb/decoder.h" - -upb_filedef **upb_loaddescriptor(const char *buf, size_t n, const void *owner, - upb_status *status) { - /* Create handlers. */ - const upb_pbdecodermethod *decoder_m; - const upb_handlers *reader_h = upb_descreader_newhandlers(&reader_h); - upb_env env; - upb_pbdecodermethodopts opts; - upb_pbdecoder *decoder; - upb_descreader *reader; - bool ok; - size_t i; - upb_filedef **ret = NULL; - - upb_pbdecodermethodopts_init(&opts, reader_h); - decoder_m = upb_pbdecodermethod_new(&opts, &decoder_m); - - upb_env_init(&env); - upb_env_reporterrorsto(&env, status); - - reader = upb_descreader_create(&env, reader_h); - decoder = upb_pbdecoder_create(&env, decoder_m, upb_descreader_input(reader)); - - /* Push input data. */ - ok = upb_bufsrc_putbuf(buf, n, upb_pbdecoder_input(decoder)); - - if (!ok) { - goto cleanup; - } - - ret = upb_gmalloc(sizeof (*ret) * (upb_descreader_filecount(reader) + 1)); - - if (!ret) { - goto cleanup; - } - - for (i = 0; i < upb_descreader_filecount(reader); i++) { - ret[i] = upb_descreader_file(reader, i); - upb_filedef_ref(ret[i], owner); - } - - ret[i] = NULL; - -cleanup: - upb_env_uninit(&env); - upb_handlers_unref(reader_h, &reader_h); - upb_pbdecodermethod_unref(decoder_m, &decoder_m); - return ret; -} diff --git a/upb/pb/glue.h b/upb/pb/glue.h deleted file mode 100644 index 716fc0e..0000000 --- a/upb/pb/glue.h +++ /dev/null @@ -1,72 +0,0 @@ -/* -** upb's core components like upb_decoder and upb_msg are carefully designed to -** avoid depending on each other for maximum orthogonality. In other words, -** you can use a upb_decoder to decode into *any* kind of structure; upb_msg is -** just one such structure. A upb_msg can be serialized/deserialized into any -** format, protobuf binary format is just one such format. -** -** However, for convenience we provide functions here for doing common -** operations like deserializing protobuf binary format into a upb_msg. The -** compromise is that this file drags in almost all of upb as a dependency, -** which could be undesirable if you're trying to use a trimmed-down build of -** upb. -** -** While these routines are convenient, they do not reuse any encoding/decoding -** state. For example, if a decoder is JIT-based, it will be re-JITted every -** time these functions are called. For this reason, if you are parsing lots -** of data and efficiency is an issue, these may not be the best functions to -** use (though they are useful for prototyping, before optimizing). -*/ - -#ifndef UPB_GLUE_H -#define UPB_GLUE_H - -#include <stdbool.h> -#include "upb/def.h" - -#ifdef __cplusplus -#include <vector> - -extern "C" { -#endif - -/* Loads a binary descriptor and returns a NULL-terminated array of unfrozen - * filedefs. The caller owns the returned array, which must be freed with - * upb_gfree(). */ -upb_filedef **upb_loaddescriptor(const char *buf, size_t n, const void *owner, - upb_status *status); - -#ifdef __cplusplus -} /* extern "C" */ - -namespace upb { - -inline bool LoadDescriptor(const char* buf, size_t n, Status* status, - std::vector<reffed_ptr<FileDef> >* files) { - FileDef** parsed_files = upb_loaddescriptor(buf, n, &parsed_files, status); - - if (parsed_files) { - FileDef** p = parsed_files; - while (*p) { - files->push_back(reffed_ptr<FileDef>(*p, &parsed_files)); - ++p; - } - free(parsed_files); - return true; - } else { - return false; - } -} - -/* Templated so it can accept both string and std::string. */ -template <typename T> -bool LoadDescriptor(const T& desc, Status* status, - std::vector<reffed_ptr<FileDef> >* files) { - return LoadDescriptor(desc.c_str(), desc.size(), status, files); -} - -} /* namespace upb */ - -#endif - -#endif /* UPB_GLUE_H */ diff --git a/upb/pb/textprinter.c b/upb/pb/textprinter.c index abfc2eb..934130e 100644 --- a/upb/pb/textprinter.c +++ b/upb/pb/textprinter.c @@ -18,7 +18,7 @@ struct upb_textprinter { upb_sink input_; - upb_bytessink *output_; + upb_bytessink output_; int indent_depth_; bool single_line_; void *subc; @@ -183,7 +183,7 @@ static bool textprinter_putenum(void *closure, const void *handler_data, int32_t val) { upb_textprinter *p = closure; const upb_fielddef *f = handler_data; - const upb_enumdef *enum_def = upb_downcast_enumdef(upb_fielddef_subdef(f)); + const upb_enumdef *enum_def = upb_fielddef_enumsubdef(f); const char *label = upb_enumdef_iton(enum_def, val); if (label) { indent(p); @@ -260,8 +260,8 @@ static void onmreg(const void *c, upb_handlers *h) { !upb_msg_field_done(&i); upb_msg_field_next(&i)) { upb_fielddef *f = upb_msg_iter_field(&i); - upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; - upb_handlerattr_sethandlerdata(&attr, f); + upb_handlerattr attr = UPB_HANDLERATTR_INIT; + attr.handler_data = f; switch (upb_fielddef_type(f)) { case UPB_TYPE_INT32: upb_handlers_setint32(h, f, textprinter_putint32, &attr); @@ -292,10 +292,10 @@ static void onmreg(const void *c, upb_handlers *h) { break; case UPB_TYPE_MESSAGE: { const char *name = - upb_fielddef_istagdelim(f) + upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_GROUP ? shortname(upb_msgdef_fullname(upb_fielddef_msgsubdef(f))) : upb_fielddef_name(f); - upb_handlerattr_sethandlerdata(&attr, name); + attr.handler_data = name; upb_handlers_setstartsubmsg(h, f, textprinter_startsubmsg, &attr); upb_handlers_setendsubmsg(h, f, textprinter_endsubmsg, &attr); break; @@ -315,9 +315,9 @@ static void textprinter_reset(upb_textprinter *p, bool single_line) { /* Public API *****************************************************************/ -upb_textprinter *upb_textprinter_create(upb_env *env, const upb_handlers *h, - upb_bytessink *output) { - upb_textprinter *p = upb_env_malloc(env, sizeof(upb_textprinter)); +upb_textprinter *upb_textprinter_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output) { + upb_textprinter *p = upb_arena_malloc(arena, sizeof(upb_textprinter)); if (!p) return NULL; p->output_ = output; @@ -327,12 +327,11 @@ upb_textprinter *upb_textprinter_create(upb_env *env, const upb_handlers *h, return p; } -const upb_handlers *upb_textprinter_newhandlers(const upb_msgdef *m, - const void *owner) { - return upb_handlers_newfrozen(m, owner, &onmreg, NULL); +upb_handlercache *upb_textprinter_newcache() { + return upb_handlercache_new(&onmreg, NULL); } -upb_sink *upb_textprinter_input(upb_textprinter *p) { return &p->input_; } +upb_sink upb_textprinter_input(upb_textprinter *p) { return p->input_; } void upb_textprinter_setsingleline(upb_textprinter *p, bool single_line) { p->single_line_ = single_line; diff --git a/upb/pb/textprinter.h b/upb/pb/textprinter.h index 2f40ed8..0af2b1a 100644 --- a/upb/pb/textprinter.h +++ b/upb/pb/textprinter.h @@ -12,68 +12,57 @@ #ifdef __cplusplus namespace upb { namespace pb { -class TextPrinter; +class TextPrinterPtr; } /* namespace pb */ } /* namespace upb */ #endif -UPB_DECLARE_TYPE(upb::pb::TextPrinter, upb_textprinter) +/* upb_textprinter ************************************************************/ + +struct upb_textprinter; +typedef struct upb_textprinter upb_textprinter; + +#ifdef __cplusplus +extern "C" { +#endif + +/* C API. */ +upb_textprinter *upb_textprinter_create(upb_arena *arena, const upb_handlers *h, + upb_bytessink output); +void upb_textprinter_setsingleline(upb_textprinter *p, bool single_line); +upb_sink upb_textprinter_input(upb_textprinter *p); +upb_handlercache *upb_textprinter_newcache(); #ifdef __cplusplus +} /* extern "C" */ -class upb::pb::TextPrinter { +class upb::pb::TextPrinterPtr { public: + TextPrinterPtr(upb_textprinter* ptr) : ptr_(ptr) {} + /* The given handlers must have come from NewHandlers(). It must outlive the * TextPrinter. */ - static TextPrinter *Create(Environment *env, const upb::Handlers *handlers, - BytesSink *output); + static TextPrinterPtr Create(Arena *arena, upb::HandlersPtr *handlers, + BytesSink output) { + return TextPrinterPtr( + upb_textprinter_create(arena->ptr(), handlers->ptr(), output.sink())); + } - void SetSingleLineMode(bool single_line); + void SetSingleLineMode(bool single_line) { + upb_textprinter_setsingleline(ptr_, single_line); + } - Sink* input(); + Sink input() { return upb_textprinter_input(ptr_); } /* If handler caching becomes a requirement we can add a code cache as in * decoder.h */ - static reffed_ptr<const Handlers> NewHandlers(const MessageDef* md); -}; - -#endif - -UPB_BEGIN_EXTERN_C + static HandlerCache NewCache() { + return HandlerCache(upb_textprinter_newcache()); + } -/* C API. */ -upb_textprinter *upb_textprinter_create(upb_env *env, const upb_handlers *h, - upb_bytessink *output); -void upb_textprinter_setsingleline(upb_textprinter *p, bool single_line); -upb_sink *upb_textprinter_input(upb_textprinter *p); - -const upb_handlers *upb_textprinter_newhandlers(const upb_msgdef *m, - const void *owner); - -UPB_END_EXTERN_C - -#ifdef __cplusplus - -namespace upb { -namespace pb { -inline TextPrinter *TextPrinter::Create(Environment *env, - const upb::Handlers *handlers, - BytesSink *output) { - return upb_textprinter_create(env, handlers, output); -} -inline void TextPrinter::SetSingleLineMode(bool single_line) { - upb_textprinter_setsingleline(this, single_line); -} -inline Sink* TextPrinter::input() { - return upb_textprinter_input(this); -} -inline reffed_ptr<const Handlers> TextPrinter::NewHandlers( - const MessageDef *md) { - const Handlers* h = upb_textprinter_newhandlers(md, &h); - return reffed_ptr<const Handlers>(h, &h); -} -} /* namespace pb */ -} /* namespace upb */ + private: + upb_textprinter* ptr_; +}; #endif diff --git a/upb/pb/varint.int.h b/upb/pb/varint.int.h index 9c54311..b216a94 100644 --- a/upb/pb/varint.int.h +++ b/upb/pb/varint.int.h @@ -24,6 +24,18 @@ extern "C" { * descriptor type (upb_descriptortype_t). */ extern const uint8_t upb_pb_native_wire_types[]; +UPB_INLINE uint64_t byteswap64(uint64_t val) +{ + return ((((val) & 0xff00000000000000ull) >> 56) + | (((val) & 0x00ff000000000000ull) >> 40) + | (((val) & 0x0000ff0000000000ull) >> 24) + | (((val) & 0x000000ff00000000ull) >> 8) + | (((val) & 0x00000000ff000000ull) << 8) + | (((val) & 0x0000000000ff0000ull) << 24) + | (((val) & 0x000000000000ff00ull) << 40) + | (((val) & 0x00000000000000ffull) << 56)); +} + /* Zig-zag encoding/decoding **************************************************/ UPB_INLINE int32_t upb_zzdec_32(uint32_t n) { @@ -130,6 +142,9 @@ UPB_INLINE uint64_t upb_vencode32(uint32_t val) { uint64_t ret = 0; UPB_ASSERT(bytes <= 5); memcpy(&ret, buf, bytes); +#ifdef UPB_BIG_ENDIAN + ret = byteswap64(ret); +#endif UPB_ASSERT(ret <= 0xffffffffffU); return ret; } |