diff options
Diffstat (limited to 'upb/pb')
| -rw-r--r-- | upb/pb/decoder.c | 361 | ||||
| -rw-r--r-- | upb/pb/decoder.h | 102 | ||||
| -rw-r--r-- | upb/pb/decoder_x64.dasc | 322 | ||||
| -rw-r--r-- | upb/pb/glue.c | 33 | ||||
| -rw-r--r-- | upb/pb/glue.h | 4 | ||||
| -rw-r--r-- | upb/pb/varint.h | 40 |
6 files changed, 545 insertions, 317 deletions
diff --git a/upb/pb/decoder.c b/upb/pb/decoder.c index ae54e47..1b5fc17 100644 --- a/upb/pb/decoder.c +++ b/upb/pb/decoder.c @@ -13,14 +13,95 @@ #include "upb/pb/decoder.h" #include "upb/pb/varint.h" +/* upb_decoderplan ************************************************************/ + #ifdef UPB_USE_JIT_X64 -#define Dst_DECL upb_decoder *d -#define Dst_REF (d->dynasm) -#define Dst (d) +// These defines are necessary for DynASM codegen. +// See dynasm/dasm_proto.h for more info. +#define Dst_DECL upb_decoderplan *plan +#define Dst_REF (plan->dynasm) +#define Dst (plan) + +// In debug mode, make DynASM do internal checks (must be defined before any +// dasm header is included. +#ifndef NDEBUG +#define DASM_CHECKS +#endif + #include "dynasm/dasm_proto.h" #include "upb/pb/decoder_x64.h" #endif +typedef struct { + upb_fhandlers base; + void (*decode)(struct _upb_decoder *d, struct _upb_fieldent *f); +#ifdef UPB_USE_JIT_X64 + uint32_t jit_pclabel; + uint32_t jit_pclabel_notypecheck; +#endif +} upb_dplanfield; + +typedef struct { + upb_mhandlers base; +#ifdef UPB_USE_JIT_X64 + uint32_t jit_startmsg_pclabel; + uint32_t jit_endofbuf_pclabel; + uint32_t jit_endofmsg_pclabel; + uint32_t jit_dyndispatch_pclabel; + uint32_t jit_unknownfield_pclabel; + int32_t jit_parent_field_done_pclabel; + uint32_t max_field_number; + // Currently keyed on field number. Could also try keying it + // on encoded or decoded tag, or on encoded field number. + void **tablearray; +#endif +} upb_dplanmsg; + +static void *upb_decoderplan_fptrs[]; + +void upb_decoderplan_initfhandlers(upb_fhandlers *f) { + f->decode = upb_decoderplan_fptrs[f->type]; +} + +upb_decoderplan *upb_decoderplan_new(upb_handlers *h, bool allowjit) { + upb_decoderplan *p = malloc(sizeof(*p)); + p->handlers = h; + upb_handlers_ref(h); + h->should_jit = allowjit; +#ifdef UPB_USE_JIT_X64 + p->jit_code = NULL; + if (allowjit) upb_decoderplan_makejit(p); +#endif + // Set function pointers for each field's decode function. + for (int i = 0; i < h->msgs_len; i++) { + upb_mhandlers *m = h->msgs[i]; + for(upb_inttable_iter i = upb_inttable_begin(&m->fieldtab); + !upb_inttable_done(i); + i = upb_inttable_next(&m->fieldtab, i)) { + upb_itofhandlers_ent *e = upb_inttable_iter_value(i); + upb_fhandlers *f = e->f; + upb_decoderplan_initfhandlers(f); + } + } + return p; +} + +void upb_decoderplan_unref(upb_decoderplan *p) { + // TODO: make truly refcounted. + upb_handlers_unref(p->handlers); +#ifdef UPB_USE_JIT_X64 + if (p->jit_code) upb_decoderplan_freejit(p); +#endif + free(p); +} + +bool upb_decoderplan_hasjitcode(upb_decoderplan *p) { + return p->jit_code != NULL; +} + + +/* upb_decoder ****************************************************************/ + // It's unfortunate that we have to micro-manage the compiler this way, // especially since this tuning is necessarily specific to one hardware // configuration. But emperically on a Core i7, performance increases 30-50% @@ -29,18 +110,17 @@ #define FORCEINLINE static __attribute__((always_inline)) #define NOINLINE static __attribute__((noinline)) -static void upb_decoder_exit(upb_decoder *d) { +UPB_NORETURN static void upb_decoder_exitjmp(upb_decoder *d) { // Resumable decoder would back out to completed_ptr (and possibly get a // previous buffer). siglongjmp(d->exitjmp, 1); } -static void upb_decoder_exit2(void *_d) { - upb_decoder *d = _d; - upb_decoder_exit(d); +UPB_NORETURN static void upb_decoder_exitjmp2(void *d) { + upb_decoder_exitjmp(d); } -static void upb_decoder_abort(upb_decoder *d, const char *msg) { - upb_status_seterrliteral(d->status, msg); - upb_decoder_exit(d); +UPB_NORETURN static void upb_decoder_abortjmp(upb_decoder *d, const char *msg) { + upb_status_seterrliteral(&d->status, msg); + upb_decoder_exitjmp(d); } /* Buffering ******************************************************************/ @@ -50,8 +130,12 @@ static void upb_decoder_abort(upb_decoder *d, const char *msg) { // the next one. When we've committed our progress we discard any previous // buffers' regions. -static uint32_t upb_decoder_bufleft(upb_decoder *d) { return d->end - d->ptr; } -static void upb_decoder_advance(upb_decoder *d, uint32_t len) { +static size_t upb_decoder_bufleft(upb_decoder *d) { + assert(d->end >= d->ptr); + return d->end - d->ptr; +} + +static void upb_decoder_advance(upb_decoder *d, size_t len) { assert(upb_decoder_bufleft(d) >= len); d->ptr += len; } @@ -66,29 +150,49 @@ uint64_t upb_decoder_bufendofs(upb_decoder *d) { static void upb_decoder_setmsgend(upb_decoder *d) { upb_dispatcher_frame *f = d->dispatcher.top; - uint32_t delimlen = f->end_ofs - d->bufstart_ofs; - uint32_t buflen = d->end - d->buf; + size_t delimlen = f->end_ofs - d->bufstart_ofs; + size_t buflen = d->end - d->buf; d->delim_end = (f->end_ofs != UPB_NONDELIMITED && delimlen <= buflen) ? d->buf + delimlen : NULL; // NULL if not in this buf. d->top_is_packed = f->is_packed; + d->dispatch_table = &d->dispatcher.msgent->fieldtab; } -static bool upb_trypullbuf(upb_decoder *d) { - assert(upb_decoder_bufleft(d) == 0); - d->bufstart_ofs = upb_decoder_offset(d); +static void upb_decoder_skiptonewbuf(upb_decoder *d, uint64_t ofs) { + assert(ofs >= upb_decoder_offset(d)); + if (ofs > upb_byteregion_endofs(d->input)) + upb_decoder_abortjmp(d, "Unexpected EOF"); d->buf = NULL; d->ptr = NULL; d->end = NULL; - if (upb_byteregion_available(d->input, upb_decoder_offset(d)) == 0 && - !upb_byteregion_fetch(d->input, d->status)) { - if (upb_eof(d->status)) return false; - upb_decoder_exit(d); // Non-EOF error. + d->delim_end = NULL; +#ifdef UPB_USE_JIT_X64 + d->jit_end = NULL; +#endif + d->bufstart_ofs = ofs; +} + +static bool upb_trypullbuf(upb_decoder *d) { + assert(upb_decoder_bufleft(d) == 0); + upb_decoder_skiptonewbuf(d, upb_decoder_offset(d)); + if (upb_byteregion_available(d->input, d->bufstart_ofs) == 0) { + switch (upb_byteregion_fetch(d->input)) { + case UPB_BYTE_OK: + assert(upb_byteregion_available(d->input, d->bufstart_ofs) > 0); + break; + case UPB_BYTE_EOF: return false; + case UPB_BYTE_ERROR: upb_decoder_abortjmp(d, "I/O error in input"); + // Decoder resuming is not yet supported. + case UPB_BYTE_WOULDBLOCK: + upb_decoder_abortjmp(d, "Input returned WOULDBLOCK"); + } } - uint32_t len; + size_t len; d->buf = upb_byteregion_getptr(d->input, d->bufstart_ofs, &len); assert(len > 0); d->ptr = d->buf; d->end = d->buf + len; + upb_decoder_setmsgend(d); #ifdef UPB_USE_JIT_X64 // If we start parsing a value, we can parse up to 20 bytes without // having to bounds-check anything (2 10-byte varints). Since the @@ -96,27 +200,29 @@ static bool upb_trypullbuf(upb_decoder *d) { // JIT bails if there are not 20 bytes available. d->jit_end = d->end - 20; #endif - upb_decoder_setmsgend(d); + assert(upb_decoder_bufleft(d) > 0); return true; } static void upb_pullbuf(upb_decoder *d) { - if (!upb_trypullbuf(d)) upb_decoder_abort(d, "Unexpected EOF"); + if (!upb_trypullbuf(d)) upb_decoder_abortjmp(d, "Unexpected EOF"); } -void upb_decoder_skipto(upb_decoder *d, uint64_t ofs) { - if (ofs < upb_decoder_bufendofs(d)) { +void upb_decoder_checkpoint(upb_decoder *d) { + upb_byteregion_discard(d->input, upb_decoder_offset(d)); +} + +void upb_decoder_discardto(upb_decoder *d, uint64_t ofs) { + if (ofs <= upb_decoder_bufendofs(d)) { upb_decoder_advance(d, ofs - upb_decoder_offset(d)); } else { - d->buf = NULL; - d->ptr = NULL; - d->end = NULL; - d->bufstart_ofs = ofs; + upb_decoder_skiptonewbuf(d, ofs); } + upb_decoder_checkpoint(d); } -void upb_decoder_checkpoint(upb_decoder *d) { - upb_byteregion_discard(d->input, upb_decoder_offset(d)); +void upb_decoder_discard(upb_decoder *d, size_t bytes) { + upb_decoder_discardto(d, upb_decoder_offset(d) + bytes); } @@ -126,15 +232,13 @@ NOINLINE uint64_t upb_decode_varint_slow(upb_decoder *d) { uint8_t byte = 0x80; uint64_t u64 = 0; int bitpos; - const char *ptr = d->ptr; for(bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) { - if (upb_decoder_bufleft(d) == 0) { - upb_pullbuf(d); - ptr = d->ptr; - } - u64 |= ((uint64_t)(byte = *ptr++) & 0x7F) << bitpos; + if (upb_decoder_bufleft(d) == 0) upb_pullbuf(d); + u64 |= ((uint64_t)(byte = *d->ptr) & 0x7F) << bitpos; + upb_decoder_advance(d, 1); } - if(bitpos == 70 && (byte & 0x80)) upb_decoder_abort(d, "Unterminated varint"); + if(bitpos == 70 && (byte & 0x80)) + upb_decoder_abortjmp(d, "Unterminated varint"); return u64; } @@ -151,7 +255,7 @@ FORCEINLINE uint32_t upb_decode_varint32(upb_decoder *d) { if ((*(p++) & 0x80) == 0) goto done; // likely slow: u64 = upb_decode_varint_slow(d); - if (u64 > 0xffffffff) upb_decoder_abort(d, "Unterminated 32-bit varint"); + if (u64 > UINT32_MAX) upb_decoder_abortjmp(d, "Unterminated 32-bit varint"); ret = (uint32_t)u64; p = d->ptr; // Turn the next line into a nop. done: @@ -174,7 +278,7 @@ FORCEINLINE uint64_t upb_decode_varint(upb_decoder *d) { if (upb_decoder_bufleft(d) >= 10) { // Fast case. upb_decoderet r = upb_vdecode_fast(d->ptr); - if (r.p == NULL) upb_decoder_abort(d, "Unterminated varint"); + if (r.p == NULL) upb_decoder_abortjmp(d, "Unterminated varint"); upb_decoder_advance(d, r.p - d->ptr); return r.val; } else if (upb_decoder_bufleft(d) > 0) { @@ -200,11 +304,12 @@ FORCEINLINE void upb_decode_fixed(upb_decoder *d, char *buf, size_t bytes) { } else { // Slow case. size_t read = 0; - while (read < bytes) { - size_t avail = upb_decoder_bufleft(d); + while (1) { + size_t avail = UPB_MIN(upb_decoder_bufleft(d), bytes - read); memcpy(buf + read, d->ptr, avail); upb_decoder_advance(d, avail); read += avail; + if (read == bytes) break; upb_pullbuf(d); } } @@ -213,26 +318,28 @@ FORCEINLINE void upb_decode_fixed(upb_decoder *d, char *buf, size_t bytes) { FORCEINLINE uint32_t upb_decode_fixed32(upb_decoder *d) { uint32_t u32; upb_decode_fixed(d, (char*)&u32, sizeof(uint32_t)); - return u32; // TODO: proper byte swapping + return u32; // TODO: proper byte swapping for big-endian machines. } FORCEINLINE uint64_t upb_decode_fixed64(upb_decoder *d) { uint64_t u64; upb_decode_fixed(d, (char*)&u64, sizeof(uint64_t)); - return u64; // TODO: proper byte swapping + return u64; // TODO: proper byte swapping for big-endian machines. } INLINE upb_byteregion *upb_decode_string(upb_decoder *d) { uint32_t strlen = upb_decode_varint32(d); uint64_t offset = upb_decoder_offset(d); + if (offset + strlen > upb_byteregion_endofs(d->input)) + upb_decoder_abortjmp(d, "Unexpected EOF"); upb_byteregion_reset(&d->str_byteregion, d->input, offset, strlen); // Could make it an option on the callback whether we fetchall() first or not. - upb_byteregion_fetchall(&d->str_byteregion, d->status); - if (!upb_ok(d->status)) upb_decoder_exit(d); - upb_decoder_skipto(d, offset + strlen); + if (upb_byteregion_fetchall(&d->str_byteregion) != UPB_BYTE_OK) + upb_decoder_abortjmp(d, "Couldn't fetchall() on string."); + upb_decoder_discardto(d, offset + strlen); return &d->str_byteregion; } -INLINE void upb_push(upb_decoder *d, upb_fhandlers *f, uint64_t end) { +INLINE void upb_push_msg(upb_decoder *d, upb_fhandlers *f, uint64_t end) { upb_dispatch_startsubmsg(&d->dispatcher, f)->end_ofs = end; upb_decoder_setmsgend(d); } @@ -253,8 +360,6 @@ INLINE void upb_push(upb_decoder *d, upb_fhandlers *f, uint64_t end) { static double upb_asdouble(uint64_t n) { double d; memcpy(&d, &n, 8); return d; } static float upb_asfloat(uint32_t n) { float f; memcpy(&f, &n, 4); return f; } -static int32_t upb_zzdec_32(uint32_t n) { return (n >> 1) ^ -(int32_t)(n & 1); } -static int64_t upb_zzdec_64(uint64_t n) { return (n >> 1) ^ -(int64_t)(n & 1); } T(INT32, varint, int32, int32_t) T(INT64, varint, int64, int64_t) @@ -271,9 +376,10 @@ T(FLOAT, fixed32, float, upb_asfloat) T(SINT32, varint, int32, upb_zzdec_32) T(SINT64, varint, int64, upb_zzdec_64) T(STRING, string, byteregion, upb_byteregion*) +#undef T static void upb_decode_GROUP(upb_decoder *d, upb_fhandlers *f) { - upb_push(d, f, UPB_NONDELIMITED); + upb_push_msg(d, f, UPB_NONDELIMITED); } static void upb_endgroup(upb_decoder *d, upb_fhandlers *f) { (void)f; @@ -281,15 +387,30 @@ static void upb_endgroup(upb_decoder *d, upb_fhandlers *f) { upb_decoder_setmsgend(d); } static void upb_decode_MESSAGE(upb_decoder *d, upb_fhandlers *f) { - upb_push(d, f, upb_decode_varint32(d) + upb_decoder_offset(d)); + uint32_t len = upb_decode_varint32(d); + upb_push_msg(d, f, upb_decoder_offset(d) + len); } +#define F(type) &upb_decode_ ## type +static void *upb_decoderplan_fptrs[] = { + &upb_endgroup, F(DOUBLE), F(FLOAT), F(INT64), + F(UINT64), F(INT32), F(FIXED64), F(FIXED32), F(BOOL), F(STRING), + F(GROUP), F(MESSAGE), F(STRING), F(UINT32), F(ENUM), F(SFIXED32), + F(SFIXED64), F(SINT32), F(SINT64)}; +#undef F + /* The main decoding loop *****************************************************/ static void upb_decoder_checkdelim(upb_decoder *d) { + // TODO: This doesn't work for the case that no buffer is currently loaded + // (ie. d->buf == NULL) because delim_end is NULL even if we are at + // end-of-delim. Need to add a test that exercises this by putting a buffer + // seam in the middle of the final delimited value in a proto that we skip + // for some reason (like because it's unknown and we have no unknown field + // handler). while (d->delim_end != NULL && d->ptr >= d->delim_end) { - if (d->ptr > d->delim_end) upb_decoder_abort(d, "Bad submessage end"); + if (d->ptr > d->delim_end) upb_decoder_abortjmp(d, "Bad submessage end"); if (d->dispatcher.top->is_sequence) { upb_dispatch_endseq(&d->dispatcher); } else { @@ -299,33 +420,36 @@ static void upb_decoder_checkdelim(upb_decoder *d) { } } -static void upb_decoder_enterjit(upb_decoder *d) { - (void)d; -#ifdef UPB_USE_JIT_X64 - if (d->jit_code && d->dispatcher.top == d->dispatcher.stack && d->ptr < d->jit_end) { - // Decodes as many fields as possible, updating d->ptr appropriately, - // before falling through to the slow(er) path. - void (*upb_jit_decode)(upb_decoder *d) = (void*)d->jit_code; - upb_jit_decode(d); - } -#endif -} - INLINE upb_fhandlers *upb_decode_tag(upb_decoder *d) { while (1) { uint32_t tag; if (!upb_trydecode_varint32(d, &tag)) return NULL; uint8_t wire_type = tag & 0x7; - upb_fhandlers *f = upb_dispatcher_lookup(&d->dispatcher, tag); + uint32_t fieldnum = tag >> 3; + upb_itofhandlers_ent *e = upb_inttable_fastlookup( + d->dispatch_table, fieldnum, sizeof(upb_itofhandlers_ent)); + upb_fhandlers *f = e ? e->f : NULL; + + if (f) { + // Wire type check. + if (wire_type == upb_types[f->type].native_wire_type || + (wire_type == UPB_WIRE_TYPE_DELIMITED && + upb_types[f->type].is_numeric)) { + // Wire type is ok. + } else { + f = NULL; + } + } // There are no explicit "startseq" or "endseq" markers in protobuf // streams, so we have to infer them by noticing when a repeated field // starts or ends. - if (d->dispatcher.top->is_sequence && d->dispatcher.top->f != f) { + upb_dispatcher_frame *fr = d->dispatcher.top; + if (fr->is_sequence && fr->f != f) { upb_dispatch_endseq(&d->dispatcher); upb_decoder_setmsgend(d); } - if (f && f->repeated && d->dispatcher.top->f != f) { + if (f && f->repeated && (!fr->is_sequence || fr->f != f)) { uint64_t old_end = d->dispatcher.top->end_ofs; upb_dispatcher_frame *fr = upb_dispatch_startseq(&d->dispatcher, f); if (wire_type != UPB_WIRE_TYPE_DELIMITED || @@ -334,7 +458,8 @@ INLINE upb_fhandlers *upb_decode_tag(upb_decoder *d) { fr->end_ofs = old_end; } else { // Packed primitive field. - fr->end_ofs = upb_decoder_offset(d) + upb_decode_varint(d); + uint32_t len = upb_decode_varint32(d); + fr->end_ofs = upb_decoder_offset(d) + len; fr->is_packed = true; } upb_decoder_setmsgend(d); @@ -343,14 +468,20 @@ INLINE upb_fhandlers *upb_decode_tag(upb_decoder *d) { if (f) return f; // Unknown field. + if (fieldnum == 0 || fieldnum > UPB_MAX_FIELDNUMBER) + upb_decoder_abortjmp(d, "Invalid field number"); switch (wire_type) { case UPB_WIRE_TYPE_VARINT: upb_decode_varint(d); break; - case UPB_WIRE_TYPE_32BIT: upb_decoder_advance(d, 4); break; - case UPB_WIRE_TYPE_64BIT: upb_decoder_advance(d, 8); break; + case UPB_WIRE_TYPE_32BIT: upb_decoder_discard(d, 4); break; + case UPB_WIRE_TYPE_64BIT: upb_decoder_discard(d, 8); break; case UPB_WIRE_TYPE_DELIMITED: - upb_decoder_advance(d, upb_decode_varint32(d)); break; + upb_decoder_discard(d, upb_decode_varint32(d)); break; + case UPB_WIRE_TYPE_START_GROUP: + upb_decoder_abortjmp(d, "Can't handle unknown groups yet"); + case UPB_WIRE_TYPE_END_GROUP: + upb_decoder_abortjmp(d, "Unmatched ENDGROUP tag"); default: - upb_decoder_abort(d, "Invalid wire type"); + upb_decoder_abortjmp(d, "Invalid wire type"); } // TODO: deliver to unknown field callback. upb_decoder_checkpoint(d); @@ -358,16 +489,22 @@ INLINE upb_fhandlers *upb_decode_tag(upb_decoder *d) { } } -void upb_decoder_decode(upb_decoder *d, upb_status *status) { - if (sigsetjmp(d->exitjmp, 0)) { assert(!upb_ok(status)); return; } - d->status = status; +upb_success_t upb_decoder_decode(upb_decoder *d) { + assert(d->input); + if (sigsetjmp(d->exitjmp, 0)) { + assert(!upb_ok(&d->status)); + return UPB_ERROR; + } upb_dispatch_startmsg(&d->dispatcher); // Prime the buf so we can hit the JIT immediately. upb_trypullbuf(d); upb_fhandlers *f = d->dispatcher.top->f; - while(1) { // Main loop: executed once per tag/field pair. + while(1) { upb_decoder_checkdelim(d); +#ifdef UPB_USE_JIT_X64 upb_decoder_enterjit(d); + upb_decoder_checkpoint(d); +#endif if (!d->top_is_packed) f = upb_decode_tag(d); if (!f) { // Sucessful EOF. We may need to dispatch a top-level implicit frame. @@ -375,64 +512,46 @@ void upb_decoder_decode(upb_decoder *d, upb_status *status) { assert(d->dispatcher.top->is_sequence); upb_dispatch_endseq(&d->dispatcher); } - return; + return UPB_OK; } f->decode(d, f); upb_decoder_checkpoint(d); } } -static void upb_decoder_skip(void *_d, upb_dispatcher_frame *f) { - upb_decoder *d = _d; - if (f->end_ofs != UPB_NONDELIMITED) { - upb_decoder_skipto(d, d->dispatcher.top->end_ofs); - } else { - // TODO: how to support skipping groups? Dispatcher could drop callbacks, - // or it could be special-cased inside the decoder. - } +void upb_decoder_init(upb_decoder *d) { + upb_status_init(&d->status); + upb_dispatcher_init(&d->dispatcher, &d->status, &upb_decoder_exitjmp2, d); + d->plan = NULL; + d->input = NULL; } -void upb_decoder_init(upb_decoder *d, upb_handlers *handlers) { - upb_dispatcher_init( - &d->dispatcher, handlers, upb_decoder_skip, upb_decoder_exit2, d); -#ifdef UPB_USE_JIT_X64 - d->jit_code = NULL; - if (d->dispatcher.handlers->should_jit) upb_decoder_makejit(d); -#endif - // Set function pointers for each field's decode function. - for (int i = 0; i < handlers->msgs_len; i++) { - upb_mhandlers *m = handlers->msgs[i]; - for(upb_inttable_iter i = upb_inttable_begin(&m->fieldtab); !upb_inttable_done(i); - i = upb_inttable_next(&m->fieldtab, i)) { - upb_fhandlers *f = upb_inttable_iter_value(i); -#define F(type) &upb_decode_ ## type - static void *fptrs[] = {&upb_endgroup, F(DOUBLE), F(FLOAT), F(INT64), - F(UINT64), F(INT32), F(FIXED64), F(FIXED32), F(BOOL), F(STRING), - F(GROUP), F(MESSAGE), F(STRING), F(UINT32), F(ENUM), F(SFIXED32), - F(SFIXED64), F(SINT32), F(SINT64)}; - f->decode = fptrs[f->type]; - } - } +void upb_decoder_resetplan(upb_decoder *d, upb_decoderplan *p, int msg_offset) { + assert(msg_offset >= 0); + assert(msg_offset < p->handlers->msgs_len); + d->plan = p; + d->msg_offset = msg_offset; + d->input = NULL; } -void upb_decoder_reset(upb_decoder *d, upb_byteregion *input, void *closure) { - upb_dispatcher_frame *f = upb_dispatcher_reset(&d->dispatcher, closure); +void upb_decoder_resetinput(upb_decoder *d, upb_byteregion *input, + void *closure) { + assert(d->plan); + upb_dispatcher_frame *f = + upb_dispatcher_reset(&d->dispatcher, closure, d->plan->handlers->msgs[0]); + upb_status_clear(&d->status); f->end_ofs = UPB_NONDELIMITED; d->input = input; - d->bufstart_ofs = upb_byteregion_startofs(input); - d->buf = NULL; - d->ptr = NULL; - d->end = NULL; // Force a buffer pull. - d->delim_end = NULL; // But don't let end-of-message get triggered. d->str_byteregion.bytesrc = input->bytesrc; -#ifdef UPB_USE_JIT_X64 - d->jit_end = NULL; -#endif + + // Protect against assert in skiptonewbuf(). + d->bufstart_ofs = 0; + d->ptr = NULL; + d->buf = NULL; + upb_decoder_skiptonewbuf(d, upb_byteregion_startofs(input)); } void upb_decoder_uninit(upb_decoder *d) { -#ifdef UPB_USE_JIT_X64 - if (d->dispatcher.handlers->should_jit) upb_decoder_freejit(d); -#endif upb_dispatcher_uninit(&d->dispatcher); + upb_status_uninit(&d->status); } diff --git a/upb/pb/decoder.h b/upb/pb/decoder.h index c35bec4..13e5774 100644 --- a/upb/pb/decoder.h +++ b/upb/pb/decoder.h @@ -21,15 +21,43 @@ extern "C" { #endif -/* upb_decoder *****************************************************************/ +/* upb_decoderplan ************************************************************/ + +// A decoderplan contains whatever data structures and generated (JIT-ted) code +// are necessary to decode protobuf data of a specific type to a specific set +// of handlers. By generating the plan ahead of time, we avoid having to +// redo this work every time we decode. +// +// A decoderplan is threadsafe, meaning that it can be used concurrently by +// different upb_decoders in different threads. However, the upb_decoders are +// *not* thread-safe. +struct _upb_decoderplan; +typedef struct _upb_decoderplan upb_decoderplan; + +// TODO: add parameter for a list of other decoder plans that we can share +// generated code with. +upb_decoderplan *upb_decoderplan_new(upb_handlers *h, bool allowjit); +void upb_decoderplan_unref(upb_decoderplan *p); + +// Returns true if the plan contains JIT-ted code. This may not be the same as +// the "allowjit" parameter to the constructor if support for JIT-ting was not +// compiled in. +bool upb_decoderplan_hasjitcode(upb_decoderplan *p); + + +/* upb_decoder ****************************************************************/ struct dasm_State; typedef struct _upb_decoder { - upb_byteregion *input; // Input data (serialized). - upb_dispatcher dispatcher; // Dispatcher to which we push parsed data. - upb_status *status; // Where we will store any errors that occur. - upb_byteregion str_byteregion; // For passing string data to callbacks. + upb_decoderplan *plan; + int msg_offset; // Which message from the plan is top-level. + upb_byteregion *input; // Input data (serialized), not owned. + upb_dispatcher dispatcher; // Dispatcher to which we push parsed data. + upb_status status; // Where we store errors that occur. + upb_byteregion str_byteregion; // For passing string data to callbacks. + + upb_inttable *dispatch_table; // Current input buffer and its stream offset. const char *buf, *ptr, *end; @@ -37,40 +65,64 @@ typedef struct _upb_decoder { // End of the delimited region, relative to ptr, or NULL if not in this buf. const char *delim_end; + // True if the top stack frame represents a packed field. bool top_is_packed; #ifdef UPB_USE_JIT_X64 // For JIT, which doesn't do bounds checks in the middle of parsing a field. const char *jit_end, *effective_end; // == MIN(jit_end, submsg_end) - - // JIT-generated machine code (else NULL). - char *jit_code; - size_t jit_size; - char *debug_info; - - struct dasm_State *dynasm; #endif // For exiting the decoder on error. sigjmp_buf exitjmp; } upb_decoder; -// Initializes/uninitializes a decoder for calling into the given handlers -// or to write into the given msgdef, given its accessors). Takes a ref -// on the handlers. -void upb_decoder_init(upb_decoder *d, upb_handlers *h); +void upb_decoder_init(upb_decoder *d); void upb_decoder_uninit(upb_decoder *d); -// Resets the internal state of an already-allocated decoder. This puts it in a -// state where it has not seen any data, and expects the next data to be from -// the beginning of a new protobuf. Decoders must be reset before they can be -// used. A decoder can be reset multiple times. "input" must live until the -// decoder is reset again (or destroyed). -void upb_decoder_reset(upb_decoder *d, upb_byteregion *input, void *closure); +// Resets the plan that the decoder will parse from. "msg_offset" indicates +// which message from the plan will be used as the top-level message. +// +// This will also reset the decoder's input to be uninitialized -- +// upb_decoder_resetinput() must be called before parsing can occur. The plan +// must live until the decoder is destroyed or reset to a different plan. +// +// Must be called before upb_decoder_resetinput() or upb_decoder_decode(). +void upb_decoder_resetplan(upb_decoder *d, upb_decoderplan *p, int msg_offset); + +// Resets the input of an already-allocated decoder. This puts it in a state +// where it has not seen any data, and expects the next data to be from the +// beginning of a new protobuf. Decoders must have their input reset before +// they can be used. A decoder can have its input reset multiple times. +// "input" must live until the decoder is destroyed or has it input reset +// again. "c" is the closure that will be passed to the handlers. +// +// Must be called before upb_decoder_decode(). +void upb_decoder_resetinput(upb_decoder *d, upb_byteregion *input, void *c); + +// Decodes serialized data (calling handlers as the data is parsed), returning +// the success of the operation (call upb_decoder_status() for details). +upb_success_t upb_decoder_decode(upb_decoder *d); + +INLINE const upb_status *upb_decoder_status(upb_decoder *d) { + return &d->status; +} + +// Implementation details + +struct _upb_decoderplan { + upb_handlers *handlers; // owns reference. + +#ifdef UPB_USE_JIT_X64 + // JIT-generated machine code (else NULL). + char *jit_code; + size_t jit_size; + char *debug_info; -// Decodes serialized data (calling handlers as the data is parsed) until error -// or EOF (see *status for details). -void upb_decoder_decode(upb_decoder *d, upb_status *status); + // This pointer is allocated by dasm_init() and freed by dasm_free(). + struct dasm_State *dynasm; +#endif +}; #ifdef __cplusplus } /* extern "C" */ diff --git a/upb/pb/decoder_x64.dasc b/upb/pb/decoder_x64.dasc index 75e5b6b..807191b 100644 --- a/upb/pb/decoder_x64.dasc +++ b/upb/pb/decoder_x64.dasc @@ -4,20 +4,15 @@ |// Copyright (c) 2011 Google Inc. See LICENSE for details. |// Author: Josh Haberman <jhaberman@gmail.com> |// -|// JIT compiler for upb_decoder on x86. Given a upb_handlers object, -|// generates code specialized to parsing the specific message and -|// calling specific handlers. +|// JIT compiler for upb_decoder on x86. Given a upb_decoderplan object (which +|// contains an embedded set of upb_handlers), generates code specialized to +|// parsing the specific message and calling specific handlers. |// |// Since the JIT can call other functions (the JIT'ted code is not a leaf |// function) we must respect alignment rules. On OS X, this means aligning |// the stack to 16 bytes. -#define UPB_NONE -1 -#define UPB_MULTIPLE -2 -#define UPB_TOPLEVEL_ONE -3 - #include <sys/mman.h> -#include "dynasm/dasm_proto.h" #include "dynasm/dasm_x86.h" #ifndef MAP_ANONYMOUS @@ -73,15 +68,15 @@ gdb_jit_descriptor __jit_debug_descriptor = {1, GDB_JIT_NOACTION, NULL, NULL}; void __attribute__((noinline)) __jit_debug_register_code() { __asm__ __volatile__(""); } -void upb_reg_jit_gdb(upb_decoder *d) { +void upb_reg_jit_gdb(upb_decoderplan *plan) { // Create debug info. size_t elf_len = sizeof(upb_jit_debug_elf_file); - d->debug_info = malloc(elf_len); - memcpy(d->debug_info, upb_jit_debug_elf_file, elf_len); - uint64_t *p = (void*)d->debug_info; - for (; (void*)(p+1) <= (void*)d->debug_info + elf_len; ++p) { - if (*p == 0x12345678) { *p = (uintptr_t)d->jit_code; } - if (*p == 0x321) { *p = d->jit_size; } + plan->debug_info = malloc(elf_len); + memcpy(plan->debug_info, upb_jit_debug_elf_file, elf_len); + uint64_t *p = (void*)plan->debug_info; + for (; (void*)(p+1) <= (void*)plan->debug_info + elf_len; ++p) { + if (*p == 0x12345678) { *p = (uintptr_t)plan->jit_code; } + if (*p == 0x321) { *p = plan->jit_size; } } // Register the JIT-ted code with GDB. @@ -89,7 +84,7 @@ void upb_reg_jit_gdb(upb_decoder *d) { e->next_entry = __jit_debug_descriptor.first_entry; e->prev_entry = NULL; if (e->next_entry) e->next_entry->prev_entry = e; - e->symfile_addr = d->debug_info; + e->symfile_addr = plan->debug_info; e->symfile_size = elf_len; __jit_debug_descriptor.first_entry = e; __jit_debug_descriptor.relevant_entry = e; @@ -99,12 +94,17 @@ void upb_reg_jit_gdb(upb_decoder *d) { #else -void upb_reg_jit_gdb(upb_decoder *d) { - (void)d; +void upb_reg_jit_gdb(upb_decoderplan *plan) { + (void)plan; } #endif +// Has to be a separate function, otherwise GCC will complain about +// expressions like (&foo != NULL) because they will never evaluate +// to false. +static void upb_assert_notnull(void *addr) { assert(addr != NULL); } + |.arch x64 |.actionlist upb_jit_actionlist |.globals UPB_JIT_GLOBAL_ @@ -126,7 +126,7 @@ void upb_reg_jit_gdb(upb_decoder *d) { |// ALL of the code in this file uses these register allocations. |// When we "call" within this file, we do not use regular calling |// conventions, but of course when calling to user callbacks we must. -|.define PTR, rbx +|.define PTR, rbx // Writing this to DECODER->ptr commits our progress. |.define CLOSURE, r12 |.type FRAME, upb_dispatcher_frame, r13 |.type BYTEREGION,upb_byteregion, r14 @@ -134,6 +134,7 @@ void upb_reg_jit_gdb(upb_decoder *d) { |.type STDARRAY, upb_stdarray | |.macro callp, addr +|| upb_assert_notnull(addr); || if ((uintptr_t)addr < 0xffffffff) { | call &addr || } else { @@ -191,11 +192,12 @@ void upb_reg_jit_gdb(upb_decoder *d) { | decode_loaded_varint, 0 | mov ecx, edx | shr ecx, 3 -| and edx, 0x7 +| and edx, 0x7 // For the type check that will happen later. | cmp ecx, m->max_field_number // Bounds-check the field. | ja ->exit_jit // In the future; could be unknown label || if ((uintptr_t)m->tablearray < 0xffffffff) { -| mov rax, qword [rcx*8 + m->tablearray] // TODO: support hybrid array/hash tables. +| // TODO: support hybrid array/hash tables. +| mov rax, qword [rcx*8 + m->tablearray] || } else { | mov64 rax, (uintptr_t)m->tablearray | mov rax, qword [rax + rcx*8] @@ -217,8 +219,9 @@ void upb_reg_jit_gdb(upb_decoder *d) { | lea rax, [FRAME + sizeof(upb_dispatcher_frame)] // rax for shorter addressing. | cmp rax, qword DECODER->dispatcher.limit | jae ->exit_jit // Frame stack overflow. -| mov qword FRAME:rax->f, f -| mov dword FRAME:rax->end_ofs, end_offset_ +| mov64 r8, (uintptr_t)f +| mov qword FRAME:rax->f, r8 +| mov qword FRAME:rax->end_ofs, end_offset_ | mov byte FRAME:rax->is_sequence, is_sequence_ | mov DECODER->dispatcher.top, rax | mov FRAME, rax @@ -294,7 +297,7 @@ void upb_reg_jit_gdb(upb_decoder *d) { | |.macro sethas, reg, hasbit || if (hasbit >= 0) { -| or byte [reg + (hasbit / 8)], (1 << (hasbit % 8)) +| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) || } |.endmacro @@ -304,8 +307,9 @@ void upb_reg_jit_gdb(upb_decoder *d) { #include "upb/msg.h" // Decodes the next val into ARG3, advances PTR. -static void upb_decoder_jit_decodefield(upb_decoder *d, upb_mhandlers *m, - uint8_t type, size_t tag_size) { +static void upb_decoderplan_jit_decodefield(upb_decoderplan *plan, + upb_mhandlers *m, + uint8_t type, size_t tag_size) { // Decode the value into arg 3 for the callback. switch (type) { case UPB_TYPE(DOUBLE): @@ -365,9 +369,9 @@ static void upb_decoder_jit_decodefield(upb_decoder *d, upb_mhandlers *m, // robust checks. | mov ecx, dword [PTR + tag_size] | decode_loaded_varint tag_size - | mov rdi, DECODER->effective_end + | mov rdi, DECODER->end | sub rdi, rax - | cmp ARG3_64, rdi // if (len > d->effective_end - str) + | cmp ARG3_64, rdi // if (len > d->end - str) | ja ->exit_jit // Can't deliver, whole string not in buf. // Update PTR to point past end of string. @@ -401,8 +405,8 @@ static void upb_decoder_jit_decodefield(upb_decoder *d, upb_mhandlers *m, #if 0 // These appear not to speed things up, but keeping around for // further experimentation. -static void upb_decoder_jit_doappend(upb_decoder *d, uint8_t size, - upb_fhandlers *f) { +static void upb_decoderplan_jit_doappend(upb_decoderplan *plan, uint8_t size, + upb_fhandlers *f) { | mov eax, STDARRAY:ARG1_64->len | cmp eax, STDARRAY:ARG1_64->size | jne >2 @@ -434,18 +438,19 @@ static void upb_decoder_jit_doappend(upb_decoder *d, uint8_t size, } #endif -static void upb_decoder_jit_callcb(upb_decoder *d, upb_fhandlers *f) { +static void upb_decoderplan_jit_callcb(upb_decoderplan *plan, + upb_fhandlers *f) { // Call callbacks. if (upb_issubmsgtype(f->type)) { if (f->type == UPB_TYPE(MESSAGE)) { | mov rsi, PTR | sub rsi, DECODER->buf - | add esi, ARG3_32 // = (d->ptr - d->buf) + delim_len + | add rsi, ARG3_64 // = (d->ptr - d->buf) + delim_len } else { assert(f->type == UPB_TYPE(GROUP)); - | mov esi, UPB_NONDELIMITED + | mov rsi, UPB_NONDELIMITED } - | pushframe f, esi, false + | pushframe f, rsi, false // Call startsubmsg handler (if any). if (f->startsubmsg) { @@ -456,15 +461,11 @@ static void upb_decoder_jit_callcb(upb_decoder *d, upb_fhandlers *f) { | mov CLOSURE, rdx } | mov qword FRAME->closure, CLOSURE + // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK + | mov DECODER->ptr, PTR const upb_mhandlers *sub_m = upb_fhandlers_getsubmsg(f); - if (sub_m->jit_parent_field_done_pclabel != UPB_MULTIPLE) { - | jmp =>sub_m->jit_startmsg_pclabel; - } else { - | call =>sub_m->jit_startmsg_pclabel; - } - - |=>f->jit_submsg_done_pclabel: + | call =>sub_m->jit_startmsg_pclabel; // Call endsubmsg handler (if any). if (f->endsubmsg) { @@ -474,6 +475,8 @@ static void upb_decoder_jit_callcb(upb_decoder *d, upb_fhandlers *f) { | callp f->endsubmsg } | popframe upb_fhandlers_getmsg(f) + // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK + | mov DECODER->ptr, PTR } else { | mov ARG1_64, CLOSURE // Test for callbacks we can specialize. @@ -499,15 +502,15 @@ static void upb_decoder_jit_callcb(upb_decoder *d, upb_fhandlers *f) { f->value == &upb_stdmsg_setuint64_r || f->value == &upb_stdmsg_setptr_r || f->value == &upb_stdmsg_setdouble_r) { - upb_decoder_jit_doappend(d, 8, f); + upb_decoderplan_jit_doappend(plan, 8, f); } else if (f->value == &upb_stdmsg_setint32_r || f->value == &upb_stdmsg_setuint32_r || f->value == &upb_stdmsg_setfloat_r) { - upb_decoder_jit_doappend(d, 4, f); + upb_decoderplan_jit_doappend(plan, 4, f); } else if (f->value == &upb_stdmsg_setbool_r) { - upb_decoder_jit_doappend(d, 1, f); + upb_decoderplan_jit_doappend(plan, 1, f); #endif - } else { + } else if (f->value) { // Load closure and fval into arg registers. ||#ifndef NDEBUG ||// Since upb_value carries type information in debug mode @@ -519,14 +522,15 @@ static void upb_decoder_jit_callcb(upb_decoder *d, upb_fhandlers *f) { | callp f->value } | sethas CLOSURE, f->valuehasbit + // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK + | mov DECODER->ptr, PTR } - // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK } // PTR should point to the beginning of the tag. -static void upb_decoder_jit_field(upb_decoder *d, uint32_t tag, - uint32_t next_tag, upb_mhandlers *m, - upb_fhandlers *f, upb_fhandlers *next_f) { +static void upb_decoderplan_jit_field(upb_decoderplan *plan, uint64_t tag, + uint64_t next_tag, upb_mhandlers *m, + upb_fhandlers *f, upb_fhandlers *next_f) { // PC-label for the dispatch table. // We check the wire type (which must be loaded in edx) because the // table is keyed on field number, not type. @@ -535,8 +539,8 @@ static void upb_decoder_jit_field(upb_decoder *d, uint32_t tag, | jne ->exit_jit // In the future: could be an unknown field or packed. |=>f->jit_pclabel_notypecheck: if (f->repeated) { - | mov esi, FRAME->end_ofs - | pushframe f, esi, true + | mov rsi, FRAME->end_ofs + | pushframe f, rsi, true if (f->startseq) { | mov ARG1_64, CLOSURE | loadfval f @@ -555,8 +559,8 @@ static void upb_decoder_jit_field(upb_decoder *d, uint32_t tag, return; } - upb_decoder_jit_decodefield(d, m, f->type, tag_size); - upb_decoder_jit_callcb(d, f); + upb_decoderplan_jit_decodefield(plan, m, f->type, tag_size); + upb_decoderplan_jit_callcb(plan, f); // Epilogue: load next tag, check for repeated field. | check_eob m @@ -586,13 +590,11 @@ static int upb_compare_uint32(const void *a, const void *b) { return *(uint32_t*)a - *(uint32_t*)b; } -static void upb_decoder_jit_msg(upb_decoder *d, upb_mhandlers *m) { +static void upb_decoderplan_jit_msg(upb_decoderplan *plan, upb_mhandlers *m) { |=>m->jit_startmsg_pclabel: + // There was a call to get here, so we need to align the stack. + | sub rsp, 8 - if (m->jit_parent_field_done_pclabel == UPB_MULTIPLE) { - // There was a call to get here, so we need to align the stack. - | sub rsp, 8 - } // Call startmsg handler (if any): if (m->startmsg) { // upb_flow_t startmsg(void *closure); @@ -615,23 +617,30 @@ static void upb_decoder_jit_msg(upb_decoder *d, upb_mhandlers *m) { int num_keys = upb_inttable_count(&m->fieldtab); uint32_t *keys = malloc(num_keys * sizeof(*keys)); int idx = 0; - for(upb_inttable_iter i = upb_inttable_begin(&m->fieldtab); !upb_inttable_done(i); + for(upb_inttable_iter i = upb_inttable_begin(&m->fieldtab); + !upb_inttable_done(i); i = upb_inttable_next(&m->fieldtab, i)) { keys[idx++] = upb_inttable_iter_key(i); } qsort(keys, num_keys, sizeof(uint32_t), &upb_compare_uint32); upb_fhandlers *last_f = NULL; - uint32_t last_tag = 0; + uint64_t last_encoded_tag = 0; for(int i = 0; i < num_keys; i++) { - uint32_t key = keys[i]; - upb_fhandlers *f = upb_inttable_lookup(&m->fieldtab, key); - uint32_t tag = upb_vencode32(key); - if (last_f) upb_decoder_jit_field(d, last_tag, tag, m, last_f, f); - last_tag = tag; + uint32_t fieldnum = keys[i]; + upb_itofhandlers_ent *e = upb_inttable_lookup(&m->fieldtab, fieldnum); + upb_fhandlers *f = e->f; + assert(f->number == fieldnum); + uint32_t tag = (f->number << 3) | upb_types[f->type].native_wire_type; + uint64_t encoded_tag = upb_vencode32(tag); + // No tag should be greater than 5 bytes. + assert(encoded_tag <= 0xffffffffff); + if (last_f) upb_decoderplan_jit_field( + plan, last_encoded_tag, encoded_tag, m, last_f, f); + last_encoded_tag = encoded_tag; last_f = f; } - upb_decoder_jit_field(d, last_tag, 0, m, last_f, NULL); + upb_decoderplan_jit_field(plan, last_encoded_tag, 0, m, last_f, NULL); free(keys); @@ -655,22 +664,29 @@ static void upb_decoder_jit_msg(upb_decoder *d, upb_mhandlers *m) { | callp m->endmsg } - if (m->jit_parent_field_done_pclabel == UPB_MULTIPLE) { - // Counter previous alignment. - | add rsp, 8 - | ret - } else if (m->jit_parent_field_done_pclabel == UPB_TOPLEVEL_ONE) { - | jmp ->exit_jit - } else { - | jmp =>m->jit_parent_field_done_pclabel + if (m->is_group) { + // Advance past the "end group" tag. + // TODO: Handle UPB_BREAK + | mov DECODER->ptr, PTR } + // Counter previous alignment. + | add rsp, 8 + | ret } -static const char *dbgfmt = - "JIT encountered unknown field! wt=%d, fn=%d\n"; - -static void upb_decoder_jit(upb_decoder *d) { +static void upb_decoderplan_jit(upb_decoderplan *plan) { + // The JIT prologue/epilogue trampoline that is generated in this function + // does not depend on the handlers, so it will never vary. Ideally we would + // put it in an object file and just link it into upb so we could have only a + // single copy of it instead of one copy for each decoderplan. But our + // options for doing that are undesirable: GCC inline assembly is + // complicated, not portable to other compilers, and comes with subtle + // caveats about incorrect things what the optimizer might do if you eg. + // execute non-local jumps. Putting this code in a .s file would force us to + // calculate the structure offsets ourself instead of symbolically + // (ie. [r15 + 0xcd] instead of DECODER->ptr). So we tolerate a bit of + // unnecessary duplication/redundancy. | push rbp | mov rbp, rsp | push r15 @@ -686,18 +702,14 @@ static void upb_decoder_jit(upb_decoder *d) { | mov CLOSURE, FRAME->closure | mov PTR, DECODER->ptr - upb_handlers *h = d->dispatcher.handlers; - if (h->msgs[0]->jit_parent_field_done_pclabel == UPB_MULTIPLE) { - | call =>h->msgs[0]->jit_startmsg_pclabel - | jmp ->exit_jit - } - // TODO: push return addresses for re-entry (will be necessary for multiple // buffer support). - for (int i = 0; i < h->msgs_len; i++) upb_decoder_jit_msg(d, h->msgs[i]); + | call ARG2_64 |->exit_jit: - | mov DECODER->ptr, PTR + // Restore stack pointer to where it was before any "call" instructions + // inside our generated code. + | lea rsp, [rbp - 48] // Counter previous alignment. | add rsp, 8 | pop rbx @@ -707,122 +719,128 @@ static void upb_decoder_jit(upb_decoder *d) { | pop r15 | leave | ret - |=>0: - | mov rdi, stderr - | mov rsi, dbgfmt - | callp fprintf - | callp abort + + upb_handlers *h = plan->handlers; + for (int i = 0; i < h->msgs_len; i++) + upb_decoderplan_jit_msg(plan, h->msgs[i]); } -void upb_decoder_jit_assignfieldlabs(upb_fhandlers *f, - uint32_t *pclabel_count) { +static void upb_decoderplan_jit_assignfieldlabs(upb_fhandlers *f, + uint32_t *pclabel_count) { f->jit_pclabel = (*pclabel_count)++; f->jit_pclabel_notypecheck = (*pclabel_count)++; - f->jit_submsg_done_pclabel = (*pclabel_count)++; } -void upb_decoder_jit_assignmsglabs(upb_mhandlers *m, uint32_t *pclabel_count) { +static void upb_decoderplan_jit_assignmsglabs(upb_mhandlers *m, + uint32_t *pclabel_count) { m->jit_startmsg_pclabel = (*pclabel_count)++; m->jit_endofbuf_pclabel = (*pclabel_count)++; m->jit_endofmsg_pclabel = (*pclabel_count)++; m->jit_dyndispatch_pclabel = (*pclabel_count)++; m->jit_unknownfield_pclabel = (*pclabel_count)++; - m->jit_parent_field_done_pclabel = UPB_NONE; m->max_field_number = 0; upb_inttable_iter i; for(i = upb_inttable_begin(&m->fieldtab); !upb_inttable_done(i); i = upb_inttable_next(&m->fieldtab, i)) { uint32_t key = upb_inttable_iter_key(i); m->max_field_number = UPB_MAX(m->max_field_number, key); - upb_fhandlers *f = upb_inttable_iter_value(i); - upb_decoder_jit_assignfieldlabs(f, pclabel_count); + upb_itofhandlers_ent *e = upb_inttable_iter_value(i); + upb_decoderplan_jit_assignfieldlabs(e->f, pclabel_count); } - // XXX: Won't work for large field numbers; will need to use a upb_table. + // TODO: support large field numbers by either using a hash table or + // generating code for a binary search. For now large field numbers + // will just fall back to the table decoder. + m->max_field_number = UPB_MIN(m->max_field_number, 16000); m->tablearray = malloc((m->max_field_number + 1) * sizeof(void*)); } -// Second pass: for messages that have only one parent, link them to the field -// from which they are called. -void upb_decoder_jit_assignmsglabs2(upb_mhandlers *m) { - upb_inttable_iter i; - for(i = upb_inttable_begin(&m->fieldtab); !upb_inttable_done(i); - i = upb_inttable_next(&m->fieldtab, i)) { - upb_fhandlers *f = upb_inttable_iter_value(i); - if (upb_issubmsgtype(f->type)) { - upb_mhandlers *sub_m = upb_fhandlers_getsubmsg(f); - if (sub_m->jit_parent_field_done_pclabel == UPB_NONE) { - sub_m->jit_parent_field_done_pclabel = f->jit_submsg_done_pclabel; - } else { - sub_m->jit_parent_field_done_pclabel = UPB_MULTIPLE; - } - } - } -} - -void upb_decoder_makejit(upb_decoder *d) { - d->debug_info = NULL; +static void upb_decoderplan_makejit(upb_decoderplan *plan) { + plan->debug_info = NULL; // Assign pclabels. - uint32_t pclabel_count = 1; - upb_handlers *h = d->dispatcher.handlers; + uint32_t pclabel_count = 0; + upb_handlers *h = plan->handlers; for (int i = 0; i < h->msgs_len; i++) - upb_decoder_jit_assignmsglabs(h->msgs[i], &pclabel_count); - for (int i = 0; i < h->msgs_len; i++) - upb_decoder_jit_assignmsglabs2(h->msgs[i]); - - if (h->msgs[0]->jit_parent_field_done_pclabel == UPB_NONE) { - h->msgs[0]->jit_parent_field_done_pclabel = UPB_TOPLEVEL_ONE; - } + upb_decoderplan_jit_assignmsglabs(h->msgs[i], &pclabel_count); void **globals = malloc(UPB_JIT_GLOBAL__MAX * sizeof(*globals)); - dasm_init(d, 1); - dasm_setupglobal(d, globals, UPB_JIT_GLOBAL__MAX); - dasm_growpc(d, pclabel_count); - dasm_setup(d, upb_jit_actionlist); + dasm_init(plan, 1); + dasm_setupglobal(plan, globals, UPB_JIT_GLOBAL__MAX); + dasm_growpc(plan, pclabel_count); + dasm_setup(plan, upb_jit_actionlist); - upb_decoder_jit(d); + upb_decoderplan_jit(plan); - dasm_link(d, &d->jit_size); + int dasm_status = dasm_link(plan, &plan->jit_size); + (void)dasm_status; + assert(dasm_status == DASM_S_OK); - d->jit_code = mmap(NULL, d->jit_size, PROT_READ | PROT_WRITE, - MAP_32BIT | MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); + plan->jit_code = mmap(NULL, plan->jit_size, PROT_READ | PROT_WRITE, + MAP_32BIT | MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - upb_reg_jit_gdb(d); + upb_reg_jit_gdb(plan); - dasm_encode(d, d->jit_code); + dasm_encode(plan, plan->jit_code); // Create dispatch tables. for (int i = 0; i < h->msgs_len; i++) { upb_mhandlers *m = h->msgs[i]; + m->jit_func = + plan->jit_code + dasm_getpclabel(plan, m->jit_startmsg_pclabel); for (uint32_t j = 0; j <= m->max_field_number; j++) { - upb_fhandlers *f = NULL; - for (int k = 0; k < 8; k++) { - f = upb_inttable_lookup(&m->fieldtab, (j << 3) | k); - if (f) break; - } + upb_itofhandlers_ent *e = upb_inttable_lookup(&m->fieldtab, j); + upb_fhandlers *f = e ? e->f : NULL; if (f) { - m->tablearray[j] = d->jit_code + dasm_getpclabel(d, f->jit_pclabel); + m->tablearray[j] = + plan->jit_code + dasm_getpclabel(plan, f->jit_pclabel); } else { - // Don't handle unknown fields yet. - m->tablearray[j] = d->jit_code + dasm_getpclabel(d, 0); + // TODO: extend the JIT to handle unknown fields. + // For the moment we exit the JIT for any unknown field. + m->tablearray[j] = globals[UPB_JIT_GLOBAL_exit_jit]; } } } - dasm_free(d); + dasm_free(plan); free(globals); - mprotect(d->jit_code, d->jit_size, PROT_EXEC | PROT_READ); + mprotect(plan->jit_code, plan->jit_size, PROT_EXEC | PROT_READ); // View with: objdump -M intel -D -b binary -mi386 -Mx86-64 /tmp/machine-code // Or: ndisasm -b 64 /tmp/machine-code FILE *f = fopen("/tmp/machine-code", "wb"); - fwrite(d->jit_code, d->jit_size, 1, f); + fwrite(plan->jit_code, plan->jit_size, 1, f); fclose(f); } -void upb_decoder_freejit(upb_decoder *d) { - munmap(d->jit_code, d->jit_size); - free(d->debug_info); +static void upb_decoderplan_freejit(upb_decoderplan *plan) { + munmap(plan->jit_code, plan->jit_size); + free(plan->debug_info); // TODO: unregister } + +static void upb_decoder_enterjit(upb_decoder *d) { + if (d->plan->jit_code && + d->dispatcher.top == d->dispatcher.stack && + d->ptr && d->ptr < d->jit_end) { +#ifndef NDEBUG + register uint64_t rbx asm ("rbx") = 11; + register uint64_t r12 asm ("r12") = 12; + register uint64_t r13 asm ("r13") = 13; + register uint64_t r14 asm ("r14") = 14; + register uint64_t r15 asm ("r15") = 15; +#endif + // Decodes as many fields as possible, updating d->ptr appropriately, + // before falling through to the slow(er) path. + void (*upb_jit_decode)(upb_decoder *d, void*) = (void*)d->plan->jit_code; + upb_jit_decode(d, d->plan->handlers->msgs[d->msg_offset]->jit_func); + assert(d->ptr <= d->end); + + // Test that callee-save registers were properly restored. + assert(rbx == 11); + assert(r12 == 12); + assert(r13 == 13); + assert(r14 == 14); + assert(r15 == 15); + } +} diff --git a/upb/pb/glue.c b/upb/pb/glue.c index 3176355..4949fe3 100644 --- a/upb/pb/glue.c +++ b/upb/pb/glue.c @@ -12,8 +12,8 @@ #include "upb/pb/glue.h" #include "upb/pb/textprinter.h" -void upb_strtomsg(const char *str, size_t len, void *msg, const upb_msgdef *md, - upb_status *status) { +bool upb_strtomsg(const char *str, size_t len, void *msg, const upb_msgdef *md, + bool allow_jit, upb_status *status) { upb_stringsrc strsrc; upb_stringsrc_init(&strsrc); upb_stringsrc_reset(&strsrc, str, len); @@ -21,13 +21,21 @@ void upb_strtomsg(const char *str, size_t len, void *msg, const upb_msgdef *md, upb_decoder d; upb_handlers *h = upb_handlers_new(); upb_accessors_reghandlers(h, md); - upb_decoder_init(&d, h); + upb_decoderplan *p = upb_decoderplan_new(h, allow_jit); + upb_decoder_init(&d); upb_handlers_unref(h); - upb_decoder_reset(&d, upb_stringsrc_allbytes(&strsrc), msg); - upb_decoder_decode(&d, status); + upb_decoder_resetplan(&d, p, 0); + upb_decoder_resetinput(&d, upb_stringsrc_allbytes(&strsrc), msg); + upb_success_t ret = upb_decoder_decode(&d); + // stringsrc and the handlers registered by upb_accessors_reghandlers() + // should not suspend. + assert((ret == UPB_OK) == upb_ok(upb_decoder_status(&d))); + if (status) upb_status_copy(status, upb_decoder_status(&d)); upb_stringsrc_uninit(&strsrc); upb_decoder_uninit(&d); + upb_decoderplan_unref(p); + return ret == UPB_OK; } void *upb_filetonewmsg(const char *fname, const upb_msgdef *md, upb_status *s) { @@ -35,7 +43,7 @@ void *upb_filetonewmsg(const char *fname, const upb_msgdef *md, upb_status *s) { size_t len; char *data = upb_readfile(fname, &len); if (!data) goto err; - upb_strtomsg(data, len, msg, md, s); + upb_strtomsg(data, len, msg, md, false, s); if (!upb_ok(s)) goto err; return msg; @@ -69,7 +77,6 @@ void upb_msgtotext(upb_string *str, upb_msg *msg, upb_msgdef *md, } #endif -// TODO: read->load. upb_def **upb_load_defs_from_descriptor(const char *str, size_t len, int *n, upb_status *status) { upb_stringsrc strsrc; @@ -79,17 +86,21 @@ upb_def **upb_load_defs_from_descriptor(const char *str, size_t len, int *n, upb_handlers *h = upb_handlers_new(); upb_descreader_reghandlers(h); + upb_decoderplan *p = upb_decoderplan_new(h, false); upb_decoder d; - upb_decoder_init(&d, h); + upb_decoder_init(&d); upb_handlers_unref(h); upb_descreader r; upb_descreader_init(&r); - upb_decoder_reset(&d, upb_stringsrc_allbytes(&strsrc), &r); + upb_decoder_resetplan(&d, p, 0); + upb_decoder_resetinput(&d, upb_stringsrc_allbytes(&strsrc), &r); - upb_decoder_decode(&d, status); + upb_success_t ret = upb_decoder_decode(&d); + if (status) upb_status_copy(status, upb_decoder_status(&d)); upb_stringsrc_uninit(&strsrc); upb_decoder_uninit(&d); - if (!upb_ok(status)) { + upb_decoderplan_unref(p); + if (ret != UPB_OK) { upb_descreader_uninit(&r); return NULL; } diff --git a/upb/pb/glue.h b/upb/pb/glue.h index 38e8d8e..ff8c85e 100644 --- a/upb/pb/glue.h +++ b/upb/pb/glue.h @@ -36,8 +36,8 @@ extern "C" { // Decodes the given string, which must be in protobuf binary format, to the // given upb_msg with msgdef "md", storing the status of the operation in "s". -void upb_strtomsg(const char *str, size_t len, void *msg, - const upb_msgdef *md, upb_status *s); +bool upb_strtomsg(const char *str, size_t len, void *msg, + const upb_msgdef *md, bool allow_jit, upb_status *s); // Parses the given file into a new message of the given type. Caller owns // the returned message (or NULL if an error occurred). diff --git a/upb/pb/varint.h b/upb/pb/varint.h index 19977e9..815a7a1 100644 --- a/upb/pb/varint.h +++ b/upb/pb/varint.h @@ -19,6 +19,18 @@ extern "C" { #endif +// The maximum number of bytes that it takes to encode a 64-bit varint. +// Note that with a better encoding this could be 9 (TODO: write up a +// wiki document about this). +#define UPB_PB_VARINT_MAX_LEN 10 + +/* Zig-zag encoding/decoding **************************************************/ + +INLINE int32_t upb_zzdec_32(uint32_t n) { return (n >> 1) ^ -(int32_t)(n & 1); } +INLINE int64_t upb_zzdec_64(uint64_t n) { return (n >> 1) ^ -(int64_t)(n & 1); } +INLINE uint32_t upb_zzenc_32(int32_t n) { return (n << 1) ^ (n >> 31); } +INLINE uint64_t upb_zzenc_64(int64_t n) { return (n << 1) ^ (n >> 63); } + /* Decoding *******************************************************************/ // All decoding functions return this struct by value. @@ -56,7 +68,7 @@ done: INLINE upb_decoderet upb_vdecode_branch64(const char *p) { uint64_t val; uint64_t b; - upb_decoderet r = {(void*)0, 0}; + upb_decoderet r = {NULL, 0}; b = *(p++); val = (b & 0x7f) ; if(!(b & 0x80)) goto done; b = *(p++); val |= (b & 0x7f) << 7; if(!(b & 0x80)) goto done; b = *(p++); val |= (b & 0x7f) << 14; if(!(b & 0x80)) goto done; @@ -124,17 +136,33 @@ INLINE int upb_value_size(uint64_t val) { return val == 0 ? 1 : high_bit / 8 + 1; } +// Encodes a 64-bit varint into buf (which must be >=UPB_PB_VARINT_MAX_LEN +// bytes long), returning how many bytes were used. +// +// TODO: benchmark and optimize if necessary. +INLINE size_t upb_vencode64(uint64_t val, char *buf) { + if (val == 0) { buf[0] = 0; return 1; } + size_t i = 0; + while (val) { + uint8_t byte = val & 0x7f; + val >>= 7; + if (val) byte |= 0x80; + buf[i++] = byte; + } + return i; +} + // Encodes a 32-bit varint, *not* sign-extended. INLINE uint64_t upb_vencode32(uint32_t val) { + char buf[UPB_PB_VARINT_MAX_LEN]; + size_t bytes = upb_vencode64(val, buf); uint64_t ret = 0; - for (int bitpos = 0; val; bitpos+=8, val >>=7) { - if (bitpos > 0) ret |= (1 << (bitpos-1)); - ret |= (val & 0x7f) << bitpos; - } + assert(bytes <= 5); + memcpy(&ret, buf, bytes); + assert(ret <= 0xffffffffff); return ret; } - #ifdef __cplusplus } /* extern "C" */ #endif |