diff options
Diffstat (limited to 'upb/pb')
| -rw-r--r-- | upb/pb/decoder.c | 252 | ||||
| -rw-r--r-- | upb/pb/decoder.h | 61 | ||||
| -rw-r--r-- | upb/pb/decoder_x64.dasc | 646 | ||||
| -rw-r--r-- | upb/pb/glue.c | 14 | ||||
| -rw-r--r-- | upb/pb/glue.h | 26 | ||||
| -rw-r--r-- | upb/pb/textprinter.c | 217 | ||||
| -rw-r--r-- | upb/pb/textprinter.h | 5 | ||||
| -rw-r--r-- | upb/pb/varint.c | 59 | ||||
| -rw-r--r-- | upb/pb/varint.h | 88 |
9 files changed, 820 insertions, 548 deletions
diff --git a/upb/pb/decoder.c b/upb/pb/decoder.c index 30f7c65..065c495 100644 --- a/upb/pb/decoder.c +++ b/upb/pb/decoder.c @@ -5,17 +5,13 @@ * Author: Josh Haberman <jhaberman@gmail.com> */ +#include <inttypes.h> #include <stddef.h> #include <stdlib.h> #include "upb/bytestream.h" -#include "upb/msg.h" #include "upb/pb/decoder.h" #include "upb/pb/varint.h" -#ifndef UINT32_MAX -#define UINT32_MAX 0xffffffff -#endif - typedef struct { uint8_t native_wire_type; bool is_numeric; @@ -62,11 +58,12 @@ static const upb_decoder_typeinfo upb_decoder_types[] = { #include "upb/pb/decoder_x64.h" #endif -upb_decoderplan *upb_decoderplan_new(upb_handlers *h, bool allowjit) { +upb_decoderplan *upb_decoderplan_new(const upb_handlers *h, bool allowjit) { + UPB_UNUSED(allowjit); upb_decoderplan *p = malloc(sizeof(*p)); + assert(upb_handlers_isfrozen(h)); p->handlers = h; - upb_handlers_ref(h); - h->should_jit = allowjit; + upb_handlers_ref(h, p); #ifdef UPB_USE_JIT_X64 p->jit_code = NULL; if (allowjit) upb_decoderplan_makejit(p); @@ -76,7 +73,7 @@ upb_decoderplan *upb_decoderplan_new(upb_handlers *h, bool allowjit) { void upb_decoderplan_unref(upb_decoderplan *p) { // TODO: make truly refcounted. - upb_handlers_unref(p->handlers); + upb_handlers_unref(p->handlers, p); #ifdef UPB_USE_JIT_X64 if (p->jit_code) upb_decoderplan_freejit(p); #endif @@ -100,8 +97,8 @@ bool upb_decoderplan_hasjitcode(upb_decoderplan *p) { // configuration. But emperically on a Core i7, performance increases 30-50% // with these annotations. Every instance where these appear, gcc 4.2.1 made // the wrong decision and degraded performance in benchmarks. -#define FORCEINLINE static __attribute__((__always_inline__)) -#define NOINLINE static __attribute__((__noinline__)) +#define FORCEINLINE static inline __attribute__((always_inline)) +#define NOINLINE static __attribute__((noinline)) UPB_NORETURN static void upb_decoder_exitjmp(upb_decoder *d) { // Resumable decoder would back out to completed_ptr (and possibly get a @@ -141,14 +138,23 @@ uint64_t upb_decoder_bufendofs(upb_decoder *d) { return d->bufstart_ofs + (d->end - d->buf); } +static bool upb_decoder_islegalend(upb_decoder *d) { + if (d->top == d->stack) return true; + if (d->top - 1 == d->stack && + d->top->is_sequence && !d->top->is_packed) return true; + return false; +} + +// Calculates derived values that we cache for speed. These reflect a +// combination of the current buffer and the stack, so must be called whenever +// either is updated. static void upb_decoder_setmsgend(upb_decoder *d) { - upb_dispatcher_frame *f = d->dispatcher.top; + upb_decoder_frame *f = d->top; size_t delimlen = f->end_ofs - d->bufstart_ofs; size_t buflen = d->end - d->buf; d->delim_end = (f->end_ofs != UPB_NONDELIMITED && delimlen <= buflen) ? d->buf + delimlen : NULL; // NULL if not in this buf. d->top_is_packed = f->is_packed; - d->dispatch_table = &d->dispatcher.msgent->fieldtab; } static void upb_decoder_skiptonewbuf(upb_decoder *d, uint64_t ofs) { @@ -201,11 +207,11 @@ static void upb_pullbuf(upb_decoder *d) { if (!upb_trypullbuf(d)) upb_decoder_abortjmp(d, "Unexpected EOF"); } -void upb_decoder_checkpoint(upb_decoder *d) { +static void upb_decoder_checkpoint(upb_decoder *d) { upb_byteregion_discard(d->input, upb_decoder_offset(d)); } -void upb_decoder_discardto(upb_decoder *d, uint64_t ofs) { +static void upb_decoder_discardto(upb_decoder *d, uint64_t ofs) { if (ofs <= upb_decoder_bufendofs(d)) { upb_decoder_advance(d, ofs - upb_decoder_offset(d)); } else { @@ -214,7 +220,7 @@ void upb_decoder_discardto(upb_decoder *d, uint64_t ofs) { upb_decoder_checkpoint(d); } -void upb_decoder_discard(upb_decoder *d, size_t bytes) { +static void upb_decoder_discard(upb_decoder *d, size_t bytes) { upb_decoder_discardto(d, upb_decoder_offset(d) + bytes); } @@ -259,7 +265,7 @@ done: // Returns true on success or false if we've hit a valid EOF. FORCEINLINE bool upb_trydecode_varint32(upb_decoder *d, uint32_t *val) { if (upb_decoder_bufleft(d) == 0 && - upb_dispatcher_islegalend(&d->dispatcher) && + upb_decoder_islegalend(d) && !upb_trypullbuf(d)) { return false; } @@ -319,21 +325,45 @@ FORCEINLINE uint64_t upb_decode_fixed64(upb_decoder *d) { return u64; // TODO: proper byte swapping for big-endian machines. } -INLINE upb_byteregion *upb_decode_string(upb_decoder *d) { - uint32_t strlen = upb_decode_varint32(d); - uint64_t offset = upb_decoder_offset(d); - if (offset + strlen > upb_byteregion_endofs(d->input)) - upb_decoder_abortjmp(d, "Unexpected EOF"); - upb_byteregion_reset(&d->str_byteregion, d->input, offset, strlen); - // Could make it an option on the callback whether we fetchall() first or not. - if (upb_byteregion_fetchall(&d->str_byteregion) != UPB_BYTE_OK) - upb_decoder_abortjmp(d, "Couldn't fetchall() on string."); - upb_decoder_discardto(d, offset + strlen); - return &d->str_byteregion; +INLINE void upb_push_msg(upb_decoder *d, const upb_fielddef *f, uint64_t end) { + upb_decoder_frame *fr = d->top + 1; + if (!upb_sink_startsubmsg(&d->sink, f) || fr > d->limit) { + upb_decoder_abortjmp(d, "Nesting too deep."); + } + fr->f = f; + fr->is_sequence = false; + fr->is_packed = false; + fr->end_ofs = end; + fr->group_fieldnum = end == UPB_NONDELIMITED ? + (int32_t)upb_fielddef_number(f) : -1; + d->top = fr; + upb_decoder_setmsgend(d); } -INLINE void upb_push_msg(upb_decoder *d, upb_fhandlers *f, uint64_t end) { - upb_dispatch_startsubmsg(&d->dispatcher, f)->end_ofs = end; +INLINE void upb_push_seq(upb_decoder *d, const upb_fielddef *f, bool packed, + uint64_t end_ofs) { + upb_decoder_frame *fr = d->top + 1; + if (!upb_sink_startseq(&d->sink, f) || fr > d->limit) { + upb_decoder_abortjmp(d, "Nesting too deep."); + } + fr->f = f; + fr->is_sequence = true; + fr->group_fieldnum = -1; + fr->is_packed = packed; + fr->end_ofs = end_ofs; + d->top = fr; + upb_decoder_setmsgend(d); +} + +INLINE void upb_pop_submsg(upb_decoder *d) { + upb_sink_endsubmsg(&d->sink, d->top->f); + d->top--; + upb_decoder_setmsgend(d); +} + +INLINE void upb_pop_seq(upb_decoder *d) { + upb_sink_endseq(&d->sink, d->top->f); + d->top--; upb_decoder_setmsgend(d); } @@ -344,13 +374,14 @@ INLINE void upb_push_msg(upb_decoder *d, upb_fhandlers *f, uint64_t end) { // properly sign-extended. We could detect this and error about the data loss, // but proto2 does not do this, so we pass. -#define T(type, wt, valtype, convfunc) \ - INLINE void upb_decode_ ## type(upb_decoder *d, upb_fhandlers *f) { \ - upb_value val; \ - upb_value_set ## valtype(&val, (convfunc)(upb_decode_ ## wt(d))); \ - upb_dispatch_value(&d->dispatcher, f, val); \ +#define T(type, wt, name, convfunc) \ + INLINE void upb_decode_ ## type(upb_decoder *d, const upb_fielddef *f) { \ + upb_sink_put ## name(&d->sink, f, (convfunc)(upb_decode_ ## wt(d))); \ } \ +static double upb_asdouble(uint64_t n) { double d; memcpy(&d, &n, 8); return d; } +static float upb_asfloat(uint32_t n) { float f; memcpy(&f, &n, 4); return f; } + T(INT32, varint, int32, int32_t) T(INT64, varint, int64, int64_t) T(UINT32, varint, uint32, uint32_t) @@ -361,43 +392,44 @@ T(SFIXED32, fixed32, int32, int32_t) T(SFIXED64, fixed64, int64, int64_t) T(BOOL, varint, bool, bool) T(ENUM, varint, int32, int32_t) +T(DOUBLE, fixed64, double, upb_asdouble) +T(FLOAT, fixed32, float, upb_asfloat) T(SINT32, varint, int32, upb_zzdec_32) T(SINT64, varint, int64, upb_zzdec_64) -T(STRING, string, byteregion, upb_byteregion*) - #undef T -INLINE void upb_decode_DOUBLE(upb_decoder *d, upb_fhandlers *f) { - upb_value val; - double dbl; - uint64_t wireval = upb_decode_fixed64(d); - memcpy(&dbl, &wireval, 8); - upb_value_setdouble(&val, dbl); - upb_dispatch_value(&d->dispatcher, f, val); -} - -INLINE void upb_decode_FLOAT(upb_decoder *d, upb_fhandlers *f) { - upb_value val; - float flt; - uint64_t wireval = upb_decode_fixed32(d); - memcpy(&flt, &wireval, 4); - upb_value_setfloat(&val, flt); - upb_dispatch_value(&d->dispatcher, f, val); -} - -static void upb_decode_GROUP(upb_decoder *d, upb_fhandlers *f) { +static void upb_decode_GROUP(upb_decoder *d, const upb_fielddef *f) { upb_push_msg(d, f, UPB_NONDELIMITED); } -static void upb_endgroup(upb_decoder *d, upb_fhandlers *f) { - (void)f; - upb_dispatch_endsubmsg(&d->dispatcher); - upb_decoder_setmsgend(d); -} -static void upb_decode_MESSAGE(upb_decoder *d, upb_fhandlers *f) { + +static void upb_decode_MESSAGE(upb_decoder *d, const upb_fielddef *f) { uint32_t len = upb_decode_varint32(d); upb_push_msg(d, f, upb_decoder_offset(d) + len); } +static void upb_decode_STRING(upb_decoder *d, const upb_fielddef *f) { + uint32_t strlen = upb_decode_varint32(d); + uint64_t offset = upb_decoder_offset(d); + uint64_t end = offset + strlen; + if (end > upb_byteregion_endofs(d->input)) + upb_decoder_abortjmp(d, "Unexpected EOF"); + upb_sink_startstr(&d->sink, f, strlen); + while (strlen > 0) { + if (upb_byteregion_available(d->input, offset) == 0) + upb_pullbuf(d); + size_t len; + const char *ptr = upb_byteregion_getptr(d->input, offset, &len); + len = UPB_MIN(len, strlen); + len = upb_sink_putstring(&d->sink, f, ptr, len); + if (len > strlen) + upb_decoder_abortjmp(d, "Skipped too many bytes."); + offset += len; + strlen -= len; + upb_decoder_discardto(d, offset); + } + upb_sink_endstr(&d->sink, f); +} + /* The main decoding loop *****************************************************/ @@ -410,33 +442,33 @@ static void upb_decoder_checkdelim(upb_decoder *d) { // handler). while (d->delim_end != NULL && d->ptr >= d->delim_end) { if (d->ptr > d->delim_end) upb_decoder_abortjmp(d, "Bad submessage end"); - if (d->dispatcher.top->is_sequence) { - upb_dispatch_endseq(&d->dispatcher); + if (d->top->is_sequence) { + upb_pop_seq(d); } else { - upb_dispatch_endsubmsg(&d->dispatcher); + upb_pop_submsg(d); } - upb_decoder_setmsgend(d); } } -INLINE upb_fhandlers *upb_decode_tag(upb_decoder *d) { +INLINE const upb_fielddef *upb_decode_tag(upb_decoder *d) { while (1) { uint32_t tag; if (!upb_trydecode_varint32(d, &tag)) return NULL; uint8_t wire_type = tag & 0x7; - uint32_t fieldnum = tag >> 3; - const upb_value *val = upb_inttable_lookup32(d->dispatch_table, fieldnum); - upb_fhandlers *f = val ? upb_value_getptr(*val) : NULL; - bool is_packed = false; + uint32_t fieldnum = tag >> 3; const upb_fielddef *f = NULL; + const upb_handlers *h = upb_sink_tophandlers(&d->sink); + f = upb_msgdef_itof(upb_handlers_msgdef(h), fieldnum); + bool packed = false; if (f) { // Wire type check. - if (wire_type == upb_decoder_types[f->type].native_wire_type) { + upb_fieldtype_t type = upb_fielddef_type(f); + if (wire_type == upb_decoder_types[type].native_wire_type) { // Wire type is ok. } else if ((wire_type == UPB_WIRE_TYPE_DELIMITED && - upb_decoder_types[f->type].is_numeric)) { + upb_decoder_types[type].is_numeric)) { // Wire type is ok (and packed). - is_packed = true; + packed = true; } else { f = NULL; } @@ -445,29 +477,24 @@ INLINE upb_fhandlers *upb_decode_tag(upb_decoder *d) { // There are no explicit "startseq" or "endseq" markers in protobuf // streams, so we have to infer them by noticing when a repeated field // starts or ends. - upb_dispatcher_frame *fr = d->dispatcher.top; + upb_decoder_frame *fr = d->top; if (fr->is_sequence && fr->f != f) { - upb_dispatch_endseq(&d->dispatcher); - upb_decoder_setmsgend(d); - fr = d->dispatcher.top; + upb_pop_seq(d); + fr = d->top; } - if (f && f->repeated && !fr->is_sequence) { - upb_dispatcher_frame *fr2 = upb_dispatch_startseq(&d->dispatcher, f); - if (is_packed) { - // Packed primitive field. + + if (f && upb_fielddef_isseq(f) && !fr->is_sequence) { + if (packed) { uint32_t len = upb_decode_varint32(d); - fr2->end_ofs = upb_decoder_offset(d) + len; - fr2->is_packed = true; + upb_push_seq(d, f, true, upb_decoder_offset(d) + len); } else { - // Non-packed field -- this tag pertains to only a single message. - fr2->end_ofs = fr->end_ofs; + upb_push_seq(d, f, false, fr->end_ofs); } - upb_decoder_setmsgend(d); } if (f) return f; - // Unknown field. + // Unknown field or ENDGROUP. if (fieldnum == 0 || fieldnum > UPB_MAX_FIELDNUMBER) upb_decoder_abortjmp(d, "Invalid field number"); switch (wire_type) { @@ -479,7 +506,12 @@ INLINE upb_fhandlers *upb_decode_tag(upb_decoder *d) { case UPB_WIRE_TYPE_START_GROUP: upb_decoder_abortjmp(d, "Can't handle unknown groups yet"); case UPB_WIRE_TYPE_END_GROUP: - upb_decoder_abortjmp(d, "Unmatched ENDGROUP tag"); + if (fieldnum != fr->group_fieldnum) + upb_decoder_abortjmp(d, "Unmatched ENDGROUP tag"); + upb_sink_endsubmsg(&d->sink, fr->f); + d->top--; + upb_decoder_setmsgend(d); + break; default: upb_decoder_abortjmp(d, "Invalid wire type"); } @@ -495,30 +527,30 @@ upb_success_t upb_decoder_decode(upb_decoder *d) { assert(!upb_ok(&d->status)); return UPB_ERROR; } - upb_dispatch_startmsg(&d->dispatcher); + upb_sink_startmsg(&d->sink); // Prime the buf so we can hit the JIT immediately. upb_trypullbuf(d); - upb_fhandlers *f = d->dispatcher.top->f; + const upb_fielddef *f = d->top->f; while(1) { - upb_decoder_checkdelim(d); #ifdef UPB_USE_JIT_X64 upb_decoder_enterjit(d); upb_decoder_checkpoint(d); + upb_decoder_setmsgend(d); #endif + upb_decoder_checkdelim(d); if (!d->top_is_packed) f = upb_decode_tag(d); if (!f) { // Sucessful EOF. We may need to dispatch a top-level implicit frame. - if (d->dispatcher.top->is_sequence) { - assert(d->dispatcher.top == d->dispatcher.stack + 1); - upb_dispatch_endseq(&d->dispatcher); + if (d->top->is_sequence) { + assert(d->sink.top == d->sink.stack + 1); + upb_pop_seq(d); } - assert(d->dispatcher.top == d->dispatcher.stack); - upb_dispatch_endmsg(&d->dispatcher, &d->status); + assert(d->top == d->stack); + upb_sink_endmsg(&d->sink, &d->status); return UPB_OK; } - switch (f->type) { - case UPB_TYPE_ENDGROUP: upb_endgroup(d, f); break; + switch (upb_fielddef_type(f)) { case UPB_TYPE(DOUBLE): upb_decode_DOUBLE(d, f); break; case UPB_TYPE(FLOAT): upb_decode_FLOAT(d, f); break; case UPB_TYPE(INT64): upb_decode_INT64(d, f); break; @@ -545,28 +577,29 @@ upb_success_t upb_decoder_decode(upb_decoder *d) { void upb_decoder_init(upb_decoder *d) { upb_status_init(&d->status); - upb_dispatcher_init(&d->dispatcher, &d->status, &upb_decoder_exitjmp2, d); d->plan = NULL; d->input = NULL; + d->limit = &d->stack[UPB_MAX_NESTING]; } -void upb_decoder_resetplan(upb_decoder *d, upb_decoderplan *p, int msg_offset) { - assert(msg_offset >= 0); - assert(msg_offset < p->handlers->msgs_len); +void upb_decoder_resetplan(upb_decoder *d, upb_decoderplan *p) { d->plan = p; - d->msg_offset = msg_offset; d->input = NULL; + upb_sink_init(&d->sink, p->handlers); } void upb_decoder_resetinput(upb_decoder *d, upb_byteregion *input, - void *closure) { + void *c) { assert(d->plan); - upb_dispatcher_frame *f = - upb_dispatcher_reset(&d->dispatcher, closure, d->plan->handlers->msgs[0]); upb_status_clear(&d->status); - f->end_ofs = UPB_NONDELIMITED; + upb_sink_reset(&d->sink, c); d->input = input; - d->str_byteregion.bytesrc = input->bytesrc; + + d->top = d->stack; + d->top->is_sequence = false; + d->top->is_packed = false; + d->top->group_fieldnum = UINT32_MAX; + d->top->end_ofs = UPB_NONDELIMITED; // Protect against assert in skiptonewbuf(). d->bufstart_ofs = 0; @@ -576,6 +609,5 @@ void upb_decoder_resetinput(upb_decoder *d, upb_byteregion *input, } void upb_decoder_uninit(upb_decoder *d) { - upb_dispatcher_uninit(&d->dispatcher); upb_status_uninit(&d->status); } diff --git a/upb/pb/decoder.h b/upb/pb/decoder.h index df65468..690ebb9 100644 --- a/upb/pb/decoder.h +++ b/upb/pb/decoder.h @@ -13,9 +13,8 @@ #define UPB_DECODER_H_ #include <setjmp.h> -#include <stdbool.h> -#include <stdint.h> -#include "upb/handlers.h" +#include "upb/bytestream.h" +#include "upb/sink.h" #ifdef __cplusplus extern "C" { @@ -34,9 +33,12 @@ extern "C" { struct _upb_decoderplan; typedef struct _upb_decoderplan upb_decoderplan; -// TODO: add parameter for a list of other decoder plans that we can share -// generated code with. -upb_decoderplan *upb_decoderplan_new(upb_handlers *h, bool allowjit); +// TODO(haberman): +// - add support for letting any message in the plan be at the top level. +// - make this object a handlers instead (when bytesrc/bytesink are merged +// into handlers). +// - add support for sharing code with previously-built plans/handlers. +upb_decoderplan *upb_decoderplan_new(const upb_handlers *h, bool allowjit); void upb_decoderplan_unref(upb_decoderplan *p); // Returns true if the plan contains JIT-ted code. This may not be the same as @@ -49,15 +51,28 @@ bool upb_decoderplan_hasjitcode(upb_decoderplan *p); struct dasm_State; +typedef struct { + const upb_fielddef *f; + uint64_t end_ofs; + uint32_t group_fieldnum; // UINT32_MAX for non-groups. + bool is_sequence; // frame represents seq or submsg? (f might be both). + bool is_packed; // !upb_issubmsg(f) && end_ofs != UINT64_MAX + // (strings aren't pushed). +} upb_decoder_frame; + typedef struct _upb_decoder { upb_decoderplan *plan; - int msg_offset; // Which message from the plan is top-level. upb_byteregion *input; // Input data (serialized), not owned. - upb_dispatcher dispatcher; // Dispatcher to which we push parsed data. upb_status status; // Where we store errors that occur. - upb_byteregion str_byteregion; // For passing string data to callbacks. - upb_inttable *dispatch_table; + // Where we push parsed data. + // TODO(haberman): make this a pointer and make upb_decoder_resetinput() take + // one of these instead of a void*. + upb_sink sink; + + // Our internal stack. + upb_decoder_frame *top, *limit; + upb_decoder_frame stack[UPB_MAX_NESTING]; // Current input buffer and its stream offset. const char *buf, *ptr, *end; @@ -70,7 +85,11 @@ typedef struct _upb_decoder { #ifdef UPB_USE_JIT_X64 // For JIT, which doesn't do bounds checks in the middle of parsing a field. - const char *jit_end, *effective_end; // == MIN(jit_end, submsg_end) + const char *jit_end, *effective_end; // == MIN(jit_end, delim_end) + + // Used momentarily by the generated code to store a value while a user + // function is called. + uint32_t tmp_len; #endif // For exiting the decoder on error. @@ -88,7 +107,7 @@ void upb_decoder_uninit(upb_decoder *d); // must live until the decoder is destroyed or reset to a different plan. // // Must be called before upb_decoder_resetinput() or upb_decoder_decode(). -void upb_decoder_resetplan(upb_decoder *d, upb_decoderplan *p, int msg_offset); +void upb_decoder_resetplan(upb_decoder *d, upb_decoderplan *p); // Resets the input of an already-allocated decoder. This puts it in a state // where it has not seen any data, and expects the next data to be from the @@ -111,7 +130,8 @@ INLINE const upb_status *upb_decoder_status(upb_decoder *d) { // Implementation details struct _upb_decoderplan { - upb_handlers *handlers; // owns reference. + // The top-level handlers that this plan calls into. We own a ref. + const upb_handlers *handlers; #ifdef UPB_USE_JIT_X64 // JIT-generated machine code (else NULL). @@ -119,8 +139,23 @@ struct _upb_decoderplan { size_t jit_size; char *debug_info; + // For storing upb_jitmsginfo, which contains per-msg runtime data needed + // by the JIT. + // Maps upb_handlers* -> upb_jitmsginfo. + upb_inttable msginfo; + + // The following members are used only while the JIT is being built. + // This pointer is allocated by dasm_init() and freed by dasm_free(). struct dasm_State *dynasm; + + // For storing pclabel bases while we are building the JIT. + // Maps (upb_handlers* or upb_fielddef*) -> int32 pclabel_base + upb_inttable pclabels; + + // This is not the same as len(pclabels) because the table only contains base + // offsets for each def, but each def can have many pclabels. + uint32_t pclabel_count; #endif }; diff --git a/upb/pb/decoder_x64.dasc b/upb/pb/decoder_x64.dasc index f58e403..cd09cfe 100644 --- a/upb/pb/decoder_x64.dasc +++ b/upb/pb/decoder_x64.dasc @@ -12,6 +12,7 @@ |// function) we must respect alignment rules. All x86-64 systems require |// 16-byte stack alignment. +#include <stdio.h> #include <sys/mman.h> #include "dynasm/dasm_x86.h" @@ -28,6 +29,44 @@ #define MAP_32BIT 0 #endif +// These are used to track jump targets for messages and fields. +enum { + STARTMSG = 0, + AFTER_STARTMSG = 1, + ENDOFBUF = 2, + ENDOFMSG = 3, + DYNDISPATCH = 4, + TOTAL_MSG_PCLABELS = 5, +}; + +enum { + FIELD = 0, + FIELD_NO_TYPECHECK = 1, + TOTAL_FIELD_PCLABELS = 2, +}; + +typedef struct { + uint32_t max_field_number; + // Currently keyed on field number. Could also try keying it + // on encoded or decoded tag, or on encoded field number. + void **tablearray; + // Pointer to the JIT code for parsing this message. + void *jit_func; +} upb_jitmsginfo; + +static uint32_t upb_getpclabel(upb_decoderplan *plan, const void *obj, int n) { + const upb_value *v = upb_inttable_lookupptr(&plan->pclabels, obj); + assert(v); + return upb_value_getuint32(*v) + n; +} + +static upb_jitmsginfo *upb_getmsginfo(upb_decoderplan *plan, + const upb_handlers *h) { + const upb_value *v = upb_inttable_lookupptr(&plan->msginfo, h); + assert(v); + return upb_value_getptr(*v); +} + // To debug JIT-ted code with GDB we need to tell GDB about the JIT-ted code // at runtime. GDB 7.x+ has defined an interface for doing this, and these // structure/function defintions are copied out of gdb/jit.h @@ -66,7 +105,9 @@ typedef struct { gdb_jit_descriptor __jit_debug_descriptor = {1, GDB_JIT_NOACTION, NULL, NULL}; -void __attribute__((noinline)) __jit_debug_register_code() { __asm__ __volatile__(""); } +void __attribute__((noinline)) __jit_debug_register_code() { + __asm__ __volatile__(""); +} void upb_reg_jit_gdb(upb_decoderplan *plan) { // Create debug info. @@ -120,7 +161,8 @@ static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } |.define ARG3_32, edx |.define ARG3_64, rdx |.define ARG4_64, rcx -|.define ARG5_32, r8d +|.define XMMARG1, xmm0 + | |// Register allocation / type map. |// ALL of the code in this file uses these register allocations. @@ -128,13 +170,15 @@ static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } |// conventions, but of course when calling to user callbacks we must. |.define PTR, rbx // Writing this to DECODER->ptr commits our progress. |.define CLOSURE, r12 -|.type FRAME, upb_dispatcher_frame, r13 -|.type BYTEREGION,upb_byteregion, r14 +|.type SINKFRAME, upb_sink_frame, r13 +|.type FRAME, upb_decoder_frame, r14 |.type DECODER, upb_decoder, r15 -|.type STDARRAY, upb_stdarray | |.macro callp, addr || upb_assert_notnull(addr); +|// TODO(haberman): fix this. I believe the predicate we should actually be +|// testing is whether the jump distance is greater than INT32_MAX, not the +|// absolute address of the target. || if ((uintptr_t)addr < 0xffffffff) { | call &addr || } else { @@ -143,14 +187,22 @@ static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } || } |.endmacro | -|// Checks PTR for end-of-buffer. -|.macro check_eob, m +|// Checkpoints our progress by writing PTR to DECODER, and +|// checks for end-of-buffer. +|.macro checkpoint, h +| mov DECODER->ptr, PTR | cmp PTR, DECODER->effective_end -|| if (m->is_group) { - | jae ->exit_jit -|| } else { - | jae =>m->jit_endofbuf_pclabel -|| } +| jae =>upb_getpclabel(plan, h, ENDOFBUF) +|.endmacro +| +|.macro check_bool_ret +| test al, al +| jz ->exit_jit +|.endmacro +| +|.macro check_ptr_ret +| test rax, rax +| jz ->exit_jit |.endmacro | |// Decodes varint from [PTR + offset] -> ARG3. @@ -172,8 +224,7 @@ static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } | mov ARG1_64, rax | mov ARG2_32, ARG3_32 | callp upb_vdecode_max8_fast -| test rax, rax -| jz ->exit_jit // >10-byte varint. +| check_ptr_ret // Check for unterminated, >10-byte varint. |9: |.endmacro | @@ -187,74 +238,103 @@ static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } |// Could specialize this by avoiding the value masking: could just key the |// table on the raw (length-masked) varint to save 3-4 cycles of latency. |// Currently only support tables where all entries are in the array part. -|.macro dyndispatch_, m -|=>m->jit_dyndispatch_pclabel: +|.macro dyndispatch_, h +|=>upb_getpclabel(plan, h, DYNDISPATCH): | decode_loaded_varint, 0 | mov ecx, edx | shr ecx, 3 -| and edx, 0x7 // For the type check that will happen later. -| cmp ecx, m->max_field_number // Bounds-check the field. -| ja ->exit_jit // In the future; could be unknown label -|| if ((uintptr_t)m->tablearray < 0xffffffff) { +| and edx, 0x7 // Note: this value is used in the FIELD pclabel below. +| cmp edx, UPB_WIRE_TYPE_END_GROUP +| je >1 +|| upb_jitmsginfo *mi = upb_getmsginfo(plan, h); +| cmp ecx, mi->max_field_number // Bounds-check the field. +| ja ->exit_jit // In the future; could be unknown label +|| if ((uintptr_t)mi->tablearray < 0xffffffff) { | // TODO: support hybrid array/hash tables. -| mov rax, qword [rcx*8 + m->tablearray] +| mov rax, qword [rcx*8 + mi->tablearray] || } else { -| mov64 rax, (uintptr_t)m->tablearray +| mov64 rax, (uintptr_t)mi->tablearray | mov rax, qword [rax + rcx*8] || } | jmp rax // Dispatch: unpredictable jump. +|1: +|// End group. +| cmp ecx, FRAME->group_fieldnum +| jne ->exit_jit // Unexpected END_GROUP tag. +| mov PTR, rax // rax came from decode_loaded_varint +| mov DECODER->ptr, PTR +| jmp =>upb_getpclabel(plan, h, ENDOFMSG) |.endmacro | |.if 1 | // Replicated dispatch: larger code, but better branch prediction. | .define dyndispatch, dyndispatch_ |.else -| .macro dyndispatch, m -| jmp =>m->jit_dyndispatch_pclabel +| // Single dispatch: smaller code, could be faster because of reduced +| // icache usage. We keep this around to allow for easy comparison between +| // the two. +| .macro dyndispatch, h +| jmp =>upb_getpclabel(plan, h, DYNDISPATCH) | .endmacro |.endif | |// Push a stack frame (not the CPU stack, the upb_decoder stack). -|.macro pushframe, f, end_offset_, is_sequence_ -| lea rax, [FRAME + sizeof(upb_dispatcher_frame)] // rax for shorter addressing. -| cmp rax, qword DECODER->dispatcher.limit +|.macro pushframe, h, field, end_offset_, endtype +|// Decoder Frame. +| lea rax, [FRAME + sizeof(upb_decoder_frame)] // rax for short addressing +| cmp rax, DECODER->limit | jae ->exit_jit // Frame stack overflow. -| mov64 r8, (uintptr_t)f -| mov qword FRAME:rax->f, r8 +| mov64 r10, (uintptr_t)field +| mov FRAME:rax->f, r10 | mov qword FRAME:rax->end_ofs, end_offset_ -| mov byte FRAME:rax->is_sequence, is_sequence_ -| mov DECODER->dispatcher.top, rax +| mov byte FRAME:rax->is_sequence, (endtype == UPB_HANDLER_ENDSEQ) +| mov byte FRAME:rax->is_packed, 0 +|| if (upb_fielddef_type(field) == UPB_TYPE_GROUP && +|| endtype == UPB_HANDLER_ENDSUBMSG) { +| mov dword FRAME:rax->group_fieldnum, upb_fielddef_number(field) +|| } else { +| mov dword FRAME:rax->group_fieldnum, 0xffffffff +|| } +| mov DECODER->top, rax | mov FRAME, rax +|// Sink Frame. +| lea rcx, [SINKFRAME + sizeof(upb_sink_frame)] // rcx for short addressing +| cmp rcx, DECODER->sink.limit +| jae ->exit_jit // Frame stack overflow. +| mov dword SINKFRAME:rcx->end, getselector(field, endtype) +|| if (upb_fielddef_issubmsg(field)) { +| mov64 r9, (uintptr_t)upb_handlers_getsubhandlers(h, field) +|| } else { +| mov64 r9, (uintptr_t)h +|| } +| mov SINKFRAME:rcx->h, r9 +| mov DECODER->sink.top, rcx +| mov SINKFRAME, rcx |.endmacro | -|.macro popframe, m -| sub FRAME, sizeof(upb_dispatcher_frame) -| mov DECODER->dispatcher.top, FRAME -| setmsgend m -| mov CLOSURE, FRAME->closure +|.macro popframe +| sub FRAME, sizeof(upb_decoder_frame) +| mov DECODER->top, FRAME +| sub SINKFRAME, sizeof(upb_sink_frame) +| mov DECODER->sink.top, SINKFRAME +| setmsgend +| mov CLOSURE, SINKFRAME->closure |.endmacro | -|.macro setmsgend, m -| mov rsi, DECODER->jit_end -|| if (m->is_group) { -| mov64 rax, 0xffffffffffffffff -| mov qword DECODER->delim_end, rax -| mov DECODER->effective_end, rsi -|| } else { -| // Could store a correctly-biased version in the frame, at the cost of -| // a larger stack. -| mov eax, dword FRAME->end_ofs -| add rax, qword DECODER->buf -| mov DECODER->delim_end, rax // delim_end = d->buf + f->end_ofs -| cmp rax, rsi -| jb >8 -| mov rax, rsi // effective_end = min(d->delim_end, d->jit_end) +|.macro setmsgend +| mov rsi, DECODER->jit_end +| mov rax, qword FRAME->end_ofs // Will be UINT64_MAX for groups. +| sub rax, qword DECODER->bufstart_ofs +| add rax, qword DECODER->buf // rax = d->buf + f->end_ofs - d->bufstart_ofs +| jc >8 // If the addition overflowed, use jit_end +| cmp rax, rsi +| ja >8 // If jit_end is less, use jit_end +| mov rsi, rax // Use frame end. |8: -| mov DECODER->effective_end, rax -|| } +| mov DECODER->effective_end, rsi |.endmacro | -|// rax contains the tag, compare it against "tag", but since it is a varint +|// rcx contains the tag, compare it against "tag", but since it is a varint |// we must only compare as many bytes as actually have data. |.macro checktag, tag || switch (upb_value_size(tag)) { @@ -279,22 +359,6 @@ static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } || } |.endmacro | -|// TODO: optimize for 0 (xor) and 32-bits. -|.macro loadfval, f -||#ifndef NDEBUG -||// Since upb_value carries type information in debug mode -||// only, we need to pass the arguments slightly differently. -| mov ARG3_32, f->fval.type -||#endif -|| if (f->fval.val.uint64 == 0) { -| xor ARG2_32, ARG2_32 -|| } else if (f->fval.val.uint64 < 0xffffffff) { -| mov ARG2_32, f->fval.val.uint64 -|| } else { -| mov64 ARG2_64, f->fval.val.uint64 -|| } -|.endmacro -| |.macro sethas, reg, hasbit || if (hasbit >= 0) { | or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) @@ -304,14 +368,37 @@ static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } #include <stdlib.h> #include "upb/pb/varint.h" -#include "upb/msg.h" + +static upb_selector_t getselector(const upb_fielddef *f, + upb_handlertype_t type) { + upb_selector_t selector; + bool ok = upb_getselector(f, type, &selector); + UPB_ASSERT_VAR(ok, ok); + return selector; +} + +static upb_func *gethandler(const upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + return upb_handlers_gethandler(h, getselector(f, type)); +} + +static uintptr_t gethandlerdata(const upb_handlers *h, const upb_fielddef *f, + upb_handlertype_t type) { + return (uintptr_t)upb_handlers_gethandlerdata(h, getselector(f, type)); +} // Decodes the next val into ARG3, advances PTR. static void upb_decoderplan_jit_decodefield(upb_decoderplan *plan, - uint8_t type, size_t tag_size) { + uint8_t type, size_t tag_size, + const upb_handlers *h, + const upb_fielddef *f) { // Decode the value into arg 3 for the callback. switch (type) { case UPB_TYPE(DOUBLE): + | movsd XMMARG1, qword [PTR + tag_size] + | add PTR, 8 + tag_size + break; + case UPB_TYPE(FIXED64): case UPB_TYPE(SFIXED64): | mov ARG3_64, qword [PTR + tag_size] @@ -319,6 +406,10 @@ static void upb_decoderplan_jit_decodefield(upb_decoderplan *plan, break; case UPB_TYPE(FLOAT): + | movss XMMARG1, dword [PTR + tag_size] + | add PTR, 4 + tag_size + break; + case UPB_TYPE(FIXED32): case UPB_TYPE(SFIXED32): | mov ARG3_32, dword [PTR + tag_size] @@ -362,7 +453,7 @@ static void upb_decoderplan_jit_decodefield(upb_decoderplan *plan, break; case UPB_TYPE(STRING): - case UPB_TYPE(BYTES): + case UPB_TYPE(BYTES): { // We only handle the case where the entire string is in our current // buf, which sidesteps any security problems. The C path has more // robust checks. @@ -372,22 +463,42 @@ static void upb_decoderplan_jit_decodefield(upb_decoderplan *plan, | sub rdi, rax | cmp ARG3_64, rdi // if (len > d->end - str) | ja ->exit_jit // Can't deliver, whole string not in buf. + | mov PTR, rax + + upb_func *handler = gethandler(h, f, UPB_HANDLER_STARTSTR); + if (handler) { + | mov DECODER->tmp_len, ARG3_64 + | mov ARG1_64, CLOSURE + | mov64 ARG2_64, gethandlerdata(h, f, UPB_HANDLER_STARTSTR) + | callp handler + | check_ptr_ret + | mov ARG1_64, rax // sub-closure + | mov ARG4_64, DECODER->tmp_len + } else { + | mov ARG1_64, CLOSURE + | mov ARG4_64, ARG3_64 + } + + handler = gethandler(h, f, UPB_HANDLER_STRING); + if (handler) { + | mov64 ARG2_64, gethandlerdata(h, f, UPB_HANDLER_STRING) + | mov ARG3_64, PTR + | callp handler + // TODO: properly handle returns other than "n" (the whole string). + | add PTR, rax + } else { + | add PTR, ARG4_64 + } - // Update PTR to point past end of string. - | mov rdi, rax - | add rdi, ARG3_64 - | mov PTR, rdi - - // Populate BYTEREGION appropriately. - | sub rax, DECODER->buf - | add rax, DECODER->bufstart_ofs // = d->ptr - d->buf + d->bufstart_ofs - | mov BYTEREGION->start, rax - | mov BYTEREGION->discard, rax - | add rax, ARG3_64 - | mov BYTEREGION->end, rax - | mov BYTEREGION->fetch, rax // Fast path ensures whole string is loaded - | mov ARG3_64, BYTEREGION + handler = gethandler(h, f, UPB_HANDLER_ENDSTR); + if (handler) { + | mov ARG1_64, CLOSURE + | mov64 ARG2_64, gethandlerdata(h, f, UPB_HANDLER_ENDSTR) + | callp handler + | check_bool_ret + } break; + } // Will dispatch callbacks and call submessage in a second. case UPB_TYPE(MESSAGE): @@ -402,85 +513,85 @@ static void upb_decoderplan_jit_decodefield(upb_decoderplan *plan, } static void upb_decoderplan_jit_callcb(upb_decoderplan *plan, - upb_fhandlers *f) { + const upb_handlers *h, + const upb_fielddef *f) { // Call callbacks. Specializing the append accessors didn't yield a speed // increase in benchmarks. - if (upb_issubmsgtype(f->type)) { - if (f->type == UPB_TYPE(MESSAGE)) { + if (upb_fielddef_issubmsg(f)) { + if (upb_fielddef_type(f) == UPB_TYPE(MESSAGE)) { | mov rsi, PTR | sub rsi, DECODER->buf | add rsi, ARG3_64 // = (d->ptr - d->buf) + delim_len } else { - assert(f->type == UPB_TYPE(GROUP)); + assert(upb_fielddef_type(f) == UPB_TYPE(GROUP)); | mov rsi, UPB_NONDELIMITED } - | pushframe f, rsi, false + | pushframe h, f, rsi, UPB_HANDLER_ENDSUBMSG // Call startsubmsg handler (if any). - if (f->startsubmsg) { + upb_func *startsubmsg = gethandler(h, f, UPB_HANDLER_STARTSUBMSG); + if (startsubmsg) { // upb_sflow_t startsubmsg(void *closure, upb_value fval) | mov ARG1_64, CLOSURE - | loadfval f - | callp f->startsubmsg - | sethas CLOSURE, f->hasbit - | mov CLOSURE, rdx - } else { - | sethas CLOSURE, f->hasbit + | mov64 ARG2_64, gethandlerdata(h, f, UPB_HANDLER_STARTSUBMSG); + | callp startsubmsg + | check_ptr_ret + | mov CLOSURE, rax } - | mov qword FRAME->closure, CLOSURE - // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK - | mov DECODER->ptr, PTR + | mov qword SINKFRAME->closure, CLOSURE - const upb_mhandlers *sub_m = upb_fhandlers_getsubmsg(f); - | call =>sub_m->jit_startmsg_pclabel; - | popframe upb_fhandlers_getmsg(f) + // TODO: have to decide what to do with NULLs subhandlers (or whether to + // disallow them and require a full handlers tree to match the def tree). + const upb_handlers *sub_h = upb_handlers_getsubhandlers(h, f); + assert(sub_h); + | call =>upb_getpclabel(plan, sub_h, STARTMSG) + | popframe // Call endsubmsg handler (if any). - if (f->endsubmsg) { + upb_func *endsubmsg = gethandler(h, f, UPB_HANDLER_ENDSUBMSG); + if (endsubmsg) { // upb_flow_t endsubmsg(void *closure, upb_value fval); | mov ARG1_64, CLOSURE - | loadfval f - | callp f->endsubmsg + | mov64 ARG2_64, gethandlerdata(h, f, UPB_HANDLER_ENDSUBMSG); + | callp endsubmsg + | check_bool_ret } - // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK - | mov DECODER->ptr, PTR - } else { + } else if (!upb_fielddef_isstring(f)) { | mov ARG1_64, CLOSURE + upb_handlertype_t handlertype = upb_handlers_getprimitivehandlertype(f); + upb_func *handler = gethandler(h, f, handlertype); + const upb_stdmsg_fval *fv = (void*)gethandlerdata(h, f, handlertype); // Test for callbacks we can specialize. // Can't switch() on function pointers. - if (f->value == &upb_stdmsg_setint64 || - f->value == &upb_stdmsg_setuint64 || - f->value == &upb_stdmsg_setptr || - f->value == &upb_stdmsg_setdouble) { - const upb_fielddef *fd = upb_value_getfielddef(f->fval); - | mov [ARG1_64 + fd->offset], ARG3_64 - } else if (f->value == &upb_stdmsg_setint32 || - f->value == &upb_stdmsg_setuint32 || - f->value == &upb_stdmsg_setfloat) { - const upb_fielddef *fd = upb_value_getfielddef(f->fval); - | mov [ARG1_64 + fd->offset], ARG3_32 - } else if (f->value == &upb_stdmsg_setbool) { - const upb_fielddef *fd = upb_value_getfielddef(f->fval); - | mov [ARG1_64 + fd->offset], ARG3_8 - } else if (f->value) { + if (handler == (void*)&upb_stdmsg_setint64 || + handler == (void*)&upb_stdmsg_setuint64) { + | mov [ARG1_64 + fv->offset], ARG3_64 + | sethas CLOSURE, fv->hasbit + } else if (handler == (void*)&upb_stdmsg_setdouble) { + | movsd qword [ARG1_64 + fv->offset], XMMARG1 + | sethas CLOSURE, fv->hasbit + } else if (handler == (void*)&upb_stdmsg_setint32 || + handler == (void*)&upb_stdmsg_setuint32) { + | mov [ARG1_64 + fv->offset], ARG3_32 + | sethas CLOSURE, fv->hasbit + } else if (handler == (void*)&upb_stdmsg_setfloat) { + | movss dword [ARG1_64 + fv->offset], XMMARG1 + | sethas CLOSURE, fv->hasbit + } else if (handler == (void*)&upb_stdmsg_setbool) { + | mov [ARG1_64 + fv->offset], ARG3_8 + | sethas CLOSURE, fv->hasbit + } else if (handler) { // Load closure and fval into arg registers. - ||#ifndef NDEBUG - ||// Since upb_value carries type information in debug mode - ||// only, we need to pass the arguments slightly differently. - | mov ARG4_64, ARG3_64 - | mov ARG5_32, upb_types[f->type].inmemory_type - ||#endif - | loadfval f - | callp f->value + | mov64 ARG2_64, gethandlerdata(h, f, handlertype); + | callp handler + | check_bool_ret } - | sethas CLOSURE, f->hasbit - // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK - | mov DECODER->ptr, PTR } } -static uint64_t upb_get_encoded_tag(upb_fhandlers *f) { - uint32_t tag = (f->number << 3) | upb_decoder_types[f->type].native_wire_type; +static uint64_t upb_get_encoded_tag(const upb_fielddef *f) { + uint32_t tag = (upb_fielddef_number(f) << 3) | + upb_decoder_types[upb_fielddef_type(f)].native_wire_type; uint64_t encoded_tag = upb_vencode32(tag); // No tag should be greater than 5 bytes. assert(encoded_tag <= 0xffffffffff); @@ -488,118 +599,121 @@ static uint64_t upb_get_encoded_tag(upb_fhandlers *f) { } // PTR should point to the beginning of the tag. -static void upb_decoderplan_jit_field(upb_decoderplan *plan, upb_mhandlers *m, - upb_fhandlers *f, upb_fhandlers *next_f) { +static void upb_decoderplan_jit_field(upb_decoderplan *plan, + const upb_handlers *h, + const upb_fielddef *f, + const upb_fielddef *next_f) { uint64_t tag = upb_get_encoded_tag(f); uint64_t next_tag = next_f ? upb_get_encoded_tag(next_f) : 0; + int tag_size = upb_value_size(tag); // PC-label for the dispatch table. // We check the wire type (which must be loaded in edx) because the // table is keyed on field number, not type. - |=>f->jit_pclabel: + |=>upb_getpclabel(plan, f, FIELD): | cmp edx, (tag & 0x7) | jne ->exit_jit // In the future: could be an unknown field or packed. - |=>f->jit_pclabel_notypecheck: - if (f->repeated) { + |=>upb_getpclabel(plan, f, FIELD_NO_TYPECHECK): + if (upb_fielddef_isseq(f)) { | mov rsi, FRAME->end_ofs - | pushframe f, rsi, true - if (f->startseq) { + | pushframe h, f, rsi, UPB_HANDLER_ENDSEQ + upb_func *startseq = gethandler(h, f, UPB_HANDLER_STARTSEQ); + if (startseq) { | mov ARG1_64, CLOSURE - | loadfval f - | callp f->startseq - | sethas CLOSURE, f->hasbit - | mov CLOSURE, rdx - } else { - | sethas CLOSURE, f->hasbit + | mov64 ARG2_64, gethandlerdata(h, f, UPB_HANDLER_STARTSEQ); + | callp startseq + | check_ptr_ret + | mov CLOSURE, rax } - | mov qword FRAME->closure, CLOSURE + | mov qword SINKFRAME->closure, CLOSURE } |1: // Label for repeating this field. - int tag_size = upb_value_size(tag); - if (f->type == UPB_TYPE_ENDGROUP) { - | add PTR, tag_size - | jmp =>m->jit_endofmsg_pclabel - return; - } - - upb_decoderplan_jit_decodefield(plan, f->type, tag_size); - upb_decoderplan_jit_callcb(plan, f); + upb_decoderplan_jit_decodefield(plan, upb_fielddef_type(f), tag_size, h, f); + upb_decoderplan_jit_callcb(plan, h, f); // Epilogue: load next tag, check for repeated field. - | check_eob m + | checkpoint h | mov rcx, qword [PTR] - if (f->repeated) { + if (upb_fielddef_isseq(f)) { | checktag tag | je <1 - if (f->endseq) { + upb_func *endseq = gethandler(h, f, UPB_HANDLER_ENDSEQ); + if (endseq) { | mov ARG1_64, CLOSURE - | loadfval f - | callp f->endseq + | mov64 ARG2_64, gethandlerdata(h, f, UPB_HANDLER_ENDSEQ); + | callp endseq } - | popframe m + | popframe + // Load next tag again (popframe clobbered it). + | mov rcx, qword [PTR] } + if (next_tag != 0) { | checktag next_tag - | je =>next_f->jit_pclabel_notypecheck + | je =>upb_getpclabel(plan, next_f, FIELD_NO_TYPECHECK) } // Fall back to dynamic dispatch. - | dyndispatch m - |1: + | dyndispatch h } static int upb_compare_uint32(const void *a, const void *b) { - // TODO: always put ENDGROUP at the end. return *(uint32_t*)a - *(uint32_t*)b; } -static void upb_decoderplan_jit_msg(upb_decoderplan *plan, upb_mhandlers *m) { - |=>m->jit_afterstartmsg_pclabel: +static void upb_decoderplan_jit_msg(upb_decoderplan *plan, + const upb_handlers *h) { + |=>upb_getpclabel(plan, h, AFTER_STARTMSG): // There was a call to get here, so we need to align the stack. | sub rsp, 8 | jmp >1 - |=>m->jit_startmsg_pclabel: + |=>upb_getpclabel(plan, h, STARTMSG): // There was a call to get here, so we need to align the stack. | sub rsp, 8 // Call startmsg handler (if any): - if (m->startmsg) { + upb_startmsg_handler *startmsg = upb_handlers_getstartmsg(h); + if (startmsg) { // upb_flow_t startmsg(void *closure); - | mov ARG1_64, FRAME->closure - | callp m->startmsg - // TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK + | mov ARG1_64, SINKFRAME->closure + | callp startmsg + | check_bool_ret } |1: - | setmsgend m - | check_eob m + | setmsgend + | checkpoint h | mov ecx, dword [PTR] - | dyndispatch_ m + | dyndispatch_ h // --------- New code section (does not fall through) ------------------------ // Emit code for parsing each field (dynamic dispatch contains pointers to // all of these). - // Create an ordering over the fields (inttable ordering is undefined). - int num_keys = upb_inttable_count(&m->fieldtab); + // Create an ordering over the fields in field number order. + // Parsing will theoretically be fastest if we emit code in the same + // order as field numbers are seen on-the-wire because of an optimization + // in the generated code that skips dynamic dispatch if the next field is + // as expected. + const upb_msgdef *md = upb_handlers_msgdef(h); + int num_keys = upb_msgdef_numfields(md); uint32_t *keys = malloc(num_keys * sizeof(*keys)); int idx = 0; - upb_inttable_iter i; - upb_inttable_begin(&i, &m->fieldtab); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - keys[idx++] = upb_inttable_iter_key(&i); + upb_msg_iter i; + for(upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) { + keys[idx++] = upb_fielddef_number(upb_msg_iter_field(&i)); } qsort(keys, num_keys, sizeof(uint32_t), &upb_compare_uint32); for(int i = 0; i < num_keys; i++) { - upb_fhandlers *f = upb_mhandlers_lookup(m, keys[i]); - upb_fhandlers *next_f = - (i + 1 < num_keys) ? upb_mhandlers_lookup(m, keys[i + 1]) : NULL; - upb_decoderplan_jit_field(plan, m, f, next_f); + const upb_fielddef *f = upb_msgdef_itof(md, keys[i]); + const upb_fielddef *next_f = + (i + 1 < num_keys) ? upb_msgdef_itof(md, keys[i + 1]) : NULL; + upb_decoderplan_jit_field(plan, h, f, next_f); } free(keys); @@ -607,27 +721,19 @@ static void upb_decoderplan_jit_msg(upb_decoderplan *plan, upb_mhandlers *m) { // --------- New code section (does not fall through) ------------------------ // End-of-buf / end-of-message. - if (!m->is_group) { - // This case doesn't exist for groups, because there eob really means - // eob, so that case just exits the jit directly. - |=>m->jit_endofbuf_pclabel: - | cmp PTR, DECODER->delim_end - | jb ->exit_jit // We are at eob, but not end-of-submsg. - } + // We hit a buffer limit; either we hit jit_end or end-of-submessage. + |=>upb_getpclabel(plan, h, ENDOFBUF): + | cmp PTR, DECODER->jit_end + | jae ->exit_jit - |=>m->jit_endofmsg_pclabel: + |=>upb_getpclabel(plan, h, ENDOFMSG): // We are at end-of-submsg: call endmsg handler (if any): - if (m->endmsg) { + upb_endmsg_handler *endmsg = upb_handlers_getendmsg(h); + if (endmsg) { // void endmsg(void *closure, upb_status *status) { - | mov ARG1_64, FRAME->closure - | lea ARG2_64, DECODER->dispatcher.status - | callp m->endmsg - } - - if (m->is_group) { - // Advance past the "end group" tag. - // TODO: Handle UPB_BREAK - | mov DECODER->ptr, PTR + | mov ARG1_64, SINKFRAME->closure + | lea ARG2_64, DECODER->sink.status + | callp endmsg } // Counter previous alignment. @@ -657,9 +763,9 @@ static void upb_decoderplan_jit(upb_decoderplan *plan) { // Align stack. | sub rsp, 8 | mov DECODER, ARG1_64 - | mov FRAME, DECODER:ARG1_64->dispatcher.top - | lea BYTEREGION, DECODER:ARG1_64->str_byteregion - | mov CLOSURE, FRAME->closure + | mov FRAME, DECODER:ARG1_64->top + | mov SINKFRAME, DECODER:ARG1_64->sink.top + | mov CLOSURE, SINKFRAME->closure | mov PTR, DECODER->ptr // TODO: push return addresses for re-entry (will be necessary for multiple @@ -680,54 +786,65 @@ static void upb_decoderplan_jit(upb_decoderplan *plan) { | leave | ret - upb_handlers *h = plan->handlers; - for (int i = 0; i < h->msgs_len; i++) - upb_decoderplan_jit_msg(plan, h->msgs[i]); -} - -static void upb_decoderplan_jit_assignfieldlabs(upb_fhandlers *f, - uint32_t *pclabel_count) { - f->jit_pclabel = (*pclabel_count)++; - f->jit_pclabel_notypecheck = (*pclabel_count)++; -} - -static void upb_decoderplan_jit_assignmsglabs(upb_mhandlers *m, - uint32_t *pclabel_count) { - m->jit_startmsg_pclabel = (*pclabel_count)++; - m->jit_afterstartmsg_pclabel = (*pclabel_count)++; - m->jit_endofbuf_pclabel = (*pclabel_count)++; - m->jit_endofmsg_pclabel = (*pclabel_count)++; - m->jit_dyndispatch_pclabel = (*pclabel_count)++; - m->jit_unknownfield_pclabel = (*pclabel_count)++; - m->max_field_number = 0; upb_inttable_iter i; - upb_inttable_begin(&i, &m->fieldtab); + upb_inttable_begin(&i, &plan->msginfo); for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - uint32_t key = upb_inttable_iter_key(&i); - m->max_field_number = UPB_MAX(m->max_field_number, key); - upb_fhandlers *f = upb_value_getptr(upb_inttable_iter_value(&i)); - upb_decoderplan_jit_assignfieldlabs(f, pclabel_count); + const upb_handlers *h = (const upb_handlers*)upb_inttable_iter_key(&i); + upb_decoderplan_jit_msg(plan, h); + } +} + +static void upb_decoderplan_jit_assignpclabels(upb_decoderplan *plan, + const upb_handlers *h) { + // Limit the DFS. + if (upb_inttable_lookupptr(&plan->pclabels, h)) return; + + upb_inttable_insertptr(&plan->pclabels, h, + upb_value_uint32(plan->pclabel_count)); + plan->pclabel_count += TOTAL_MSG_PCLABELS; + + upb_jitmsginfo *info = malloc(sizeof(*info)); + info->max_field_number = 0; + upb_inttable_insertptr(&plan->msginfo, h, upb_value_ptr(info)); + + upb_msg_iter i; + upb_msg_begin(&i, upb_handlers_msgdef(h)); + for(; !upb_msg_done(&i); upb_msg_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + info->max_field_number = + UPB_MAX(info->max_field_number, upb_fielddef_number(f)); + upb_inttable_insertptr(&plan->pclabels, f, + upb_value_uint32(plan->pclabel_count)); + plan->pclabel_count += TOTAL_FIELD_PCLABELS; + + // Discover the whole graph of handlers depth-first. We will probably + // revise this later to be more explicit about the list of handlers that + // the plan should include. + if (upb_fielddef_issubmsg(f)) { + const upb_handlers *subh = upb_handlers_getsubhandlers(h, f); + if (subh) upb_decoderplan_jit_assignpclabels(plan, subh); + } } // TODO: support large field numbers by either using a hash table or // generating code for a binary search. For now large field numbers // will just fall back to the table decoder. - m->max_field_number = UPB_MIN(m->max_field_number, 16000); - m->tablearray = malloc((m->max_field_number + 1) * sizeof(void*)); + info->max_field_number = UPB_MIN(info->max_field_number, 16000); + info->tablearray = malloc((info->max_field_number + 1) * sizeof(void*)); } static void upb_decoderplan_makejit(upb_decoderplan *plan) { + upb_inttable_init(&plan->msginfo, UPB_CTYPE_PTR); plan->debug_info = NULL; // Assign pclabels. - uint32_t pclabel_count = 0; - upb_handlers *h = plan->handlers; - for (int i = 0; i < h->msgs_len; i++) - upb_decoderplan_jit_assignmsglabs(h->msgs[i], &pclabel_count); + plan->pclabel_count = 0; + upb_inttable_init(&plan->pclabels, UPB_CTYPE_UINT32); + upb_decoderplan_jit_assignpclabels(plan, plan->handlers); void **globals = malloc(UPB_JIT_GLOBAL__MAX * sizeof(*globals)); dasm_init(plan, 1); dasm_setupglobal(plan, globals, UPB_JIT_GLOBAL__MAX); - dasm_growpc(plan, pclabel_count); + dasm_growpc(plan, plan->pclabel_count); dasm_setup(plan, upb_jit_actionlist); upb_decoderplan_jit(plan); @@ -744,38 +861,53 @@ static void upb_decoderplan_makejit(upb_decoderplan *plan) { dasm_encode(plan, plan->jit_code); // Create dispatch tables. - for (int i = 0; i < h->msgs_len; i++) { - upb_mhandlers *m = h->msgs[i]; + upb_inttable_iter i; + upb_inttable_begin(&i, &plan->msginfo); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + const upb_handlers *h = (const upb_handlers*)upb_inttable_iter_key(&i); + upb_jitmsginfo *mi = upb_getmsginfo(plan, h); // We jump to after the startmsg handler since it is called before entering // the JIT (either by upb_decoder or by a previous call to the JIT). - m->jit_func = - plan->jit_code + dasm_getpclabel(plan, m->jit_afterstartmsg_pclabel); - for (uint32_t j = 0; j <= m->max_field_number; j++) { - upb_fhandlers *f = upb_mhandlers_lookup(m, j); + mi->jit_func = plan->jit_code + + dasm_getpclabel(plan, upb_getpclabel(plan, h, AFTER_STARTMSG)); + for (uint32_t j = 0; j <= mi->max_field_number; j++) { + const upb_fielddef *f = upb_msgdef_itof(upb_handlers_msgdef(h), j); if (f) { - m->tablearray[j] = - plan->jit_code + dasm_getpclabel(plan, f->jit_pclabel); + mi->tablearray[j] = plan->jit_code + + dasm_getpclabel(plan, upb_getpclabel(plan, f, FIELD)); } else { // TODO: extend the JIT to handle unknown fields. // For the moment we exit the JIT for any unknown field. - m->tablearray[j] = globals[UPB_JIT_GLOBAL_exit_jit]; + mi->tablearray[j] = globals[UPB_JIT_GLOBAL_exit_jit]; } } } + upb_inttable_uninit(&plan->pclabels); + dasm_free(plan); free(globals); mprotect(plan->jit_code, plan->jit_size, PROT_EXEC | PROT_READ); +#ifndef NDEBUG // View with: objdump -M intel -D -b binary -mi386 -Mx86-64 /tmp/machine-code // Or: ndisasm -b 64 /tmp/machine-code FILE *f = fopen("/tmp/machine-code", "wb"); fwrite(plan->jit_code, plan->jit_size, 1, f); fclose(f); +#endif } static void upb_decoderplan_freejit(upb_decoderplan *plan) { + upb_inttable_iter i; + upb_inttable_begin(&i, &plan->msginfo); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_jitmsginfo *mi = upb_value_getptr(upb_inttable_iter_value(&i)); + free(mi->tablearray); + free(mi); + } + upb_inttable_uninit(&plan->msginfo); munmap(plan->jit_code, plan->jit_size); free(plan->debug_info); // TODO: unregister @@ -783,7 +915,7 @@ static void upb_decoderplan_freejit(upb_decoderplan *plan) { static void upb_decoder_enterjit(upb_decoder *d) { if (d->plan->jit_code && - d->dispatcher.top == d->dispatcher.stack && + d->sink.top == d->sink.stack && d->ptr && d->ptr < d->jit_end) { #ifndef NDEBUG register uint64_t rbx asm ("rbx") = 11; @@ -795,7 +927,9 @@ static void upb_decoder_enterjit(upb_decoder *d) { // Decodes as many fields as possible, updating d->ptr appropriately, // before falling through to the slow(er) path. void (*upb_jit_decode)(upb_decoder *d, void*) = (void*)d->plan->jit_code; - upb_jit_decode(d, d->plan->handlers->msgs[d->msg_offset]->jit_func); + upb_jitmsginfo *mi = upb_getmsginfo(d->plan, d->plan->handlers); + assert(mi); + upb_jit_decode(d, mi->jit_func); assert(d->ptr <= d->end); // Test that callee-save registers were properly restored. diff --git a/upb/pb/glue.c b/upb/pb/glue.c index 40b901d..4e69c0c 100644 --- a/upb/pb/glue.c +++ b/upb/pb/glue.c @@ -5,10 +5,14 @@ * Author: Josh Haberman <jhaberman@gmail.com> */ +#include "upb/pb/glue.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> #include "upb/bytestream.h" #include "upb/descriptor/reader.h" #include "upb/pb/decoder.h" -#include "upb/pb/glue.h" upb_def **upb_load_defs_from_descriptor(const char *str, size_t len, int *n, void *owner, upb_status *status) { @@ -16,16 +20,14 @@ upb_def **upb_load_defs_from_descriptor(const char *str, size_t len, int *n, upb_stringsrc_init(&strsrc); upb_stringsrc_reset(&strsrc, str, len); - upb_handlers *h = upb_handlers_new(); - upb_descreader_reghandlers(h); - + const upb_handlers *h = upb_descreader_newhandlers(&h); upb_decoderplan *p = upb_decoderplan_new(h, false); upb_decoder d; upb_decoder_init(&d); - upb_handlers_unref(h); + upb_handlers_unref(h, &h); upb_descreader r; upb_descreader_init(&r); - upb_decoder_resetplan(&d, p, 0); + upb_decoder_resetplan(&d, p); upb_decoder_resetinput(&d, upb_stringsrc_allbytes(&strsrc), &r); upb_success_t ret = upb_decoder_decode(&d); diff --git a/upb/pb/glue.h b/upb/pb/glue.h index 6179d8d..4bbc975 100644 --- a/upb/pb/glue.h +++ b/upb/pb/glue.h @@ -27,8 +27,7 @@ #define UPB_GLUE_H #include <stdbool.h> -#include "upb/upb.h" -#include "upb/def.h" +#include "upb/symtab.h" #ifdef __cplusplus extern "C" { @@ -55,6 +54,29 @@ char *upb_readfile(const char *filename, size_t *len); #ifdef __cplusplus } /* extern "C" */ + +namespace upb { + +// All routines that load descriptors expect the descriptor to be a +// FileDescriptorSet. +inline bool LoadDescriptorFileIntoSymtab(SymbolTable* s, const char *fname, + Status* status) { + return upb_load_descriptor_file_into_symtab(s, fname, status); +} + +inline bool LoadDescriptorIntoSymtab(SymbolTable* s, const char* str, + size_t len, Status* status) { + return upb_load_descriptor_into_symtab(s, str, len, status); +} + +// Templated so it can accept both string and std::string. +template <typename T> +bool LoadDescriptorIntoSymtab(SymbolTable* s, const T& desc, Status* status) { + return upb_load_descriptor_into_symtab(s, desc.c_str(), desc.size(), status); +} + +} // namespace upb + #endif #endif diff --git a/upb/pb/textprinter.c b/upb/pb/textprinter.c index 2fe3452..3770afc 100644 --- a/upb/pb/textprinter.c +++ b/upb/pb/textprinter.c @@ -5,11 +5,14 @@ * Author: Josh Haberman <jhaberman@gmail.com> */ +#include "upb/pb/textprinter.h" + #include <ctype.h> #include <float.h> #include <inttypes.h> +#include <stdio.h> #include <stdlib.h> -#include "upb/pb/textprinter.h" +#include <string.h> struct _upb_textprinter { upb_bytesink *sink; @@ -20,7 +23,7 @@ struct _upb_textprinter { #define CHECK(x) if ((x) < 0) goto err; -static int upb_textprinter_indent(upb_textprinter *p) { +static int indent(upb_textprinter *p) { if (!p->single_line) CHECK(upb_bytesink_putrepeated(p->sink, ' ', p->indent_depth*2)); return 0; @@ -28,37 +31,32 @@ err: return -1; } -static int upb_textprinter_endfield(upb_textprinter *p) { +static int endfield(upb_textprinter *p) { CHECK(upb_bytesink_putc(p->sink, p->single_line ? ' ' : '\n')); return 0; err: return -1; } -static int upb_textprinter_putescaped(upb_textprinter *p, - const upb_byteregion *bytes, - bool preserve_utf8) { +static int putescaped(upb_textprinter *p, const char *buf, size_t len, + bool preserve_utf8) { // Based on CEscapeInternal() from Google's protobuf release. - // TODO; we could read directly from a bytesrc's buffer instead. - // TODO; we could write byteregions to the sink when possible. - char dstbuf[512], *dst = dstbuf, *dstend = dstbuf + sizeof(dstbuf); - char *buf = malloc(upb_byteregion_len(bytes)), *src = buf; - char *end = src + upb_byteregion_len(bytes); - upb_byteregion_copyall(bytes, buf); + char dstbuf[4096], *dst = dstbuf, *dstend = dstbuf + sizeof(dstbuf); + const char *end = buf + len; // I think hex is prettier and more useful, but proto2 uses octal; should // investigate whether it can parse hex also. const bool use_hex = false; bool last_hex_escape = false; // true if last output char was \xNN - for (; src < end; src++) { + for (; buf < end; buf++) { if (dstend - dst < 4) { CHECK(upb_bytesink_write(p->sink, dstbuf, dst - dstbuf)); dst = dstbuf; } bool is_hex_escape = false; - switch (*src) { + switch (*buf) { case '\n': *(dst++) = '\\'; *(dst++) = 'n'; break; case '\r': *(dst++) = '\\'; *(dst++) = 'r'; break; case '\t': *(dst++) = '\\'; *(dst++) = 't'; break; @@ -66,123 +64,123 @@ static int upb_textprinter_putescaped(upb_textprinter *p, case '\'': *(dst++) = '\\'; *(dst++) = '\''; break; case '\\': *(dst++) = '\\'; *(dst++) = '\\'; break; default: - // Note that if we emit \xNN and the src character after that is a hex + // Note that if we emit \xNN and the buf character after that is a hex // digit then that digit must be escaped too to prevent it being // interpreted as part of the character code by C. - if ((!preserve_utf8 || (uint8_t)*src < 0x80) && - (!isprint(*src) || (last_hex_escape && isxdigit(*src)))) { - sprintf(dst, (use_hex ? "\\x%02x" : "\\%03o"), (uint8_t)*src); + if ((!preserve_utf8 || (uint8_t)*buf < 0x80) && + (!isprint(*buf) || (last_hex_escape && isxdigit(*buf)))) { + sprintf(dst, (use_hex ? "\\x%02x" : "\\%03o"), (uint8_t)*buf); is_hex_escape = use_hex; dst += 4; } else { - *(dst++) = *src; break; + *(dst++) = *buf; break; } } last_hex_escape = is_hex_escape; } // Flush remaining data. CHECK(upb_bytesink_write(p->sink, dst, dst - dstbuf)); - free(buf); return 0; err: - free(buf); return -1; } -#define TYPE(member, fmt) \ - static upb_flow_t upb_textprinter_put ## member(void *_p, upb_value fval, \ - upb_value val) { \ +#define TYPE(name, ctype, fmt) \ + static bool put ## name(void *_p, void *fval, ctype val) { \ upb_textprinter *p = _p; \ - const upb_fielddef *f = upb_value_getfielddef(fval); \ - uint64_t start_ofs = upb_bytesink_getoffset(p->sink); \ - CHECK(upb_textprinter_indent(p)); \ + const upb_fielddef *f = fval; \ + CHECK(indent(p)); \ CHECK(upb_bytesink_writestr(p->sink, upb_fielddef_name(f))); \ CHECK(upb_bytesink_writestr(p->sink, ": ")); \ - CHECK(upb_bytesink_printf(p->sink, fmt, upb_value_get ## member(val))); \ - CHECK(upb_textprinter_endfield(p)); \ - return UPB_CONTINUE; \ + CHECK(upb_bytesink_printf(p->sink, fmt, val)); \ + CHECK(endfield(p)); \ + return true; \ err: \ - upb_bytesink_rewind(p->sink, start_ofs); \ - return UPB_BREAK; \ + return false; \ } #define STRINGIFY_HELPER(x) #x #define STRINGIFY_MACROVAL(x) STRINGIFY_HELPER(x) -TYPE(double, "%." STRINGIFY_MACROVAL(DBL_DIG) "g") -TYPE(float, "%." STRINGIFY_MACROVAL(FLT_DIG) "g") -TYPE(int64, "%" PRId64) -TYPE(uint64, "%" PRIu64) -TYPE(int32, "%" PRId32) -TYPE(uint32, "%" PRIu32); -TYPE(bool, "%hhu"); +TYPE(int32, int32_t, "%" PRId32) +TYPE(int64, int64_t, "%" PRId64) +TYPE(uint32, uint32_t, "%" PRIu32); +TYPE(uint64, uint64_t, "%" PRIu64) +TYPE(float, float, "%." STRINGIFY_MACROVAL(FLT_DIG) "g") +TYPE(double, double, "%." STRINGIFY_MACROVAL(DBL_DIG) "g") +TYPE(bool, bool, "%hhu"); // Output a symbolic value from the enum if found, else just print as int32. -static upb_flow_t upb_textprinter_putenum(void *_p, upb_value fval, - upb_value val) { +static bool putenum(void *_p, void *fval, int32_t val) { upb_textprinter *p = _p; - uint64_t start_ofs = upb_bytesink_getoffset(p->sink); - const upb_fielddef *f = upb_value_getfielddef(fval); - const upb_enumdef *enum_def = - upb_downcast_enumdef_const(upb_fielddef_subdef(f)); - const char *label = upb_enumdef_iton(enum_def, upb_value_getint32(val)); + const upb_fielddef *f = fval; + const upb_enumdef *enum_def = upb_downcast_enumdef(upb_fielddef_subdef(f)); + const char *label = upb_enumdef_iton(enum_def, val); if (label) { CHECK(upb_bytesink_writestr(p->sink, label)); } else { - CHECK(upb_textprinter_putint32(_p, fval, val)); + CHECK(putint32(_p, fval, val)); } - return UPB_CONTINUE; + return true; err: - upb_bytesink_rewind(p->sink, start_ofs); - return UPB_BREAK; + return false; } -static upb_flow_t upb_textprinter_putstr(void *_p, upb_value fval, - upb_value val) { +static void *startstr(void *_p, void *fval, size_t size_hint) { + UPB_UNUSED(size_hint); + UPB_UNUSED(fval); upb_textprinter *p = _p; - uint64_t start_ofs = upb_bytesink_getoffset(p->sink); - const upb_fielddef *f = upb_value_getfielddef(fval); - CHECK(upb_bytesink_putc(p->sink, '"')); - CHECK(upb_textprinter_putescaped(p, upb_value_getbyteregion(val), - f->type == UPB_TYPE(STRING))); CHECK(upb_bytesink_putc(p->sink, '"')); - return UPB_CONTINUE; + return p; err: - upb_bytesink_rewind(p->sink, start_ofs); return UPB_BREAK; } -static upb_sflow_t upb_textprinter_startsubmsg(void *_p, upb_value fval) { +static bool endstr(void *_p, void *fval) { + UPB_UNUSED(fval); + upb_textprinter *p = _p; + CHECK(upb_bytesink_putc(p->sink, '"')); + return true; +err: + return false; +} + +static size_t putstr(void *_p, void *fval, const char *buf, size_t len) { upb_textprinter *p = _p; - uint64_t start_ofs = upb_bytesink_getoffset(p->sink); - const upb_fielddef *f = upb_value_getfielddef(fval); - CHECK(upb_textprinter_indent(p)); + const upb_fielddef *f = fval; + CHECK(putescaped(p, buf, len, upb_fielddef_type(f) == UPB_TYPE(STRING))); + return len; +err: + return 0; +} + +static void *startsubmsg(void *_p, void *fval) { + upb_textprinter *p = _p; + const upb_fielddef *f = fval; + CHECK(indent(p)); CHECK(upb_bytesink_printf(p->sink, "%s {", upb_fielddef_name(f))); if (!p->single_line) CHECK(upb_bytesink_putc(p->sink, '\n')); p->indent_depth++; - return UPB_CONTINUE_WITH(_p); + return _p; err: - upb_bytesink_rewind(p->sink, start_ofs); - return UPB_SBREAK; + return UPB_BREAK; } -static upb_flow_t upb_textprinter_endsubmsg(void *_p, upb_value fval) { - (void)fval; +static bool endsubmsg(void *_p, void *fval) { + UPB_UNUSED(fval); upb_textprinter *p = _p; - uint64_t start_ofs = upb_bytesink_getoffset(p->sink); p->indent_depth--; - CHECK(upb_textprinter_indent(p)); + CHECK(indent(p)); CHECK(upb_bytesink_putc(p->sink, '}')); - CHECK(upb_textprinter_endfield(p)); - return UPB_CONTINUE; + CHECK(endfield(p)); + return true; err: - upb_bytesink_rewind(p->sink, start_ofs); - return UPB_BREAK; + return false; } -upb_textprinter *upb_textprinter_new(void) { +upb_textprinter *upb_textprinter_new() { upb_textprinter *p = malloc(sizeof(*p)); return p; } @@ -196,22 +194,61 @@ void upb_textprinter_reset(upb_textprinter *p, upb_bytesink *sink, p->indent_depth = 0; } -static void upb_textprinter_onfreg(void *c, upb_fhandlers *fh, const upb_fielddef *f) { +static void onmreg(void *c, upb_handlers *h) { (void)c; - upb_fhandlers_setstartsubmsg(fh, &upb_textprinter_startsubmsg); - upb_fhandlers_setendsubmsg(fh, &upb_textprinter_endsubmsg); -#define F(type) &upb_textprinter_put ## type - static upb_value_handler *fptrs[] = {NULL, F(double), F(float), F(int64), - F(uint64), F(int32), F(uint64), F(uint32), F(bool), F(str), - NULL, NULL, F(str), F(uint32), F(enum), F(int32), - F(int64), F(int32), F(int64)}; - upb_fhandlers_setvalue(fh, fptrs[f->type]); - upb_value fval; - upb_value_setfielddef(&fval, f); - upb_fhandlers_setfval(fh, fval); + const upb_msgdef *m = upb_handlers_msgdef(h); + upb_msg_iter i; + for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) { + upb_fielddef *f = upb_msg_iter_field(&i); + switch (upb_fielddef_type(f)) { + case UPB_TYPE_INT32: + case UPB_TYPE_SINT32: + case UPB_TYPE_SFIXED32: + upb_handlers_setint32(h, f, putint32, f, NULL); + break; + case UPB_TYPE_SINT64: + case UPB_TYPE_SFIXED64: + case UPB_TYPE_INT64: + upb_handlers_setint64(h, f, putint64, f, NULL); + break; + case UPB_TYPE_UINT32: + case UPB_TYPE_FIXED32: + upb_handlers_setuint32(h, f, putuint32, f, NULL); + break; + case UPB_TYPE_UINT64: + case UPB_TYPE_FIXED64: + upb_handlers_setuint64(h, f, putuint64, f, NULL); + break; + case UPB_TYPE_FLOAT: + upb_handlers_setfloat(h, f, putfloat, f, NULL); + break; + case UPB_TYPE_DOUBLE: + upb_handlers_setdouble(h, f, putdouble, f, NULL); + break; + case UPB_TYPE_BOOL: + upb_handlers_setbool(h, f, putbool, f, NULL); + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + upb_handlers_setstartstr(h, f, startstr, f, NULL); + upb_handlers_setstring(h, f, putstr, f, NULL); + upb_handlers_setendstr(h, f, endstr, f, NULL); + break; + case UPB_TYPE_GROUP: + case UPB_TYPE_MESSAGE: + upb_handlers_setstartsubmsg(h, f, &startsubmsg, f, NULL); + upb_handlers_setendsubmsg(h, f, &endsubmsg, f, NULL); + break; + case UPB_TYPE_ENUM: + upb_handlers_setint32(h, f, putenum, f, NULL); + default: + assert(false); + break; + } + } } -upb_mhandlers *upb_textprinter_reghandlers(upb_handlers *h, const upb_msgdef *m) { - return upb_handlers_regmsgdef( - h, m, NULL, &upb_textprinter_onfreg, NULL); +const upb_handlers *upb_textprinter_newhandlers(const void *owner, + const upb_msgdef *m) { + return upb_handlers_newfrozen(m, owner, &onmreg, NULL); } diff --git a/upb/pb/textprinter.h b/upb/pb/textprinter.h index 174148e..6d111d2 100644 --- a/upb/pb/textprinter.h +++ b/upb/pb/textprinter.h @@ -18,11 +18,12 @@ extern "C" { struct _upb_textprinter; typedef struct _upb_textprinter upb_textprinter; -upb_textprinter *upb_textprinter_new(void); +upb_textprinter *upb_textprinter_new(); void upb_textprinter_free(upb_textprinter *p); void upb_textprinter_reset(upb_textprinter *p, upb_bytesink *sink, bool single_line); -upb_mhandlers *upb_textprinter_reghandlers(upb_handlers *h, const upb_msgdef *m); +const upb_handlers *upb_textprinter_newhandlers(const void *owner, + const upb_msgdef *m); #ifdef __cplusplus } /* extern "C" */ diff --git a/upb/pb/varint.c b/upb/pb/varint.c index 45caec1..d6d6161 100644 --- a/upb/pb/varint.c +++ b/upb/pb/varint.c @@ -7,16 +7,64 @@ #include "upb/pb/varint.h" +// A basic branch-based decoder, uses 32-bit values to get good performance +// on 32-bit architectures (but performs well on 64-bits also). +// This scheme comes from the original Google Protobuf implementation (proto2). +upb_decoderet upb_vdecode_max8_branch32(upb_decoderet r) { + upb_decoderet err = {NULL, 0}; + const char *p = r.p; + uint32_t low = (uint32_t)r.val; + uint32_t high = 0; + uint32_t b; + b = *(p++); low |= (b & 0x7fU) << 14; if (!(b & 0x80)) goto done; + b = *(p++); low |= (b & 0x7fU) << 21; if (!(b & 0x80)) goto done; + b = *(p++); low |= (b & 0x7fU) << 28; + high = (b & 0x7fU) >> 4; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 3; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 10; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 17; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 24; if (!(b & 0x80)) goto done; + b = *(p++); high |= (b & 0x7fU) << 31; if (!(b & 0x80)) goto done; + return err; + +done: + r.val = ((uint64_t)high << 32) | low; + r.p = p; + return r; +} + +// Like the previous, but uses 64-bit values. +upb_decoderet upb_vdecode_max8_branch64(upb_decoderet r) { + const char *p = r.p; + uint64_t val = r.val; + uint64_t b; + upb_decoderet err = {NULL, 0}; + b = *(p++); val |= (b & 0x7fU) << 14; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 21; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 28; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 35; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 42; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 49; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 56; if (!(b & 0x80)) goto done; + b = *(p++); val |= (b & 0x7fU) << 63; if (!(b & 0x80)) goto done; + return err; + +done: + r.val = val; + r.p = p; + return r; +} + // Given an encoded varint v, returns an integer with a single bit set that // indicates the end of the varint. Subtracting one from this value will // yield a mask that leaves only bits that are part of the varint. Returns // 0 if the varint is unterminated. -INLINE uint64_t upb_get_vstopbit(uint64_t v) { +static uint64_t upb_get_vstopbit(uint64_t v) { uint64_t cbits = v | 0x7f7f7f7f7f7f7f7fULL; return ~cbits & (cbits+1); } -INLINE uint64_t upb_get_vmask(uint64_t v) { return upb_get_vstopbit(v) - 1; } +// A branchless decoder. Credit to Pascal Massimino for the bit-twiddling. upb_decoderet upb_vdecode_max8_massimino(upb_decoderet r) { uint64_t b; memcpy(&b, r.p, sizeof(b)); @@ -35,14 +83,15 @@ upb_decoderet upb_vdecode_max8_massimino(upb_decoderet r) { return my_r; } +// A branchless decoder. Credit to Daniel Wright for the bit-twiddling. upb_decoderet upb_vdecode_max8_wright(upb_decoderet r) { uint64_t b; memcpy(&b, r.p, sizeof(b)); uint64_t stop_bit = upb_get_vstopbit(b); b &= (stop_bit - 1); - b = ((b & 0x7f007f007f007f00) >> 1) | (b & 0x007f007f007f007f); - b = ((b & 0xffff0000ffff0000) >> 2) | (b & 0x0000ffff0000ffff); - b = ((b & 0xffffffff00000000) >> 4) | (b & 0x00000000ffffffff); + b = ((b & 0x7f007f007f007f00ULL) >> 1) | (b & 0x007f007f007f007fULL); + b = ((b & 0xffff0000ffff0000ULL) >> 2) | (b & 0x0000ffff0000ffffULL); + b = ((b & 0xffffffff00000000ULL) >> 4) | (b & 0x00000000ffffffffULL); if (stop_bit == 0) { // Error: unterminated varint. upb_decoderet err_r = {(void*)0, 0}; diff --git a/upb/pb/varint.h b/upb/pb/varint.h index c0e0134..c4d67ba 100644 --- a/upb/pb/varint.h +++ b/upb/pb/varint.h @@ -49,71 +49,32 @@ typedef struct { uint64_t val; } upb_decoderet; -// A basic branch-based decoder, uses 32-bit values to get good performance -// on 32-bit architectures (but performs well on 64-bits also). -INLINE upb_decoderet upb_vdecode_branch32(const char *p) { - upb_decoderet r = {NULL, 0}; - uint32_t low, high = 0; - uint32_t b; - b = *(p++); low = (b & 0x7f) ; if(!(b & 0x80)) goto done; - b = *(p++); low |= (b & 0x7f) << 7; if(!(b & 0x80)) goto done; - b = *(p++); low |= (b & 0x7f) << 14; if(!(b & 0x80)) goto done; - b = *(p++); low |= (b & 0x7f) << 21; if(!(b & 0x80)) goto done; - b = *(p++); low |= (b & 0x7f) << 28; - high = (b & 0x7f) >> 4; if(!(b & 0x80)) goto done; - b = *(p++); high |= (b & 0x7f) << 3; if(!(b & 0x80)) goto done; - b = *(p++); high |= (b & 0x7f) << 10; if(!(b & 0x80)) goto done; - b = *(p++); high |= (b & 0x7f) << 17; if(!(b & 0x80)) goto done; - b = *(p++); high |= (b & 0x7f) << 24; if(!(b & 0x80)) goto done; - b = *(p++); high |= (b & 0x7f) << 31; if(!(b & 0x80)) goto done; - return r; - -done: - r.val = ((uint64_t)high << 32) | low; - r.p = p; - return r; -} - -// Like the previous, but uses 64-bit values. -INLINE upb_decoderet upb_vdecode_branch64(const char *p) { - uint64_t val; - uint64_t b; - upb_decoderet r = {NULL, 0}; - b = *(p++); val = (b & 0x7f) ; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 7; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 14; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 21; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 28; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 35; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 42; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 49; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 56; if(!(b & 0x80)) goto done; - b = *(p++); val |= (b & 0x7f) << 63; if(!(b & 0x80)) goto done; - return r; - -done: - r.val = val; - r.p = p; - return r; -} - -// Decodes a varint of at most 8 bytes without branching (except for error). +// Four functions for decoding a varint of at most eight bytes. They are all +// functionally identical, but are implemented in different ways and likely have +// different performance profiles. We keep them around for performance testing. +// +// Note that these functions may not read byte-by-byte, so they must not be used +// unless there are at least eight bytes left in the buffer! +upb_decoderet upb_vdecode_max8_branch32(upb_decoderet r); +upb_decoderet upb_vdecode_max8_branch64(upb_decoderet r); upb_decoderet upb_vdecode_max8_wright(upb_decoderet r); - -// Another implementation of the previous. upb_decoderet upb_vdecode_max8_massimino(upb_decoderet r); // Template for a function that checks the first two bytes with branching -// and dispatches 2-10 bytes with a separate function. -#define UPB_VARINT_DECODER_CHECK2(name, decode_max8_function) \ -INLINE upb_decoderet upb_vdecode_check2_ ## name(const char *_p) { \ - uint8_t *p = (uint8_t*)_p; \ - if ((*p & 0x80) == 0) { upb_decoderet r = {_p + 1, *p & 0x7f}; return r; } \ - upb_decoderet r = {_p + 2, (*p & 0x7f) | ((*(p + 1) & 0x7f) << 7)}; \ - if ((*(p + 1) & 0x80) == 0) return r; \ - return decode_max8_function(r); \ +// and dispatches 2-10 bytes with a separate function. Note that this may read +// up to 10 bytes, so it must not be used unless there are at least ten bytes +// left in the buffer! +#define UPB_VARINT_DECODER_CHECK2(name, decode_max8_function) \ +INLINE upb_decoderet upb_vdecode_check2_ ## name(const char *_p) { \ + uint8_t *p = (uint8_t*)_p; \ + if ((*p & 0x80) == 0) { upb_decoderet r = {_p + 1, *p & 0x7fU}; return r; } \ + upb_decoderet r = {_p + 2, (*p & 0x7fU) | ((*(p + 1) & 0x7fU) << 7)}; \ + if ((*(p + 1) & 0x80) == 0) return r; \ + return decode_max8_function(r); \ } +UPB_VARINT_DECODER_CHECK2(branch32, upb_vdecode_max8_branch32); +UPB_VARINT_DECODER_CHECK2(branch64, upb_vdecode_max8_branch64); UPB_VARINT_DECODER_CHECK2(wright, upb_vdecode_max8_wright); UPB_VARINT_DECODER_CHECK2(massimino, upb_vdecode_max8_massimino); #undef UPB_VARINT_DECODER_CHECK2 @@ -121,11 +82,10 @@ UPB_VARINT_DECODER_CHECK2(massimino, upb_vdecode_max8_massimino); // Our canonical functions for decoding varints, based on the currently // favored best-performing implementations. INLINE upb_decoderet upb_vdecode_fast(const char *p) { - // Use nobranch2 on 64-bit, branch32 on 32-bit. if (sizeof(long) == 8) return upb_vdecode_check2_massimino(p); else - return upb_vdecode_branch32(p); + return upb_vdecode_check2_branch32(p); } INLINE upb_decoderet upb_vdecode_max8_fast(upb_decoderet r) { @@ -154,9 +114,9 @@ INLINE size_t upb_vencode64(uint64_t val, char *buf) { if (val == 0) { buf[0] = 0; return 1; } size_t i = 0; while (val) { - uint8_t byte = val & 0x7f; + uint8_t byte = val & 0x7fU; val >>= 7; - if (val) byte |= 0x80; + if (val) byte |= 0x80U; buf[i++] = byte; } return i; @@ -169,7 +129,7 @@ INLINE uint64_t upb_vencode32(uint32_t val) { uint64_t ret = 0; assert(bytes <= 5); memcpy(&ret, buf, bytes); - assert(ret <= 0xffffffffff); + assert(ret <= 0xffffffffffU); return ret; } |