diff options
Diffstat (limited to 'upb/pb')
| -rw-r--r-- | upb/pb/compile_decoder.c | 855 | ||||
| -rw-r--r-- | upb/pb/compile_decoder_x64.c | 368 | ||||
| -rw-r--r-- | upb/pb/compile_decoder_x64.dasc | 1087 | ||||
| -rw-r--r-- | upb/pb/decoder.c | 1198 | ||||
| -rw-r--r-- | upb/pb/decoder.h | 7 | ||||
| -rw-r--r-- | upb/pb/decoder.int.h | 242 | ||||
| -rw-r--r-- | upb/pb/decoder_x64.dasc | 1086 | ||||
| -rw-r--r-- | upb/pb/glue.c | 6 | ||||
| -rw-r--r-- | upb/pb/varint.c | 2 | ||||
| -rw-r--r-- | upb/pb/varint.int.h (renamed from upb/pb/varint.h) | 5 |
10 files changed, 3133 insertions, 1723 deletions
diff --git a/upb/pb/compile_decoder.c b/upb/pb/compile_decoder.c new file mode 100644 index 0000000..c86a171 --- /dev/null +++ b/upb/pb/compile_decoder.c @@ -0,0 +1,855 @@ +/* + * upb - a minimalist implementation of protocol buffers. + * + * Copyright (c) 2013 Google Inc. See LICENSE for details. + * Author: Josh Haberman <jhaberman@gmail.com> + * + * Code to compile a upb::MessageDef into bytecode for decoding that message. + * Bytecode definition is in decoder.int.h. + */ + +#include <stdarg.h> +#include "upb/pb/decoder.int.h" +#include "upb/pb/varint.int.h" +#include "upb/bytestream.h" + +#ifdef UPB_DUMP_BYTECODE +#include <stdio.h> +#endif + +#define MAXLABEL 5 +#define EMPTYLABEL -1 + +/* upb_pbdecodermethod ********************************************************/ + +static upb_pbdecodermethod *newmethod(const upb_msgdef *msg, + const upb_handlers *dest_handlers) { + upb_pbdecodermethod *ret = malloc(sizeof(upb_pbdecodermethod)); + ret->msg = msg; + ret->dest_handlers = dest_handlers; + ret->native_code = false; // If we JIT, it will update this later. + upb_inttable_init(&ret->dispatch, UPB_CTYPE_UINT64); + + if (ret->dest_handlers) { + upb_handlers_ref(ret->dest_handlers, ret); + } + return ret; +} + +static void freemethod(upb_pbdecodermethod *method) { + if (method->dest_handlers) { + upb_handlers_unref(method->dest_handlers, method); + } + + upb_inttable_uninit(&method->dispatch); + free(method); +} + + +/* upb_pbdecoderplan **********************************************************/ + +upb_pbdecoderplan *newplan() { + upb_pbdecoderplan *p = malloc(sizeof(*p)); + upb_inttable_init(&p->methods, UPB_CTYPE_PTR); + p->code = NULL; + p->code_end = NULL; + return p; +} + +void freeplan(void *_p) { + upb_pbdecoderplan *p = _p; + + upb_inttable_iter i; + upb_inttable_begin(&i, &p->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i)); + freemethod(method); + } + upb_inttable_uninit(&p->methods); + free(p->code); +#ifdef UPB_USE_JIT_X64 + upb_pbdecoder_freejit(p); +#endif + free(p); +} + +void set_bytecode_handlers(upb_pbdecoderplan *p, upb_handlers *h) { + upb_handlers_setstartstr(h, UPB_BYTESTREAM_BYTES, upb_pbdecoder_start, p, + NULL); + upb_handlers_setstring(h, UPB_BYTESTREAM_BYTES, upb_pbdecoder_decode, p, + freeplan); + upb_handlers_setendstr(h, UPB_BYTESTREAM_BYTES, upb_pbdecoder_end, p, NULL); +} + +static const upb_pbdecoderplan *getdecoderplan(const upb_handlers *h) { + if (upb_handlers_frametype(h) != &upb_pbdecoder_frametype) + return NULL; + upb_selector_t sel; + if (!upb_handlers_getselector(UPB_BYTESTREAM_BYTES, UPB_HANDLER_STARTSTR, + &sel)) { + return NULL; + } + return upb_handlers_gethandlerdata(h, sel); +} + + +/* compiler *******************************************************************/ + +// Data used only at compilation time. +typedef struct { + upb_pbdecoderplan *plan; + + uint32_t *pc; + int fwd_labels[MAXLABEL]; + int back_labels[MAXLABEL]; +} compiler; + +static compiler *newcompiler(upb_pbdecoderplan *plan) { + compiler *ret = malloc(sizeof(compiler)); + ret->plan = plan; + for (int i = 0; i < MAXLABEL; i++) { + ret->fwd_labels[i] = EMPTYLABEL; + ret->back_labels[i] = EMPTYLABEL; + } + return ret; +} + +static void freecompiler(compiler *c) { + free(c); +} + +const size_t ptr_words = sizeof(void*) / sizeof(uint32_t); + +// How many words an instruction is. +static int instruction_len(uint32_t instr) { + switch (getop(instr)) { + case OP_SETDISPATCH: return 1 + ptr_words; + case OP_TAGN: return 3; + case OP_SETBIGGROUPNUM: return 2; + default: return 1; + } +} + +bool op_has_longofs(int32_t instruction) { + switch (getop(instruction)) { + case OP_CALL: + case OP_BRANCH: + case OP_CHECKDELIM: + return true; + // The "tag" instructions only have 8 bytes available for the jump target, + // but that is ok because these opcodes only require short jumps. + case OP_TAG1: + case OP_TAG2: + case OP_TAGN: + return false; + default: + assert(false); + return false; + } +} + +static int32_t getofs(uint32_t instruction) { + if (op_has_longofs(instruction)) { + return (int32_t)instruction >> 8; + } else { + return (int8_t)(instruction >> 8); + } +} + +static void setofs(uint32_t *instruction, int32_t ofs) { + if (op_has_longofs(*instruction)) { + *instruction = getop(*instruction) | ofs << 8; + } else { + *instruction = (*instruction & ~0xff00) | ((ofs & 0xff) << 8); + } + assert(getofs(*instruction) == ofs); // Would fail in cases of overflow. +} + +static uint32_t pcofs(compiler *c) { return c->pc - c->plan->code; } + +// Defines a local label at the current PC location. All previous forward +// references are updated to point to this location. The location is noted +// for any future backward references. +static void label(compiler *c, unsigned int label) { + assert(label < MAXLABEL); + int val = c->fwd_labels[label]; + uint32_t *codep = (val == EMPTYLABEL) ? NULL : c->plan->code + val; + while (codep) { + int ofs = getofs(*codep); + setofs(codep, c->pc - codep - instruction_len(*codep)); + codep = ofs ? codep + ofs : NULL; + } + c->fwd_labels[label] = EMPTYLABEL; + c->back_labels[label] = pcofs(c); +} + +// Creates a reference to a numbered label; either a forward reference +// (positive arg) or backward reference (negative arg). For forward references +// the value returned now is actually a "next" pointer into a linked list of all +// instructions that use this label and will be patched later when the label is +// defined with label(). +// +// The returned value is the offset that should be written into the instruction. +static int32_t labelref(compiler *c, int label) { + assert(label < MAXLABEL); + if (label == LABEL_DISPATCH) { + // No resolving required. + return 0; + } else if (label < 0) { + // Backward local label. Relative to the next instruction. + uint32_t from = (c->pc + 1) - c->plan->code; + return c->back_labels[-label] - from; + } else { + // Forward local label: prepend to (possibly-empty) linked list. + int *lptr = &c->fwd_labels[label]; + int32_t ret = (*lptr == EMPTYLABEL) ? 0 : *lptr - pcofs(c); + *lptr = pcofs(c); + return ret; + } +} + +static void put32(compiler *c, uint32_t v) { + if (c->pc == c->plan->code_end) { + int ofs = pcofs(c); + size_t oldsize = c->plan->code_end - c->plan->code; + size_t newsize = UPB_MAX(oldsize * 2, 64); + // TODO(haberman): handle OOM. + c->plan->code = realloc(c->plan->code, newsize * sizeof(uint32_t)); + c->plan->code_end = c->plan->code + newsize; + c->pc = c->plan->code + ofs; + } + *c->pc++ = v; +} + +static void putop(compiler *c, opcode op, ...) { + va_list ap; + va_start(ap, op); + + switch (op) { + case OP_SETDISPATCH: { + uintptr_t ptr = (uintptr_t)va_arg(ap, void*); + put32(c, OP_SETDISPATCH); + put32(c, ptr); + if (sizeof(uintptr_t) > sizeof(uint32_t)) + put32(c, (uint64_t)ptr >> 32); + break; + } + case OP_STARTMSG: + case OP_ENDMSG: + case OP_PUSHTAGDELIM: + case OP_PUSHLENDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_HALT: + put32(c, op); + break; + case OP_PARSE_DOUBLE: + case OP_PARSE_FLOAT: + case OP_PARSE_INT64: + case OP_PARSE_UINT64: + case OP_PARSE_INT32: + case OP_PARSE_FIXED64: + case OP_PARSE_FIXED32: + case OP_PARSE_BOOL: + case OP_PARSE_UINT32: + case OP_PARSE_SFIXED32: + case OP_PARSE_SFIXED64: + case OP_PARSE_SINT32: + case OP_PARSE_SINT64: + case OP_STARTSEQ: + case OP_SETGROUPNUM: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_STRING: + case OP_ENDSTR: + put32(c, op | va_arg(ap, upb_selector_t) << 8); + break; + case OP_SETBIGGROUPNUM: + put32(c, op); + put32(c, va_arg(ap, int)); + break; + case OP_CALL: { + const upb_pbdecodermethod *method = va_arg(ap, upb_pbdecodermethod *); + put32(c, op | (method->base.ofs - (pcofs(c) + 1)) << 8); + break; + } + case OP_CHECKDELIM: + case OP_BRANCH: { + uint32_t instruction = op; + int label = va_arg(ap, int); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + break; + } + case OP_TAG1: + case OP_TAG2: { + int label = va_arg(ap, int); + uint64_t tag = va_arg(ap, uint64_t); + uint32_t instruction = op | (tag << 16); + assert(tag <= 0xffff); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + break; + } + case OP_TAGN: { + int label = va_arg(ap, int); + uint64_t tag = va_arg(ap, uint64_t); + uint32_t instruction = op | (upb_value_size(tag) << 16); + setofs(&instruction, labelref(c, label)); + put32(c, instruction); + put32(c, tag); + put32(c, tag >> 32); + break; + } + } + + va_end(ap); +} + +#if defined(UPB_USE_JIT_X64) || defined(UPB_DUMP_BYTECODE) + +const char *upb_pbdecoder_getopname(unsigned int op) { +#define OP(op) [OP_ ## op] = "OP_" #op +#define T(op) OP(PARSE_##op) + static const char *names[] = { + "<no opcode>", + T(DOUBLE), T(FLOAT), T(INT64), T(UINT64), T(INT32), T(FIXED64), T(FIXED32), + T(BOOL), T(UINT32), T(SFIXED32), T(SFIXED64), T(SINT32), T(SINT64), + OP(STARTMSG), OP(ENDMSG), OP(STARTSEQ), OP(ENDSEQ), OP(STARTSUBMSG), + OP(ENDSUBMSG), OP(STARTSTR), OP(STRING), OP(ENDSTR), OP(CALL), + OP(PUSHLENDELIM), OP(PUSHTAGDELIM), OP(SETDELIM), OP(CHECKDELIM), + OP(BRANCH), OP(TAG1), OP(TAG2), OP(TAGN), OP(SETDISPATCH), OP(POP), + OP(SETGROUPNUM), OP(SETBIGGROUPNUM), OP(HALT), + }; + return op > OP_HALT ? names[0] : names[op]; +#undef OP +#undef T +} + +#endif + +#ifdef UPB_DUMP_BYTECODE + +static void dumpbc(uint32_t *p, uint32_t *end, FILE *f) { + + uint32_t *begin = p; + + while (p < end) { + fprintf(f, "%p %8tx", p, p - begin); + uint32_t instr = *p++; + uint8_t op = getop(instr); + fprintf(f, " %s", upb_pbdecoder_getopname(op)); + switch ((opcode)op) { + case OP_SETDISPATCH: { + const upb_inttable *dispatch; + memcpy(&dispatch, p, sizeof(void*)); + p += ptr_words; + const upb_pbdecodermethod *method = + (void *)((char *)dispatch - + offsetof(upb_pbdecodermethod, dispatch)); + fprintf(f, " %s", upb_msgdef_fullname(method->msg)); + break; + } + case OP_STARTMSG: + case OP_ENDMSG: + case OP_PUSHLENDELIM: + case OP_PUSHTAGDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_HALT: + break; + case OP_PARSE_DOUBLE: + case OP_PARSE_FLOAT: + case OP_PARSE_INT64: + case OP_PARSE_UINT64: + case OP_PARSE_INT32: + case OP_PARSE_FIXED64: + case OP_PARSE_FIXED32: + case OP_PARSE_BOOL: + case OP_PARSE_UINT32: + case OP_PARSE_SFIXED32: + case OP_PARSE_SFIXED64: + case OP_PARSE_SINT32: + case OP_PARSE_SINT64: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_STRING: + case OP_ENDSTR: + case OP_SETGROUPNUM: + fprintf(f, " %d", instr >> 8); + break; + case OP_SETBIGGROUPNUM: + fprintf(f, " %d", *p++); + break; + case OP_CHECKDELIM: + case OP_CALL: + case OP_BRANCH: + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + break; + case OP_TAG1: + case OP_TAG2: { + fprintf(f, " tag:0x%x", instr >> 16); + if (getofs(instr)) { + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + } + break; + } + case OP_TAGN: { + uint64_t tag = *p++; + tag |= (uint64_t)*p++ << 32; + fprintf(f, " tag:0x%llx", (long long)tag); + fprintf(f, " n:%d", instr >> 16); + if (getofs(instr)) { + fprintf(f, " =>0x%tx", p + getofs(instr) - begin); + } + break; + } + } + fputs("\n", f); + } +} + +#endif + +static uint64_t get_encoded_tag(const upb_fielddef *f, int wire_type) { + uint32_t tag = (upb_fielddef_number(f) << 3) | wire_type; + uint64_t encoded_tag = upb_vencode32(tag); + // No tag should be greater than 5 bytes. + assert(encoded_tag <= 0xffffffffff); + return encoded_tag; +} + +static void putchecktag(compiler *c, const upb_fielddef *f, + int wire_type, int dest) { + uint64_t tag = get_encoded_tag(f, wire_type); + switch (upb_value_size(tag)) { + case 1: + putop(c, OP_TAG1, dest, tag); + break; + case 2: + putop(c, OP_TAG2, dest, tag); + break; + default: + putop(c, OP_TAGN, dest, tag); + break; + } +} + +static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) { + upb_selector_t selector; + bool ok = upb_handlers_getselector(f, type, &selector); + UPB_ASSERT_VAR(ok, ok); + return selector; +} + +// Marks the current bytecode position as the dispatch target for this message, +// field, and wire type. +// +static void dispatchtarget(compiler *c, upb_pbdecodermethod *method, + const upb_fielddef *f, int wire_type) { + // Offset is relative to msg base. + uint64_t ofs = pcofs(c) - method->base.ofs; + uint32_t fn = upb_fielddef_number(f); + upb_inttable *d = &method->dispatch; + upb_value v; + if (upb_inttable_remove(d, fn, &v)) { + // TODO: prioritize based on packed setting in .proto file. + uint64_t oldval = upb_value_getuint64(v); + assert(((oldval >> 8) & 0xff) == 0); // wt2 should not be set yet. + upb_inttable_insert(d, fn, upb_value_uint64(oldval | (wire_type << 8))); + upb_inttable_insert(d, fn + UPB_MAX_FIELDNUMBER, upb_value_uint64(ofs)); + } else { + upb_inttable_insert(d, fn, upb_value_uint64((ofs << 16) | wire_type)); + } +} + +static void putpush(compiler *c, const upb_fielddef *f) { + if (upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_MESSAGE) { + putop(c, OP_PUSHLENDELIM); + } else { + uint32_t fn = upb_fielddef_number(f); + putop(c, OP_PUSHTAGDELIM); + if (fn >= 1 << 24) { + putop(c, OP_SETBIGGROUPNUM, fn); + } else { + putop(c, OP_SETGROUPNUM, fn); + } + } +} + +static upb_pbdecodermethod *find_submethod(const compiler *c, + const upb_pbdecodermethod *method, + const upb_fielddef *f) { + const void *key = method->dest_handlers ? + (const void*)upb_handlers_getsubhandlers(method->dest_handlers, f) : + (const void*)upb_downcast_msgdef(upb_fielddef_subdef(f)); + upb_value v; + bool ok = upb_inttable_lookupptr(&c->plan->methods, key, &v); + UPB_ASSERT_VAR(ok, ok); + return upb_value_getptr(v); +} + +// Adds bytecode for parsing the given message to the given decoderplan, +// while adding all dispatch targets to this message's dispatch table. +static void compile_method(compiler *c, upb_pbdecodermethod *method) { + assert(method); + + // Symbolic names for our local labels. + const int LABEL_LOOPSTART = 1; // Top of a repeated field loop. + const int LABEL_LOOPBREAK = 2; // To jump out of a repeated loop + const int LABEL_FIELD = 3; // Jump backward to find the most recent field. + const int LABEL_ENDMSG = 4; // To reach the OP_ENDMSG instr for this msg. + + // Index is descriptor type. + static const uint8_t native_wire_types[] = { + UPB_WIRE_TYPE_END_GROUP, // ENDGROUP + UPB_WIRE_TYPE_64BIT, // DOUBLE + UPB_WIRE_TYPE_32BIT, // FLOAT + UPB_WIRE_TYPE_VARINT, // INT64 + UPB_WIRE_TYPE_VARINT, // UINT64 + UPB_WIRE_TYPE_VARINT, // INT32 + UPB_WIRE_TYPE_64BIT, // FIXED64 + UPB_WIRE_TYPE_32BIT, // FIXED32 + UPB_WIRE_TYPE_VARINT, // BOOL + UPB_WIRE_TYPE_DELIMITED, // STRING + UPB_WIRE_TYPE_START_GROUP, // GROUP + UPB_WIRE_TYPE_DELIMITED, // MESSAGE + UPB_WIRE_TYPE_DELIMITED, // BYTES + UPB_WIRE_TYPE_VARINT, // UINT32 + UPB_WIRE_TYPE_VARINT, // ENUM + UPB_WIRE_TYPE_32BIT, // SFIXED32 + UPB_WIRE_TYPE_64BIT, // SFIXED64 + UPB_WIRE_TYPE_VARINT, // SINT32 + UPB_WIRE_TYPE_VARINT, // SINT64 + }; + + // Clear all entries in the dispatch table. + upb_inttable_uninit(&method->dispatch); + upb_inttable_init(&method->dispatch, UPB_CTYPE_UINT64); + + method->base.ofs = pcofs(c); + putop(c, OP_SETDISPATCH, &method->dispatch); + putop(c, OP_STARTMSG); + label(c, LABEL_FIELD); + upb_msg_iter i; + for(upb_msg_begin(&i, method->msg); !upb_msg_done(&i); upb_msg_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + upb_descriptortype_t type = upb_fielddef_descriptortype(f); + + // From a decoding perspective, ENUM is the same as INT32. + if (type == UPB_DESCRIPTOR_TYPE_ENUM) + type = UPB_DESCRIPTOR_TYPE_INT32; + + label(c, LABEL_FIELD); + + switch (upb_fielddef_type(f)) { + case UPB_TYPE_MESSAGE: { + const upb_pbdecodermethod *sub_m = find_submethod(c, method, f); + int wire_type = (type == UPB_DESCRIPTOR_TYPE_MESSAGE) ? + UPB_WIRE_TYPE_DELIMITED : UPB_WIRE_TYPE_START_GROUP; + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putop(c, OP_PUSHTAGDELIM); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); + label(c, LABEL_LOOPSTART); + putpush(c, f); + putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG)); + putop(c, OP_CALL, sub_m); + putop(c, OP_POP); + putop(c, OP_ENDSUBMSG, getsel(f, UPB_HANDLER_ENDSUBMSG)); + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + putop(c, OP_SETDELIM); + } + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, wire_type, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); + putop(c, OP_ENDSEQ, getsel(f, UPB_HANDLER_ENDSEQ)); + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putpush(c, f); + putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG)); + putop(c, OP_CALL, sub_m); + putop(c, OP_POP); + putop(c, OP_ENDSUBMSG, getsel(f, UPB_HANDLER_ENDSUBMSG)); + if (wire_type == UPB_WIRE_TYPE_DELIMITED) { + putop(c, OP_SETDELIM); + } + } + break; + } + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHTAGDELIM); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); + label(c, LABEL_LOOPSTART); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR)); + putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING)); + putop(c, OP_POP); + putop(c, OP_ENDSTR, getsel(f, UPB_HANDLER_ENDSTR)); + putop(c, OP_SETDELIM); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); + putop(c, OP_ENDSEQ, getsel(f, UPB_HANDLER_ENDSEQ)); + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR)); + putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING)); + putop(c, OP_POP); + putop(c, OP_ENDSTR, getsel(f, UPB_HANDLER_ENDSTR)); + putop(c, OP_SETDELIM); + } + break; + default: { + opcode parse_type = (opcode)type; + assert(parse_type >= 0 && parse_type <= OP_MAX); + upb_selector_t sel = getsel(f, upb_handlers_getprimitivehandlertype(f)); + int wire_type = native_wire_types[upb_fielddef_descriptortype(f)]; + if (upb_fielddef_isseq(f)) { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH); + dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED); + putop(c, OP_PUSHLENDELIM); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); // Packed + label(c, LABEL_LOOPSTART); + putop(c, parse_type, sel); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + dispatchtarget(c, method, f, wire_type); + putop(c, OP_PUSHTAGDELIM); + putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); // Non-packed + label(c, LABEL_LOOPSTART); + putop(c, parse_type, sel); + putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK); + putchecktag(c, f, wire_type, LABEL_LOOPBREAK); + putop(c, OP_BRANCH, -LABEL_LOOPSTART); + label(c, LABEL_LOOPBREAK); + putop(c, OP_POP); // Packed and non-packed join. + putop(c, OP_ENDSEQ, getsel(f, UPB_HANDLER_ENDSEQ)); + putop(c, OP_SETDELIM); // Could remove for non-packed by dup ENDSEQ. + } else { + putop(c, OP_CHECKDELIM, LABEL_ENDMSG); + putchecktag(c, f, wire_type, LABEL_DISPATCH); + dispatchtarget(c, method, f, wire_type); + putop(c, parse_type, sel); + } + } + } + } + // For now we just loop back to the last field of the message (or if none, + // the DISPATCH opcode for the message. + putop(c, OP_BRANCH, -LABEL_FIELD); + label(c, LABEL_ENDMSG); + putop(c, OP_ENDMSG); + + upb_inttable_compact(&method->dispatch); +} + +// Populate "methods" with new upb_pbdecodermethod objects reachable from "md". +// "h" can be NULL, in which case the methods will not be statically bound to +// destination handlers. +// +// Returns the method for this msgdef/handlers. +// +// Note that there is a deep difference between keying the method table on +// upb_msgdef and keying it on upb_handlers. Since upb_msgdef : upb_handlers +// can be 1:many, binding a handlers statically can result in *more* methods +// being generated than if the methods are dynamically-bound. +// +// On the other hand, if/when the optimization mentioned below is implemented, +// binding to a upb_handlers can result in *fewer* methods being generated if +// many of the submessages have no handlers bound to them. +static upb_pbdecodermethod *find_methods(compiler *c, + const upb_msgdef *md, + const upb_handlers *h) { + const void *key = h ? (const void*)h : (const void*)md; + upb_value v; + if (upb_inttable_lookupptr(&c->plan->methods, key, &v)) + return upb_value_getptr(v); + upb_pbdecodermethod *method = newmethod(md, h); + // Takes ownership of method. + upb_inttable_insertptr(&c->plan->methods, key, upb_value_ptr(method)); + + upb_msg_iter i; + for(upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) { + const upb_fielddef *f = upb_msg_iter_field(&i); + if (upb_fielddef_type(f) != UPB_TYPE_MESSAGE) + continue; + const upb_handlers *sub_h = h ? upb_handlers_getsubhandlers(h, f) : NULL; + + if (h && !sub_h && + upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_MESSAGE) { + // OPT: We could optimize away the sub-method, but would have to make sure + // this field is compiled as a string instead of a submessage. + } + + find_methods(c, upb_downcast_msgdef(upb_fielddef_subdef(f)), sub_h); + } + + return method; +} + +// (Re-)compile bytecode for all messages in "msgs", ensuring that the code +// for "md" is emitted first. Overwrites any existing bytecode in "c". +static void compile_methods(compiler *c) { + // Start over at the beginning of the bytecode. + c->pc = c->plan->code; + compile_method(c, c->plan->topmethod); + + upb_inttable_iter i; + upb_inttable_begin(&i, &c->plan->methods); + for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i)); + if (method != c->plan->topmethod) { + compile_method(c, method); + } + } +} + + +/* JIT setup. ******************************************************************/ + +#ifdef UPB_USE_JIT_X64 + +static void sethandlers(upb_pbdecoderplan *p, upb_handlers *h, bool allowjit) { + p->jit_code = NULL; + + if (allowjit) { + upb_pbdecoder_jit(p); // Compile byte-code into machine code. + upb_handlers_setstartstr(h, UPB_BYTESTREAM_BYTES, upb_pbdecoder_start, p, + freeplan); + upb_handlers_setstring(h, UPB_BYTESTREAM_BYTES, p->jit_code, NULL, NULL); + upb_handlers_setendstr(h, UPB_BYTESTREAM_BYTES, upb_pbdecoder_end, p, NULL); + } else { + set_bytecode_handlers(p, h); + } +} + +static bool bind_dynamic(bool allowjit) { + // For the moment, JIT handlers always bind statically, but bytecode handlers + // never do. + return !allowjit; +} + +#else // UPB_USE_JIT_X64 + +static void sethandlers(upb_pbdecoderplan *p, upb_handlers *h, bool allowjit) { + // No JIT compiled in; use bytecode handlers unconditionally. + UPB_UNUSED(allowjit); + set_bytecode_handlers(p, h); +} + +static bool bind_dynamic(bool allowjit) { + // Bytecode handlers never bind statically. + return true; +} + +#endif // UPB_USE_JIT_X64 + + +/* Public interface ***********************************************************/ + +bool upb_pbdecoder_isdecoder(const upb_handlers *h) { + return getdecoderplan(h) != NULL; +} + +bool upb_pbdecoderplan_hasjitcode(const upb_pbdecoderplan *p) { +#ifdef UPB_USE_JIT_X64 + return p->jit_code != NULL; +#else + UPB_UNUSED(p); + return false; +#endif +} + +bool upb_pbdecoder_hasjitcode(const upb_handlers *h) { + const upb_pbdecoderplan *p = getdecoderplan(h); + if (!p) return false; + return upb_pbdecoderplan_hasjitcode(p); +} + +uint32_t *upb_pbdecoderplan_codebase(const upb_pbdecoderplan *p) { + return p->code; +} + +upb_string_handler *upb_pbdecoderplan_jitcode(const upb_pbdecoderplan *p) { +#ifdef UPB_USE_JIT_X64 + return p->jit_code; +#else + assert(false); + return NULL; +#endif +} + +const upb_handlers *upb_pbdecoder_getdesthandlers(const upb_handlers *h) { + const upb_pbdecoderplan *p = getdecoderplan(h); + if (!p) return NULL; + return p->topmethod->dest_handlers; +} + +const upb_handlers *upb_pbdecoder_gethandlers(const upb_handlers *dest, + bool allowjit, + const void *owner) { + UPB_UNUSED(allowjit); + assert(upb_handlers_isfrozen(dest)); + const upb_msgdef *md = upb_handlers_msgdef(dest); + + upb_pbdecoderplan *p = newplan(); + compiler *c = newcompiler(p); + + if (bind_dynamic(allowjit)) { + // If binding dynamically, remove the reference against destination + // handlers. + dest = NULL; + } + + p->topmethod = find_methods(c, md, dest); + + // We compile in two passes: + // 1. all messages are assigned relative offsets from the beginning of the + // bytecode (saved in method->base). + // 2. forwards OP_CALL instructions can be correctly linked since message + // offsets have been previously assigned. + // + // Could avoid the second pass by linking OP_CALL instructions somehow. + compile_methods(c); + compile_methods(c); + p->code_end = c->pc; + +#ifdef UPB_DUMP_BYTECODE + FILE *f = fopen("/tmp/upb-bytecode", "wb"); + assert(f); + dumpbc(p->code, p->code_end, stderr); + dumpbc(p->code, p->code_end, f); + fclose(f); +#endif + + upb_handlers *h = upb_handlers_new( + UPB_BYTESTREAM, &upb_pbdecoder_frametype, owner); + sethandlers(p, h, allowjit); + + freecompiler(c); + + return h; +} diff --git a/upb/pb/compile_decoder_x64.c b/upb/pb/compile_decoder_x64.c new file mode 100644 index 0000000..214ab35 --- /dev/null +++ b/upb/pb/compile_decoder_x64.c @@ -0,0 +1,368 @@ +/* + * upb - a minimalist implementation of protocol buffers. + * + * Copyright (c) 2013 Google Inc. See LICENSE for details. + * Author: Josh Haberman <jhaberman@gmail.com> + * + * Driver code for the x64 JIT compiler. + */ + +#include <dlfcn.h> +#include <stdio.h> +#include <sys/mman.h> +#include "upb/pb/decoder.h" +#include "upb/pb/decoder.int.h" +#include "upb/pb/varint.int.h" +#include "upb/shim/shim.h" + +// These defines are necessary for DynASM codegen. +// See dynasm/dasm_proto.h for more info. +#define Dst_DECL jitcompiler *jc +#define Dst_REF (jc->dynasm) +#define Dst (jc) + +// In debug mode, make DynASM do internal checks (must be defined before any +// dasm header is included. +#ifndef NDEBUG +#define DASM_CHECKS +#endif + +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +#define DECODE_EOF -3 + +typedef struct { + upb_pbdecoderplan *plan; + uint32_t *pc; + + // This pointer is allocated by dasm_init() and freed by dasm_free(). + struct dasm_State *dynasm; + + // Maps bytecode pc location -> pclabel. + upb_inttable pclabels; + upb_inttable pcdefined; + + // For marking labels that should go into the generated code. + // Maps pclabel -> char* label (string is owned by the table). + upb_inttable asmlabels; + + // For checking that two asmlabels aren't defined for the same byte. + int lastlabelofs; + + // The total number of pclabels currently defined. + uint32_t pclabel_count; + + // Used by DynASM to store globals. + void **globals; + + bool usefp; + bool chkret; +} jitcompiler; + +// Functions called by codegen. +static int pclabel(jitcompiler *jc, const void *here); +static int define_pclabel(jitcompiler *jc, const void *here); +static void asmlabel(jitcompiler *jc, const char *fmt, ...); + +#include "dynasm/dasm_proto.h" +#include "dynasm/dasm_x86.h" +#include "upb/pb/compile_decoder_x64.h" + +static jitcompiler *newjitcompiler(upb_pbdecoderplan *plan) { + jitcompiler *jc = malloc(sizeof(jitcompiler)); + jc->usefp = false; + jc->chkret = false; + jc->plan = plan; + jc->pclabel_count = 0; + jc->lastlabelofs = -1; + upb_inttable_init(&jc->pclabels, UPB_CTYPE_UINT32); + upb_inttable_init(&jc->pcdefined, UPB_CTYPE_BOOL); + upb_inttable_init(&jc->asmlabels, UPB_CTYPE_PTR); + jc->globals = malloc(UPB_JIT_GLOBAL__MAX * sizeof(*jc->globals)); + + dasm_init(jc, 1); + dasm_setupglobal(jc, jc->globals, UPB_JIT_GLOBAL__MAX); + dasm_setup(jc, upb_jit_actionlist); + + return jc; +} + +static void freejitcompiler(jitcompiler *jc) { + upb_inttable_iter i; + upb_inttable_begin(&i, &jc->asmlabels); + for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { + free(upb_value_getptr(upb_inttable_iter_value(&i))); + } + upb_inttable_uninit(&jc->asmlabels); + upb_inttable_uninit(&jc->pclabels); + upb_inttable_uninit(&jc->pcdefined); + dasm_free(jc); + free(jc->globals); + free(jc); +} + +// Returns a pclabel associated with the given arbitrary pointer. +static int pclabel(jitcompiler *jc, const void *here) { + upb_value v; + bool found = upb_inttable_lookupptr(&jc->pclabels, here, &v); + if (!found) { + upb_value_setuint32(&v, jc->pclabel_count++); + dasm_growpc(jc, jc->pclabel_count); + upb_inttable_insertptr(&jc->pclabels, here, v); + } + return upb_value_getuint32(v); +} + +// Defines a pclabel associated with the given arbitrary pointer. +// May only be called once (to avoid redefining the pclabel). +static int define_pclabel(jitcompiler *jc, const void *here) { + // Will assert-fail if it already exists. + upb_inttable_insertptr(&jc->pcdefined, here, upb_value_bool(true)); + return pclabel(jc, here); +} + +static void upb_reg_jit_gdb(jitcompiler *jc); + +// Given a pcofs relative to method, returns the machine code offset for it +// (relative to the beginning of the machine code). +int nativeofs(jitcompiler *jc, const upb_pbdecodermethod *method, int pcofs) { + void *target = jc->plan->code + method->base.ofs + pcofs; + return dasm_getpclabel(jc, pclabel(jc, target)); +} + +// Given a pcofs relative to this method's base, returns a machine code offset +// relative to pclabel(dispatch->array) (which is used in jitdispatch as the +// machine code base for dispatch table lookups). +uint32_t dispatchofs(jitcompiler *jc, const upb_pbdecodermethod *method, + int pcofs) { + int ofs1 = dasm_getpclabel(jc, pclabel(jc, method->dispatch.array)); + int ofs2 = nativeofs(jc, method, pcofs); + assert(ofs1 > 0); + assert(ofs2 > 0); + int ret = ofs2 - ofs1; + assert(ret > 0); + return ret; +} + +// Rewrites the dispatch tables into machine code offsets. +static void patchdispatch(jitcompiler *jc) { + upb_inttable_iter i; + upb_inttable_begin(&i, &jc->plan->methods); + for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i)); + upb_inttable *dispatch = &method->dispatch; + upb_inttable_iter i2; + upb_inttable_begin(&i2, dispatch); + for (; !upb_inttable_done(&i2); upb_inttable_next(&i2)) { + uintptr_t key = upb_inttable_iter_key(&i2); + if (key == 0) continue; + uint64_t val = upb_value_getuint64(upb_inttable_iter_value(&i2)); + uint64_t newval; + if (key <= UPB_MAX_FIELDNUMBER) { + // Primary slot. + uint64_t oldofs = val >> 16; + uint64_t newofs = dispatchofs(jc, method, oldofs); + newval = (val & 0xffff) | (newofs << 16); + assert((int64_t)newval > 0); + } else { + // Secondary slot. Since we have 64 bits for the value, we use an + // absolute offset. + newval = (uint64_t)(jc->plan->jit_code + nativeofs(jc, method, val)); + } + bool ok = upb_inttable_replace(dispatch, key, upb_value_uint64(newval)); + UPB_ASSERT_VAR(ok, ok); + } + } +} + +// Define for JIT debugging. +#ifdef UPB_JIT_LOAD_SO +static void load_so(jitcompiler *jc) { + // Dump to a .so file in /tmp and load that, so all the tooling works right + // (for example, debuggers and profilers will see symbol names for the JIT-ted + // code). This is the same goal of the GDB JIT code below, but the GDB JIT + // interface is only used/understood by GDB. Hopefully a standard will + // develop for registering JIT-ted code that all tools will recognize, + // rendering this obsolete. + // + // Requires that gcc is available from the command-line. + + // Convert all asm labels from pclabel offsets to machine code offsets. + upb_inttable_iter i; + upb_inttable mclabels; + upb_inttable_init(&mclabels, UPB_CTYPE_PTR); + upb_inttable_begin(&i, &jc->asmlabels); + for (; !upb_inttable_done(&i); upb_inttable_next(&i)) { + upb_inttable_insert(&mclabels, + dasm_getpclabel(jc, upb_inttable_iter_key(&i)), + upb_inttable_iter_value(&i)); + } + + FILE *f = fopen("/tmp/upb-jit-code.s", "w"); + if (f) { + fputs(" .text\n\n", f); + size_t linelen = 0; + for (size_t i = 0; i < jc->plan->jit_size; i++) { + upb_value v; + if (upb_inttable_lookup(&mclabels, i, &v)) { + const char *label = upb_value_getptr(v); + // "X." makes our JIT syms recognizable as such, which we build into + // other tooling. + fprintf(f, "\n\nX.%s:\n", label); + fprintf(f, " .globl X.%s", label); + linelen = 1000; + } + if (linelen >= 77) { + linelen = fprintf(f, "\n .byte %u", jit_code[i]); + } else { + linelen += fprintf(f, ",%u", jit_code[i]); + } + } + fputs("\n", f); + fclose(f); + } else { + fprintf(stderr, "Couldn't open /tmp/upb-jit-code.s for writing/\n"); + } + + // TODO: racy + if (system("gcc -shared -o /tmp/upb-jit-code.so /tmp/upb-jit-code.s") != 0) { + abort(); + } + + jc->dl = dlopen("/tmp/upb-jit-code.so", RTLD_LAZY); + if (!jc->dl) { + fprintf(stderr, "Couldn't dlopen(): %s\n", dlerror()); + abort(); + } + + munmap(jit_code, jc->plan->jit_size); + jit_code = dlsym(jc->dl, "X.enterjit"); + if (!jit_code) { + fprintf(stderr, "Couldn't find enterjit sym\n"); + abort(); + } + + upb_inttable_uninit(&mclabels); +} +#endif + +void upb_pbdecoder_jit(upb_pbdecoderplan *plan) { + plan->debug_info = NULL; + plan->dl = NULL; + + jitcompiler *jc = newjitcompiler(plan); + emit_static_asm(jc); + jitbytecode(jc); + + int dasm_status = dasm_link(jc, &jc->plan->jit_size); + if (dasm_status != DASM_S_OK) { + fprintf(stderr, "DynASM error; returned status: 0x%08x\n", dasm_status); + abort(); + } + + char *jit_code = mmap(NULL, jc->plan->jit_size, PROT_READ | PROT_WRITE, + MAP_32BIT | MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); + dasm_encode(jc, jit_code); + mprotect(jit_code, jc->plan->jit_size, PROT_EXEC | PROT_READ); + upb_reg_jit_gdb(jc); + +#ifdef UPB_JIT_LOAD_SO + load_so(jc); +#endif + + jc->plan->jit_code = (upb_string_handler *)jit_code; + patchdispatch(jc); + freejitcompiler(jc); +} + +void upb_pbdecoder_freejit(upb_pbdecoderplan *plan) { + if (!plan->jit_code) return; + if (plan->dl) { + dlclose(plan->dl); + } else { + munmap(plan->jit_code, plan->jit_size); + } + free(plan->debug_info); + // TODO: unregister GDB JIT interface. +} + +// To debug JIT-ted code with GDB we need to tell GDB about the JIT-ted code +// at runtime. GDB 7.x+ has defined an interface for doing this, and these +// structure/function defintions are copied out of gdb/jit.h +// +// We need to give GDB an ELF file at runtime describing the symbols we have +// generated. To avoid implementing the ELF format, we generate an ELF file +// at compile-time and compile it in as a character string. We can replace +// a few key constants (address of JIT-ted function and its size) by looking +// for a few magic numbers and doing a dumb string replacement. +// +// Unfortunately this approach is showing its limits; we can only define one +// symbol, and this approach only works with GDB. The .so approach above is +// more reliable. + +#ifndef __APPLE__ +const unsigned char upb_jit_debug_elf_file[] = { +#include "upb/pb/jit_debug_elf_file.h" +}; + +typedef enum { + GDB_JIT_NOACTION = 0, + GDB_JIT_REGISTER, + GDB_JIT_UNREGISTER +} jit_actions_t; + +typedef struct gdb_jit_entry { + struct gdb_jit_entry *next_entry; + struct gdb_jit_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +} gdb_jit_entry; + +typedef struct { + uint32_t version; + uint32_t action_flag; + gdb_jit_entry *relevant_entry; + gdb_jit_entry *first_entry; +} gdb_jit_descriptor; + +gdb_jit_descriptor __jit_debug_descriptor = {1, GDB_JIT_NOACTION, NULL, NULL}; + +void __attribute__((noinline)) __jit_debug_register_code() { + __asm__ __volatile__(""); +} + +static void upb_reg_jit_gdb(jitcompiler *jc) { + // Create debug info. + size_t elf_len = sizeof(upb_jit_debug_elf_file); + jc->plan->debug_info = malloc(elf_len); + memcpy(jc->plan->debug_info, upb_jit_debug_elf_file, elf_len); + uint64_t *p = (void *)jc->plan->debug_info; + for (; (void *)(p + 1) <= (void *)jc->plan->debug_info + elf_len; ++p) { + if (*p == 0x12345678) { + *p = (uintptr_t)jc->plan->jit_code; + } + if (*p == 0x321) { + *p = jc->plan->jit_size; + } + } + + // Register the JIT-ted code with GDB. + gdb_jit_entry *e = malloc(sizeof(gdb_jit_entry)); + e->next_entry = __jit_debug_descriptor.first_entry; + e->prev_entry = NULL; + if (e->next_entry) e->next_entry->prev_entry = e; + e->symfile_addr = jc->plan->debug_info; + e->symfile_size = elf_len; + __jit_debug_descriptor.first_entry = e; + __jit_debug_descriptor.relevant_entry = e; + __jit_debug_descriptor.action_flag = GDB_JIT_REGISTER; + __jit_debug_register_code(); +} + +#else + +static void upb_reg_jit_gdb(jitcompiler *jc) { (void)jc; } + +#endif diff --git a/upb/pb/compile_decoder_x64.dasc b/upb/pb/compile_decoder_x64.dasc new file mode 100644 index 0000000..0bddade --- /dev/null +++ b/upb/pb/compile_decoder_x64.dasc @@ -0,0 +1,1087 @@ +|// +|// upb - a minimalist implementation of protocol buffers. +|// +|// Copyright (c) 2011-2013 Google Inc. See LICENSE for details. +|// Author: Josh Haberman <jhaberman@gmail.com> +|// +|// JIT compiler for upb_pbdecoder on x86-64. Generates machine code from the +|// bytecode generated in compile_decoder.c, but unlike the interpreter we bind +|// to a specific set of handlers for greater efficiency. +| +|.arch x64 +|.actionlist upb_jit_actionlist +|.globals UPB_JIT_GLOBAL_ +|.globalnames upb_jit_globalnames +| +|// Calling conventions. Note -- this will need to be changed for +|// Windows, which uses a different calling convention! +|.define ARG1_64, rdi +|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi. +|.define ARG2_32, esi +|.define ARG2_64, rsi +|.define ARG3_8, dl +|.define ARG3_32, edx +|.define ARG3_64, rdx +|.define ARG4_64, rcx +|.define XMMARG1, xmm0 +| +|// Register allocation / type map. +|// ALL of the code in this file uses these register allocations. +|// When we "call" within this file, we do not use regular calling +|// conventions, but of course when calling to user callbacks we must. +|.define PTR, rbx // DECODER->ptr (unsynced) +|.define DATAEND, r12 // DECODER->data_end (unsynced) +|.define CLOSURE, r13 // FRAME->closure (unsynced) +|.type FRAME, upb_pbdecoder_frame, r14 // DECODER->top (unsynced) +|.type DECODER, upb_pbdecoder, r15 // DECODER (immutable) +|.define DELIMEND, rbp +| +| // Spills unsynced registers back to memory. +|.macro commit_regs +| mov DECODER->top, FRAME +| mov DECODER->ptr, PTR +| mov DECODER->data_end, DATAEND +| sub DELIMEND, DECODER->buf +| add DELIMEND, DECODER->bufstart_ofs +| mov FRAME->end_ofs, DELIMEND +| mov FRAME->u.closure, CLOSURE +|.endmacro +| +| // Loads unsynced registers from memory back into registers. +|.macro load_regs +| mov FRAME, DECODER->top +| mov PTR, DECODER->ptr +| mov DATAEND, DECODER->data_end +| mov CLOSURE, FRAME->u.closure +| mov DELIMEND, FRAME->end_ofs +| sub DELIMEND, DECODER->bufstart_ofs +| add DELIMEND, DECODER->buf +|.endmacro +| +| // OPT: use "call rel32" where possible. +|.macro callp, addr +|| { +|| //int64_t ofs = (int64_t)addr - (int64_t)upb_status_init; +|| //if (ofs > (1 << 30) || ofs < -(1 << 30)) { +| mov64 rax, (uintptr_t)addr +| call rax +|| //} else { +| // call &addr +|| //} +|| } +|.endmacro +| +|.macro ld64, val +|| { +|| uintptr_t v = (uintptr_t)val; +|| if (v > 0xffffffff) { +| mov64 ARG2_64, v +|| } else if (v) { +| mov ARG2_32, v +|| } else { +| xor ARG2_32, ARG2_32 +|| } +|| } +|.endmacro +| +|.macro load_handler_data, h, arg +| ld64 upb_handlers_gethandlerdata(h, arg) +|.endmacro +| +|.macro chkeob, bytes, target +|| if (bytes == 1) { +| cmp PTR, DATAEND +| je target +|| } else { +| mov rcx, DATAEND +| sub rcx, PTR +| cmp rcx, bytes +| jb target +|| } +|.endmacro +| +|.macro chkneob, bytes, target +|| if (bytes == 1) { +| cmp PTR, DATAEND +| jne target +|| } else { +| mov rcx, DATAEND +| sub rcx, PTR +| cmp rcx, bytes +| jae target +|| } +|.endmacro + +|.macro sethas, reg, hasbit +|| if (hasbit >= 0) { +| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) +|| } +|.endmacro +| +| // Decodes 32-bit varint into rdx, inlining 1 byte. +|.macro dv32 +| chkeob 1, >7 +| movzx edx, byte [PTR] +| test dl, dl +| jns >8 +|7: +| call ->decodev32_fallback +|8: +| add PTR, 1 +|.endmacro + +static void asmlabel(jitcompiler *jc, const char *fmt, ...) { + int ofs = jc->dynasm->section->ofs; + assert(ofs != jc->lastlabelofs); + jc->lastlabelofs = ofs; + va_list args; + va_start(args, fmt); + + // Run once to get the length of the string. + va_list args_copy; + va_copy(args_copy, args); + int len = vsnprintf(NULL, 0, fmt, args_copy); + va_end(args_copy); + + char *str = malloc(len + 1); // + 1 for NULL terminator. + if (!str) exit(1); + int written = vsnprintf(str, len, fmt, args); + va_end(args); + UPB_ASSERT_VAR(written, written == len); + + uint32_t label = jc->pclabel_count++; + dasm_growpc(jc, jc->pclabel_count); + |=>label: + upb_inttable_insert(&jc->asmlabels, label, upb_value_ptr(str)); +} + +// Emit static assembly routines; code that does not vary based on the message +// schema. Since it's not input-dependent, we only need one single copy of it. +// For the moment we generate a single copy per generated handlers. Eventually +// we should generate this code at compile time and link it into the binary so +// we have one copy total. To do that we'll want to be sure that it is within +// 2GB of our JIT code, so that branches between the two are near (rel32). +// +// We'd put this assembly in a .s file directly, but DynASM's ability to +// calculate structure offsets automatically is too useful to pass up (it's way +// more convenient to write DECODER->sink than [rbx + 0x96], especially since +// the latter would have to be changed whenever the structure is updated). +static void emit_static_asm(jitcompiler *jc) { + | // Trampolines for entering/exiting the JIT. These are a bit tricky to + | // support full resuming; when we suspend we copy the JIT's portion of + | // the call stack into the upb_pbdecoder and restore it when we resume. + asmlabel(jc, "enterjit"); + |->enterjit: + |1: + | push rbp + if (jc->usefp) { + | mov rbp, rsp + } + | push r15 + | push r14 + | push r13 + | push r12 + | push rbx + | + | // Align stack. + | // Since the JIT can call other functions (the JIT'ted code is not a leaf + | // function) we must respect alignment rules. All x86-64 systems require + | // 16-byte stack alignment. + | sub rsp, 8 + | + | mov DECODER, rdi + | callp upb_pbdecoder_resume // Same args as us; reuse regs. + | mov DECODER->saved_rsp, rsp + | load_regs + | + | // Test whether we have a saved stack to resume. + | mov ARG3_64, DECODER->call_len + | test ARG3_64, ARG3_64 + | jnz >1 + | + | call =>pclabel(jc, jc->plan->topmethod) + | + | mov rax, DECODER->size_param + | mov qword DECODER->call_len, 0 + | add rsp, 8 // Counter previous alignment. + | pop rbx + | pop r12 + | pop r13 + | pop r14 + | pop r15 + | pop rbp + | ret + | + |1: + | // Resume decoder. + | lea ARG2_64, DECODER->callstack + | sub rsp, ARG3_64 + | mov ARG1_64, rsp + | callp memcpy // Restore stack. + | ret // Return to resumed function (not ->enterjit caller). + | + | // Other code can call this to suspend the JIT. + | // To the calling code, it will appear that the function returns when + | // the JIT resumes, and more buffer space will be available. + | // Args: eax=the value that decode() should return. + asmlabel(jc, "exitjit"); + |->exitjit: + | // Save the stack into DECODER->callstack. + | lea ARG1_64, DECODER->callstack + | mov ARG2_64, rsp + | mov ARG3_64, DECODER->saved_rsp + | sub ARG3_64, rsp + | mov DECODER->call_len, ARG3_64 // Preserve len for next resume. + | mov ebx, eax // Preserve return value across memcpy. + | callp memcpy // Copy stack into decoder. + | mov eax, ebx // This will be our return value. + | + | // Must NOT do this before the memcpy(), otherwise memcpy() will + | // clobber the stack we are trying to save! + | mov rsp, DECODER->saved_rsp + | add rsp, 8 // Counter previous alignment. + | pop rbx + | pop r12 + | pop r13 + | pop r14 + | pop r15 + | pop rbp + | ret + | + | // Like suspend() in the C decoder, except that the function appears + | // (from the caller's perspective) not to return until the decoder is + | // resumed. + asmlabel(jc, "suspend"); + |->suspend: + | cmp DECODER->ptr, PTR + | je >1 + | mov DECODER->checkpoint, PTR + |1: + | commit_regs + | mov rdi, DECODER + | callp upb_pbdecoder_suspend + | jmp ->exitjit + | + asmlabel(jc, "pushlendelim"); + |->pushlendelim: + |1: + | mov FRAME->u.closure, CLOSURE + | mov DECODER->checkpoint, PTR + | dv32 + | mov rcx, DELIMEND + | sub rcx, PTR + | sub rcx, rdx + | jb ->err // Len is greater than enclosing message. + | mov FRAME->end_ofs, rcx + | add FRAME, sizeof(upb_pbdecoder_frame) + | mov DELIMEND, PTR + | add DELIMEND, rdx + | cmp FRAME, DECODER->limit + | je >3 // Stack overflow + | test rcx, rcx + | jz >2 + | mov DATAEND, DECODER->end + | cmp PTR, DELIMEND + | ja >2 + | cmp DELIMEND, DATAEND + | ja >2 + | mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND + |2: + | ret + |3: + | // Error -- call seterr. + | mov PTR, DECODER->checkpoint // Rollback to before the delim len. + | // Prepare seterr args. + | mov ARG1_64, DECODER + | ld64 kPbDecoderStackOverflow + | callp upb_pbdecoder_seterr + | call ->suspend + | jmp <1 + | + | // For getting a value that spans a buffer seam. Falls back to C. + | // Args: rdi=C decoding function (prototype: int f(upb_pbdecoder*, void*)) + asmlabel(jc, "getvalue_slow"); + |->getvalue_slow: + | sub rsp, 16 // Stack is [8-byte value, 8-byte func pointer] + | mov [rsp + 8], rdi // Need to preserve fptr across suspends. + |1: + | mov qword [rsp], 0 // For parsing routines that only parse 32 bits. + | mov ARG1_64, DECODER + | mov ARG2_64, rsp + | mov DECODER->checkpoint, PTR + | commit_regs + | call aword [rsp + 8] + | load_regs + | test eax, eax + | jns >2 + | // Success; return parsed data (in rdx AND xmm0). + | mov rdx, [rsp] + | movsd xmm0, qword [rsp] + | add rsp, 16 + | ret + |2: + | call ->exitjit // Return eax from decode function. + | jmp <1 + | + asmlabel(jc, "parse_unknown"); + | // Args: edx=fieldnum, cl=wire type + |->parse_unknown: + | // OPT: handle directly instead of kicking to C. + | // Check for ENDGROUP. + | mov ARG1_64, DECODER + | mov ARG2_32, edx + | movzx ARG3_32, cl + | commit_regs + | callp upb_pbdecoder_skipunknown + | load_regs + | cmp eax, DECODE_ENDGROUP + | jne >1 + | ret // Return eax=DECODE_ENDGROUP, not zero + |1: + | cmp eax, DECODE_OK + | je >1 + | call ->exitjit // Return eax from decode function. + |1: + | xor eax, eax + | ret + | + | // Fallback functions for parsing single values. These are used when the + | // buffer doesn't contain enough remaining data for the fast path. Each + | // primitive type (v32, v64, f32, f64) has two functions: decode & skip. + | // Decode functions return their value in rsi/esi. + | // + | // These functions leave PTR = value_end - fast_path_bytes, so that we can + | // re-join the fast path which will add fast_path_bytes after the callback + | // completes. We also set DECODER->ptr to this value which is a signal to + | // ->suspend that DECODER->checkpoint is up to date. + asmlabel(jc, "skip_decode_f32_fallback"); + |->skipf32_fallback: + |->decodef32_fallback: + | mov64 rdi, (uintptr_t)upb_pbdecoder_decode_f32 + | call ->getvalue_slow + | sub PTR, 4 + | mov DECODER->ptr, PTR + | ret + | + asmlabel(jc, "skip_decode_f64_fallback"); + |->skipf64_fallback: + |->decodef64_fallback: + | mov64 rdi, (uintptr_t)upb_pbdecoder_decode_f64 + | call ->getvalue_slow + | sub PTR, 8 + | mov DECODER->ptr, PTR + | ret + | + | // Called for varint >= 1 byte. + asmlabel(jc, "skip_decode_v32_fallback"); + |->skipv32_fallback: + |->skipv64_fallback: + | chkeob 16, >1 + | // With at least 16 bytes left, we can do a branch-less SSE version. + | movdqu xmm0, [PTR] + | pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0. + | not eax + | bsf eax, eax + | cmp al, 10 + | jae ->decode_varint_slow // Error (>10 byte varint). + | add PTR, rax // bsf result is 0-based, so PTR=end-1, as desired. + | ret + | + |1: + | // With fewer than 16 bytes, we have to read byte by byte. + | lea rcx, [PTR + 10] + | mov rax, PTR // Preserve PTR in case of fallback to slow path. + | cmp rcx, DATAEND + | cmova rcx, DATAEND // rax = MIN(DATAEND, PTR + 10) + |2: + | add rax, 1 + | cmp rax, rcx + | je ->decode_varint_slow + | test byte [rax], 0x80 + | jnz <2 + |3: + | mov PTR, rax // PTR = varint_end - 1, as desired + | ret + | + | // Returns tag in edx + asmlabel(jc, "decode_unknown_tag_fallback"); + |->decode_unknown_tag_fallback: + | sub rsp, 16 + |1: + | cmp PTR, DELIMEND + | jne >2 + | add rsp, 16 + | xor eax, eax + | ret + |2: + | // OPT: Have a medium-fast path before falling back to _slow. + | mov ARG1_64, DECODER + | mov ARG2_64, rsp + | commit_regs + | callp upb_pbdecoder_decode_varint_slow + | load_regs + | cmp eax, 0 + | jge >3 + | mov edx, [rsp] // Success; return parsed data. + | add rsp, 16 + | ret + |3: + | call ->exitjit // Return eax from decode function. + | jmp <1 + | + | // Called for varint >= 1 byte. + asmlabel(jc, "decode_v32_v64_fallback"); + |->decodev32_fallback: + |->decodev64_fallback: + | chkeob 10, ->decode_varint_slow + | // OPT: do something faster than just calling the C version. + | mov rdi, PTR + | callp upb_vdecode_fast + | test rax, rax + | je ->decode_varint_slow // Unterminated varint. + | mov PTR, rax + | sub PTR, 1 + | mov DECODER->ptr, PTR + | ret + | + asmlabel(jc, "decode_varint_slow"); + |->decode_varint_slow: + | // Slow path: end of buffer or error (varint length >= 10). + | mov64 rdi, (uintptr_t)upb_pbdecoder_decode_varint_slow + | call ->getvalue_slow + | sub PTR, 1 + | mov DECODER->ptr, PTR + | ret + | + | // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH}) + asmlabel(jc, "checktag_fallback"); + |->checktag_fallback: + | sub rsp, 8 + | mov [rsp], rsi // Preserve expected tag. + |1: + | mov ARG1_64, DECODER + | commit_regs + | mov DECODER->checkpoint, PTR + | callp upb_pbdecoder_checktag_slow + | load_regs + | cmp eax, 0 + | jge >2 + | add rsp, 8 + | ret + |2: + | call ->exitjit + | mov rsi, [rsp] + | cmp PTR, DELIMEND + | jne <1 + | mov eax, DECODE_EOF + | add rsp, 8 + | ret + | + | // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found). + | // Preserves: rcx, rdx + | // OPT: Could write this in assembly if it's a hotspot. + asmlabel(jc, "hashlookup"); + |->hashlookup: + | push rcx + | push rdx + | sub rsp, 16 + | mov rdi, rsi + | mov rsi, rdx + | mov rdx, rsp + | callp upb_inttable_lookup + | add rsp, 16 + | pop rdx + | pop rcx + | test al, al + | jz >2 // Unknown field. + | mov rax, [rsp-32] // Value from table. + | ret + |2: + | xor rax, rax + | not rax + | ret +} + +static void jitprimitive(jitcompiler *jc, opcode op, + const upb_handlers *h, upb_selector_t sel) { + typedef enum { V32, V64, F32, F64, X } valtype_t; + static valtype_t types[] = { + X, F64, F32, V64, V64, V32, F64, F32, V64, X, X, X, X, V32, V32, F32, F64, + V32, V64 }; + static char fastpath_bytes[] = { 1, 1, 4, 8 }; + const valtype_t type = types[op]; + const int fastbytes = fastpath_bytes[type]; + upb_func *handler = upb_handlers_gethandler(h, sel); + + if (handler) { + |1: + | chkneob fastbytes, >3 + |2: + switch (type) { + case V32: + | call ->decodev32_fallback + break; + case V64: + | call ->decodev64_fallback + break; + case F32: + | call ->decodef32_fallback + break; + case F64: + | call ->decodef64_fallback + break; + case X: break; + } + | jmp >4 + + // Fast path decode; for when check_bytes bytes are available. + |3: + switch (op) { + case OP_PARSE_SFIXED32: + case OP_PARSE_FIXED32: + | mov edx, dword [PTR] + break; + case OP_PARSE_SFIXED64: + case OP_PARSE_FIXED64: + | mov rdx, qword [PTR] + break; + case OP_PARSE_FLOAT: + | movss xmm0, dword [PTR] + break; + case OP_PARSE_DOUBLE: + | movsd xmm0, qword [PTR] + break; + default: + // Inline one byte of varint decoding. + | movzx edx, byte [PTR] + | test dl, dl + | js <2 // Fallback to slow path for >1 byte varint. + break; + } + + // Second-stage decode; used for both fast and slow paths + // (only needed for a few types). + |4: + switch (op) { + case OP_PARSE_SINT32: + // 32-bit zig-zag decode. + | mov eax, edx + | shr edx, 1 + | and eax, 1 + | neg eax + | xor edx, eax + break; + case OP_PARSE_SINT64: + // 64-bit zig-zag decode. + | mov rax, rdx + | shr rdx, 1 + | and rax, 1 + | neg rax + | xor rdx, rax + break; + case OP_PARSE_BOOL: + | test rdx, rdx + | setne dl + break; + default: break; + } + + // Call callback (or specialize if we can). + upb_fieldtype_t type; + const upb_shim_data *data = upb_shim_getdata(h, sel, &type); + if (data) { + switch (type) { + case UPB_TYPE_INT64: + case UPB_TYPE_UINT64: + | mov [CLOSURE + data->offset], rdx + break; + case UPB_TYPE_INT32: + case UPB_TYPE_UINT32: + case UPB_TYPE_ENUM: + | mov [CLOSURE + data->offset], edx + break; + case UPB_TYPE_DOUBLE: + | movsd qword [CLOSURE + data->offset], XMMARG1 + break; + case UPB_TYPE_FLOAT: + | movss dword [CLOSURE + data->offset], XMMARG1 + break; + case UPB_TYPE_BOOL: + | mov [CLOSURE + data->offset], dl + break; + case UPB_TYPE_STRING: + case UPB_TYPE_BYTES: + case UPB_TYPE_MESSAGE: + assert(false); break; + } + | sethas CLOSURE, data->hasbit + } else if (handler) { + | mov ARG1_64, CLOSURE + | load_handler_data h, sel + | callp handler + if (jc->chkret) { + | test al, al + | jz >5 + | call ->suspend + | jmp <1 + |5: + } + } + + // We do this last so that the checkpoint is not advanced past the user's + // data until the callback has returned success. + | add PTR, fastbytes + } else { + // No handler registered for this value, just skip it. + | chkneob fastbytes, >3 + |2: + switch (type) { + case V32: + | call ->skipv32_fallback + break; + case V64: + | call ->skipv64_fallback + break; + case F32: + | call ->skipf32_fallback + break; + case F64: + | call ->skipf64_fallback + break; + case X: break; + } + + // Fast-path skip. + |3: + if (type == V32 || type == V64) { + | test byte [PTR], 0x80 + | jnz <2 + } + | add PTR, fastbytes + } +} + +static void jitdispatch(jitcompiler *jc, + const upb_pbdecodermethod *method) { + // Lots of room for tweaking/optimization here. + + const upb_inttable *dispatch = &method->dispatch; + bool has_hash_entries = (dispatch->t.count > 0); + + // Whether any of the fields for this message can have two wire types which + // are both valid (packed & non-packed). + // + // OPT: populate this more precisely; not all messages with hash entries have + // this characteristic. + bool has_multi_wiretype = has_hash_entries; + + |=>define_pclabel(jc, &method->dispatch): + |1: + // Decode the field tag. + // OPT: inline two bytes of varint decoding for big messages. + | mov aword DECODER->checkpoint, PTR + | chkeob 1, >6 + | movzx edx, byte [PTR] + | test dl, dl + | jns >7 + |6: + | call ->decode_unknown_tag_fallback + | test eax, eax // Hit DELIMEND? + | jnz >8 + | ret + |7: + | add PTR, 1 + |8: + | mov ecx, edx + | shr edx, 3 + | and cl, 7 + + // See comment attached to upb_pbdecodermethod.dispatch for layout of the + // dispatch table. + |2: + | cmp edx, dispatch->array_size + if (has_hash_entries) { + | jae >7 + } else { + | jae >5 + } + | // OPT: Compact the lookup arr into 32-bit entries. + if ((uintptr_t)dispatch->array > 0x7fffffff) { + | mov64 rax, (uintptr_t)dispatch->array + | mov rax, qword [rax + rdx * 8] + } else { + | mov rax, qword [rdx * 8 + dispatch->array] + } + |3: + | // We take advantage of the fact that non-present entries are stored + | // as -1, which will result in wire types that will never match. + | cmp al, cl + if (has_multi_wiretype) { + | jne >6 + } else { + | jne >5 + } + | shr rax, 16 + | lea rdx, [>4] + |=>define_pclabel(jc, dispatch->array): + |4: + | add rax, rdx + | ret + | + |5: + | // Field isn't in our table. + | call ->parse_unknown + | test eax, eax // ENDGROUP? + | jz <1 + | lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG. + | ret + + if (has_multi_wiretype) { + |6: + | // Primary wire type didn't match, check secondary wire type. + | cmp ah, cl + | jne <5 + | // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER. + | add rdx, UPB_MAX_FIELDNUMBER + | // This key will never be in the array part, so do a hash lookup. + assert(has_hash_entries); + | ld64 dispatch + | jmp ->hashlookup // Tail call. + } + + if (has_hash_entries) { + |7: + | // Hash table lookup. + | ld64 dispatch + | call ->hashlookup + | jmp <3 + } +} + +static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, + const upb_pbdecodermethod *method) { + // Internally we parse unknown fields; if this runs us into DELIMEND we jump + // to the corresponding DELIMEND target (either msg end or repeated field + // end), which we find from the OP_CHECKDELIM which must have necessarily + // preceded us. + uint32_t last_instruction = *(jc->pc - 2); + int last_arg = (int32_t)last_instruction >> 8; + assert((last_instruction & 0xff) == OP_CHECKDELIM); + uint32_t *delimend = (jc->pc - 1) + last_arg; + const size_t ptr_words = sizeof(void*) / sizeof(uint32_t); + + if (getop(*(jc->pc - 1)) == OP_TAGN) { + jc->pc += ptr_words; + } + + | chkneob n, >1 + + | // OPT: this is way too much fallback code to put here. + | // Reduce and/or move to a separate section to make better icache usage. + | ld64 tag + | call ->checktag_fallback + | cmp eax, DECODE_MISMATCH + | je >3 + | cmp eax, DECODE_EOF + | je =>pclabel(jc, delimend) + | jmp >5 + + |1: + switch (n) { + case 1: + | cmp byte [PTR], tag + break; + case 2: + | cmp word [PTR], tag + break; + case 3: + | // OPT: Slightly more efficient code, but depends on an extra byte. + | // mov eax, dword [PTR] + | // shl eax, 8 + | // cmp eax, tag << 8 + | cmp word [PTR], (tag & 0xffff) + | jne >2 + | cmp byte [PTR + 2], (tag >> 16) + |2: + break; + case 4: + | cmp dword [PTR], tag + break; + case 5: + | cmp dword [PTR], (tag & 0xffffffff) + | jne >3 + | cmp byte [PTR + 4], (tag >> 32) + } + | je >4 + |3: + if (ofs == 0) { + | call =>pclabel(jc, &method->dispatch) + | test rax, rax + | jz =>pclabel(jc, delimend) + | jmp rax + } else { + | jmp =>pclabel(jc, jc->pc + ofs) + } + |4: + | add PTR, n + |5: +} + +// Emit message-specific assembly. Overall code layout is: +// +---------------------------------------------------------------------------+ +// | Message A | +// | 1. function prologue (startmsg), jmps to OP_CHECKDELIM_RET before first | +// | OP_TAG* in 4. | +// | 2. function epilogue (endmsg), returns from function. | +// | 3. dispatch function (returns fptr to 4) | +// | - loops internally to skip unknown fields | +// | - after each unknown field does OP_CHECKDELIM_RET (returns 2) | +// | - also returns 2 for END_GROUP. +// | 4. code for each op: | +// | - OP_TAG* on mismatch calls 3 to get addr, then jumps to 4 (or 2 on EOM).| +// | - OP_CHECKDELIM_RET jumps to 2 | +// +---------------------------------------------------------------------------+ +// | Message B | +// | 1. ... | +// | ... | +// +---------------------------------------------------------------------------+ +static void jitbytecode(jitcompiler *jc) { + upb_pbdecodermethod *method = NULL; + const upb_handlers *h = NULL; + for (jc->pc = jc->plan->code; jc->pc < jc->plan->code_end; ) { + int32_t instr = *jc->pc; + opcode op = instr & 0xff; + uint32_t arg = instr >> 8; + int32_t longofs = arg; + + if (op != OP_STARTMSG && op != OP_SETDISPATCH) { + asmlabel(jc, "0x%lx.%s", jc->pc - jc->plan->code, + upb_pbdecoder_getopname(op)); + } + // TODO: optimize this to only define pclabels that are actually used. + |=>define_pclabel(jc, jc->pc): + jc->pc++; + + switch (op) { + case OP_STARTMSG: { + // This opcode serves as a function prolouge also. + const char *msgname = upb_msgdef_fullname(method->msg); + asmlabel(jc, "parse.%s", msgname); + |=>define_pclabel(jc, method): + if (jc->usefp) { + | push rbp + | mov rbp, rsp + } else { + | sub rsp, 8 + } + upb_func *startmsg = upb_handlers_gethandler(h, UPB_STARTMSG_SELECTOR); + if (startmsg) { + // bool startmsg(void *closure, const void *hd) + |1: + | mov ARG1_64, CLOSURE + | load_handler_data h, UPB_STARTMSG_SELECTOR + | callp startmsg + if (jc->chkret) { + | test al, al + | jnz <2 + | call ->suspend + |2: + } + } + break; + } + case OP_ENDMSG: { + // This opcode serves as a function epiloue also. + upb_func *endmsg = upb_handlers_gethandler(h, UPB_ENDMSG_SELECTOR); + |9: + if (endmsg) { + // bool endmsg(void *closure, const void *hd, upb_status *status) + | mov ARG1_64, CLOSURE + | load_handler_data h, UPB_ENDMSG_SELECTOR + | mov ARG3_64, DECODER->status + | callp endmsg + } + if (jc->usefp) { + | pop rbp + } else { + | add rsp, 8 + } + | ret + break; + } + case OP_SETDISPATCH: { + upb_inttable *dispatch; + memcpy(&dispatch, jc->pc, sizeof(void*)); + jc->pc += sizeof(void*) / sizeof(uint32_t); + // The OP_SETDISPATCH bytecode contains a pointer that is + // &method->dispatch; we want to go backwards and recover method. + method = + (void*)((char*)dispatch - offsetof(upb_pbdecodermethod, dispatch)); + h = method->dest_handlers; + assert(h); // We only support statically-bound handlers for now. + const char *msgname = upb_msgdef_fullname(method->msg); + asmlabel(jc, "dispatch.%s", msgname); + jitdispatch(jc, method); + break; + } + case OP_PARSE_DOUBLE: + case OP_PARSE_FLOAT: + case OP_PARSE_INT64: + case OP_PARSE_UINT64: + case OP_PARSE_INT32: + case OP_PARSE_FIXED64: + case OP_PARSE_FIXED32: + case OP_PARSE_BOOL: + case OP_PARSE_UINT32: + case OP_PARSE_SFIXED32: + case OP_PARSE_SFIXED64: + case OP_PARSE_SINT32: + case OP_PARSE_SINT64: + jitprimitive(jc, op, h, arg); + break; + case OP_STARTSEQ: + case OP_STARTSUBMSG: + case OP_STARTSTR: { + upb_func *start = upb_handlers_gethandler(h, arg); + if (start) { + // void *startseq(void *closure, const void *hd) + // void *startsubmsg(void *closure, const void *hd) + // void *startstr(void *closure, const void *hd, size_t size_hint) + |1: + | mov ARG1_64, CLOSURE + | load_handler_data h, arg + if (op == OP_STARTSTR) { + | mov ARG3_64, DELIMEND + | sub ARG3_64, PTR + } + | callp start + if (jc->chkret) { + | test rax, rax + | jnz >2 + | call ->suspend + | jmp <1 + |2: + } + | mov CLOSURE, rax + } else { + // TODO: nop is only required because of asmlabel(). + | nop + } + break; + } + case OP_ENDSEQ: + case OP_ENDSUBMSG: + case OP_ENDSTR: { + upb_func *end = upb_handlers_gethandler(h, arg); + if (end) { + // bool endseq(void *closure, const void *hd) + // bool endsubmsg(void *closure, const void *hd) + // bool endstr(void *closure, const void *hd) + |1: + | mov ARG1_64, CLOSURE + | load_handler_data h, arg + | callp end + if (jc->chkret) { + | test al, al + | jnz >2 + | call ->suspend + | jmp <1 + |2: + } + } else { + // TODO: nop is only required because of asmlabel(). + | nop + } + break; + } + case OP_STRING: { + upb_func *str = upb_handlers_gethandler(h, arg); + | cmp PTR, DELIMEND + | je >4 + |1: + | cmp PTR, DATAEND + | jne >2 + | call ->suspend + | jmp <1 + |2: + if (str) { + // size_t str(void *closure, const void *hd, const char *str, size_t n) + | mov ARG1_64, CLOSURE + | load_handler_data h, arg + | mov ARG3_64, PTR + | mov ARG4_64, DATAEND + | sub ARG4_64, PTR + | callp str + | add PTR, rax + if (jc->chkret) { + | cmp PTR, DATAEND + | je >3 + | call ->strret_fallback + |3: + } + } else { + | mov PTR, DATAEND + } + | cmp PTR, DELIMEND + | jne <1 + |4: + break; + } + case OP_PUSHTAGDELIM: + | mov FRAME->u.closure, CLOSURE + | add FRAME, sizeof(upb_pbdecoder_frame) + | cmp FRAME, DECODER->limit + | je ->err + break; + case OP_PUSHLENDELIM: + | call ->pushlendelim + break; + case OP_POP: + | sub FRAME, sizeof(upb_pbdecoder_frame) + | mov CLOSURE, FRAME->u.closure + break; + case OP_SETDELIM: + // OPT: experiment with testing vs old offset to optimize away. + | mov DATAEND, DECODER->end + | add DELIMEND, FRAME->end_ofs + | jc >1 + | cmp DELIMEND, DATAEND + | ja >1 // OPT: try cmov. + | mov DATAEND, DELIMEND + |1: + break; + case OP_SETGROUPNUM: + | mov dword FRAME->groupnum, arg + break; + case OP_SETBIGGROUPNUM: + | mov dword FRAME->groupnum, *jc->pc++ + break; + case OP_CHECKDELIM: + | cmp DELIMEND, PTR + | je =>pclabel(jc, jc->pc + longofs) + break; + case OP_CALL: + | call =>pclabel(jc, jc->pc + longofs + 3) + break; + case OP_BRANCH: + | jmp =>pclabel(jc, jc->pc + longofs); + break; + case OP_TAG1: + jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method); + break; + case OP_TAG2: + jittag(jc, (arg >> 8) & 0xffff, 2, (int8_t)arg, method); + break; + case OP_TAGN: { + uint64_t tag; + memcpy(&tag, jc->pc, 8); + jittag(jc, tag, arg >> 8, (int8_t)arg, method); + break; + } + case OP_HALT: + assert(false); + } + } + asmlabel(jc, "eof"); + | nop +} diff --git a/upb/pb/decoder.c b/upb/pb/decoder.c index 18bb430..0cfb12e 100644 --- a/upb/pb/decoder.c +++ b/upb/pb/decoder.c @@ -1,208 +1,63 @@ /* * upb - a minimalist implementation of protocol buffers. * - * Copyright (c) 2008-2011 Google Inc. See LICENSE for details. + * Copyright (c) 2008-2013 Google Inc. See LICENSE for details. * Author: Josh Haberman <jhaberman@gmail.com> */ #include <inttypes.h> #include <setjmp.h> +#include <stdarg.h> #include <stddef.h> #include <stdlib.h> #include "upb/bytestream.h" -#include "upb/pb/decoder.h" -#include "upb/pb/varint.h" +#include "upb/pb/decoder.int.h" +#include "upb/pb/varint.int.h" -#define UPB_NONDELIMITED (0xffffffffffffffffULL) - -/* upb_pbdecoder ****************************************************************/ - -struct dasm_State; - -typedef struct { - const upb_fielddef *f; - uint64_t end_ofs; - uint32_t group_fieldnum; // UINT32_MAX for non-groups. - bool is_sequence; // frame represents seq or submsg/str? (f might be both). - bool is_packed; // true for packed primitive sequences. -} frame; - -struct upb_pbdecoder { - // Where we push parsed data (not owned). - upb_sink *sink; - - // Current input buffer and its stream offset. - const char *buf, *ptr, *end, *checkpoint; - uint64_t bufstart_ofs; - - // Buffer for residual bytes not parsed from the previous buffer. - char residual[16]; - char *residual_end; - - // Stores the user buffer passed to our decode function. - const char *buf_param; - size_t size_param; - - // Equal to size_param while we are in the residual buf, 0 otherwise. - size_t userbuf_remaining; - - // Used to temporarily store the return value before calling longjmp(). - size_t ret; - - // End of the delimited region, relative to ptr, or NULL if not in this buf. - const char *delim_end; - -#ifdef UPB_USE_JIT_X64 - // For JIT, which doesn't do bounds checks in the middle of parsing a field. - const char *jit_end, *effective_end; // == MIN(jit_end, delim_end) - - // Used momentarily by the generated code to store a value while a user - // function is called. - uint32_t tmp_len; - - const void *saved_rbp; +#ifdef UPB_DUMP_BYTECODE +#include <stdio.h> #endif - // Our internal stack. - frame *top, *limit; - frame stack[UPB_MAX_NESTING]; - - // For exiting the decoder on error. - jmp_buf exitjmp; -}; - -typedef struct { - // The top-level handlers that this plan calls into. We own a ref. - const upb_handlers *dest_handlers; - -#ifdef UPB_USE_JIT_X64 - // JIT-generated machine code (else NULL). - char *jit_code; - size_t jit_size; - char *debug_info; - - // For storing upb_jitmsginfo, which contains per-msg runtime data needed - // by the JIT. - // Maps upb_handlers* -> upb_jitmsginfo. - upb_inttable msginfo; - - // The following members are used only while the JIT is being built. - - // This pointer is allocated by dasm_init() and freed by dasm_free(). - struct dasm_State *dynasm; - - // For storing pclabel bases while we are building the JIT. - // Maps (upb_handlers* or upb_fielddef*) -> int32 pclabel_base - upb_inttable pclabels; - - // For marking labels that should go into the generated code. - // Maps pclabel -> owned char* label. - upb_inttable asmlabels; - - // This is not the same as len(pclabels) because the table only contains base - // offsets for each def, but each def can have many pclabels. - uint32_t pclabel_count; -#endif -} decoderplan; - -typedef struct { - uint8_t native_wire_type; - bool is_numeric; -} upb_decoder_typeinfo; - -static const upb_decoder_typeinfo upb_decoder_types[] = { - {UPB_WIRE_TYPE_END_GROUP, false}, // ENDGROUP - {UPB_WIRE_TYPE_64BIT, true}, // DOUBLE - {UPB_WIRE_TYPE_32BIT, true}, // FLOAT - {UPB_WIRE_TYPE_VARINT, true}, // INT64 - {UPB_WIRE_TYPE_VARINT, true}, // UINT64 - {UPB_WIRE_TYPE_VARINT, true}, // INT32 - {UPB_WIRE_TYPE_64BIT, true}, // FIXED64 - {UPB_WIRE_TYPE_32BIT, true}, // FIXED32 - {UPB_WIRE_TYPE_VARINT, true}, // BOOL - {UPB_WIRE_TYPE_DELIMITED, false}, // STRING - {UPB_WIRE_TYPE_START_GROUP, false}, // GROUP - {UPB_WIRE_TYPE_DELIMITED, false}, // MESSAGE - {UPB_WIRE_TYPE_DELIMITED, false}, // BYTES - {UPB_WIRE_TYPE_VARINT, true}, // UINT32 - {UPB_WIRE_TYPE_VARINT, true}, // ENUM - {UPB_WIRE_TYPE_32BIT, true}, // SFIXED32 - {UPB_WIRE_TYPE_64BIT, true}, // SFIXED64 - {UPB_WIRE_TYPE_VARINT, true}, // SINT32 - {UPB_WIRE_TYPE_VARINT, true}, // SINT64 -}; - -static upb_selector_t getselector(const upb_fielddef *f, - upb_handlertype_t type) { - upb_selector_t selector; - bool ok = upb_handlers_getselector(f, type, &selector); - UPB_ASSERT_VAR(ok, ok); - return selector; -} - - -/* decoderplan ****************************************************************/ - -#ifdef UPB_USE_JIT_X64 -// These defines are necessary for DynASM codegen. -// See dynasm/dasm_proto.h for more info. -#define Dst_DECL decoderplan *plan -#define Dst_REF (plan->dynasm) -#define Dst (plan) - -// In debug mode, make DynASM do internal checks (must be defined before any -// dasm header is included. -#ifndef NDEBUG -#define DASM_CHECKS -#endif - -#include "dynasm/dasm_proto.h" -#include "upb/pb/decoder_x64.h" -#endif - -void freeplan(void *_p) { - decoderplan *p = _p; - upb_handlers_unref(p->dest_handlers, p); -#ifdef UPB_USE_JIT_X64 - if (p->jit_code) upb_decoderplan_freejit(p); -#endif - free(p); -} - -static const decoderplan *getdecoderplan(const upb_handlers *h) { - if (upb_handlers_frametype(h) != upb_pbdecoder_getframetype()) - return NULL; - upb_selector_t sel; - if (!upb_handlers_getselector(UPB_BYTESTREAM_BYTES, UPB_HANDLER_STRING, &sel)) - return NULL; - return upb_handlers_gethandlerdata(h, sel); -} - -bool upb_pbdecoder_isdecoder(const upb_handlers *h) { - return getdecoderplan(h) != NULL; -} - -bool upb_pbdecoder_hasjitcode(const upb_handlers *h) { -#ifdef UPB_USE_JIT_X64 - const decoderplan *p = getdecoderplan(h); - if (!p) return false; - return p->jit_code != NULL; -#else - UPB_UNUSED(h); - return false; -#endif -} - -const upb_handlers *upb_pbdecoder_getdesthandlers(const upb_handlers *h) { - const decoderplan *p = getdecoderplan(h); - if (!p) return NULL; - return p->dest_handlers; +#define CHECK_SUSPEND(x) if (!(x)) return upb_pbdecoder_suspend(d); +#define CHECK_RETURN(x) { int32_t ret = x; if (ret >= 0) return ret; } + +// Error messages that are shared between the bytecode and JIT decoders. +const char *kPbDecoderStackOverflow = "Nesting too deep."; + +// Error messages shared within this file. +static const char *kUnterminatedVarint = "Unterminated varint."; + +/* upb_pbdecoder **************************************************************/ + +static opcode halt = OP_HALT; + +// Whether an op consumes any of the input buffer. +static bool consumes_input(opcode op) { + switch (op) { + case OP_SETDISPATCH: + case OP_STARTMSG: + case OP_ENDMSG: + case OP_STARTSEQ: + case OP_ENDSEQ: + case OP_STARTSUBMSG: + case OP_ENDSUBMSG: + case OP_STARTSTR: + case OP_ENDSTR: + case OP_PUSHTAGDELIM: + case OP_POP: + case OP_SETDELIM: + case OP_SETGROUPNUM: + case OP_SETBIGGROUPNUM: + case OP_CHECKDELIM: + case OP_CALL: + case OP_BRANCH: + return false; + default: + return true; + } } - -/* upb_pbdecoder ****************************************************************/ - -static bool in_residual_buf(const upb_pbdecoder *d, const char *p); +static bool in_residual_buf(upb_pbdecoder *d, const char *p); // It's unfortunate that we have to micro-manage the compiler this way, // especially since this tuning is necessarily specific to one hardware @@ -210,68 +65,65 @@ static bool in_residual_buf(const upb_pbdecoder *d, const char *p); // with these annotations. Every instance where these appear, gcc 4.2.1 made // the wrong decision and degraded performance in benchmarks. #define FORCEINLINE static inline __attribute__((always_inline)) -#define NOINLINE static __attribute__((noinline)) +#define NOINLINE __attribute__((noinline)) -static upb_status *decoder_status(upb_pbdecoder *d) { +static void seterr(upb_pbdecoder *d, const char *msg) { // TODO(haberman): encapsulate this access to pipeline->status, but not sure // exactly what that interface should look like. - return &d->sink->pipeline_->status_; + upb_status_seterrliteral(&d->sink->pipeline_->status_, msg); } -UPB_NORETURN static void exitjmp(upb_pbdecoder *d) { - _longjmp(d->exitjmp, 1); +void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg) { + seterr(d, msg); } -UPB_NORETURN static void abortjmp(upb_pbdecoder *d, const char *msg) { - d->ret = in_residual_buf(d, d->checkpoint) ? 0 : (d->checkpoint - d->buf); - upb_status_seterrliteral(decoder_status(d), msg); - exitjmp(d); -} /* Buffering ******************************************************************/ // We operate on one buffer at a time, which is either the user's buffer passed // to our "decode" callback or some residual bytes from the previous buffer. -// How many bytes can be safely read from d->ptr. -static size_t bufleft(upb_pbdecoder *d) { - assert(d->end >= d->ptr); - return d->end - d->ptr; +// How many bytes can be safely read from d->ptr without reading past end-of-buf +// or past the current delimited end. +static size_t curbufleft(upb_pbdecoder *d) { + assert(d->data_end >= d->ptr); + return d->data_end - d->ptr; +} + +static const char *ptr(upb_pbdecoder *d) { + return d->ptr; } // Overall offset of d->ptr. -uint64_t offset(const upb_pbdecoder *d) { - return d->bufstart_ofs + (d->ptr - d->buf); +uint64_t offset(upb_pbdecoder *d) { + return d->bufstart_ofs + (ptr(d) - d->buf); } // Advances d->ptr. static void advance(upb_pbdecoder *d, size_t len) { - assert(bufleft(d) >= len); + assert(curbufleft(d) >= len); d->ptr += len; } -// Commits d->ptr progress; should be called when an entire atomic value -// (ie tag+value) has been successfully consumed. -static void checkpoint(upb_pbdecoder *d) { - d->checkpoint = d->ptr; -} - static bool in_buf(const char *p, const char *buf, const char *end) { return p >= buf && p <= end; } -static bool in_residual_buf(const upb_pbdecoder *d, const char *p) { +static bool in_residual_buf(upb_pbdecoder *d, const char *p) { return in_buf(p, d->residual, d->residual_end); } // Calculates the delim_end value, which represents a combination of the // current buffer and the stack, so must be called whenever either is updated. static void set_delim_end(upb_pbdecoder *d) { - frame *f = d->top; - size_t delimlen = f->end_ofs - d->bufstart_ofs; - size_t buflen = d->end - d->buf; - d->delim_end = (f->end_ofs != UPB_NONDELIMITED && delimlen <= buflen) ? - d->buf + delimlen : NULL; // NULL if not in this buf. + size_t delim_ofs = d->top->end_ofs - d->bufstart_ofs; + if (delim_ofs <= (d->end - d->buf)) { + d->delim_end = d->buf + delim_ofs; + d->data_end = d->delim_end; + } else { + d->data_end = d->end; + d->delim_end = NULL; + } } static void switchtobuf(upb_pbdecoder *d, const char *buf, const char *end) { @@ -279,498 +131,603 @@ static void switchtobuf(upb_pbdecoder *d, const char *buf, const char *end) { d->buf = buf; d->end = end; set_delim_end(d); -#ifdef UPB_USE_JIT_X64 - // If we start parsing a value, we can parse up to 20 bytes without - // having to bounds-check anything (2 10-byte varints). Since the - // JIT bounds-checks only *between* values (and for strings), the - // JIT bails if there are not 20 bytes available. - d->jit_end = d->end - 20; -#endif -} - -static void suspendjmp(upb_pbdecoder *d) { - switchtobuf(d, d->residual, d->residual_end); - exitjmp(d); } static void advancetobuf(upb_pbdecoder *d, const char *buf, size_t len) { - assert(d->ptr == d->end); - d->bufstart_ofs += (d->ptr - d->buf); + assert(curbufleft(d) == 0); + d->bufstart_ofs += (d->end - d->buf); switchtobuf(d, buf, buf + len); } -static void skip(upb_pbdecoder *d, size_t bytes) { - size_t avail = bufleft(d); - size_t total_avail = avail + d->userbuf_remaining; - if (avail >= bytes) { +static void checkpoint(upb_pbdecoder *d) { + // The assertion here is in the interests of efficiency, not correctness. + // We are trying to ensure that we don't checkpoint() more often than + // necessary. + assert(d->checkpoint != ptr(d)); + d->checkpoint = ptr(d); +} + +// Resumes the decoder from an initial state or from a previous suspend. +void *upb_pbdecoder_resume(upb_pbdecoder *d, void *p, const char *buf, + size_t size) { + UPB_UNUSED(p); // Useless; just for the benefit of the JIT. + d->buf_param = buf; + d->size_param = size; + d->skip = 0; + if (d->residual_end > d->residual) { + // We have residual bytes from the last buffer. + assert(ptr(d) == d->residual); + } else { + switchtobuf(d, buf, buf + size); + } + d->checkpoint = ptr(d); + return d; // For the JIT. +} + +// Suspends the decoder at the last checkpoint, without saving any residual +// bytes. If there are any unconsumed bytes, returns a short byte count. +size_t upb_pbdecoder_suspend(upb_pbdecoder *d) { + d->pc = d->last; + if (d->checkpoint == d->residual) { + // Checkpoint was in residual buf; no user bytes were consumed. + d->ptr = d->residual; + return 0; + } else { + assert(!in_residual_buf(d, d->checkpoint)); + assert(d->buf == d->buf_param); + size_t consumed = d->checkpoint - d->buf; + d->bufstart_ofs += consumed + d->skip; + d->residual_end = d->residual; + switchtobuf(d, d->residual, d->residual_end); + return consumed + d->skip; + } +} + +// Suspends the decoder at the last checkpoint, and saves any unconsumed +// bytes in our residual buffer. This is necessary if we need more user +// bytes to form a complete value, which might not be contiguous in the +// user's buffers. Always consumes all user bytes. +static size_t suspend_save(upb_pbdecoder *d) { + // We hit end-of-buffer before we could parse a full value. + // Save any unconsumed bytes (if any) to the residual buffer. + d->pc = d->last; + + if (d->checkpoint == d->residual) { + // Checkpoint was in residual buf; append user byte(s) to residual buf. + assert((d->residual_end - d->residual) + d->size_param <= + sizeof(d->residual)); + if (!in_residual_buf(d, ptr(d))) { + d->bufstart_ofs -= (d->residual_end - d->residual); + } + memcpy(d->residual_end, d->buf_param, d->size_param); + d->residual_end += d->size_param; + } else { + // Checkpoint was in user buf; old residual bytes not needed. + assert(!in_residual_buf(d, d->checkpoint)); + d->ptr = d->checkpoint; + size_t save = curbufleft(d); + assert(save <= sizeof(d->residual)); + memcpy(d->residual, ptr(d), save); + d->residual_end = d->residual + save; + d->bufstart_ofs = offset(d) + d->skip; + } + + switchtobuf(d, d->residual, d->residual_end); + return d->size_param + d->skip; +} + +static int32_t skip(upb_pbdecoder *d, size_t bytes) { + assert(!in_residual_buf(d, ptr(d)) || d->size_param == 0); + if (curbufleft(d) >= bytes) { // Skipped data is all in current buffer. advance(d, bytes); - } else if (total_avail >= bytes) { - // Skipped data is all in residual buf and param buffer. - assert(in_residual_buf(d, d->ptr)); - advance(d, avail); - advancetobuf(d, d->buf_param, d->size_param); - d->userbuf_remaining = 0; - advance(d, bytes - avail); } else { // Skipped data extends beyond currently available buffers. - // TODO: we need to do a checkdelim() equivalent that pops any frames that - // we just skipped past. - d->bufstart_ofs = offset(d) + bytes; - d->residual_end = d->residual; - d->ret += bytes - total_avail; - suspendjmp(d); + d->skip = bytes - curbufleft(d); + advance(d, curbufleft(d)); } + return DECODE_OK; } -static void consumebytes(upb_pbdecoder *d, void *buf, size_t bytes) { - assert(bytes <= bufleft(d)); - memcpy(buf, d->ptr, bytes); +FORCEINLINE void consumebytes(upb_pbdecoder *d, void *buf, size_t bytes) { + assert(bytes <= curbufleft(d)); + memcpy(buf, ptr(d), bytes); advance(d, bytes); } -NOINLINE void getbytes_slow(upb_pbdecoder *d, void *buf, size_t bytes) { - const size_t avail = bufleft(d); - if (avail + d->userbuf_remaining >= bytes) { - // Remaining residual buffer and param buffer together can satisfy. - // (We are only called from getbytes() which has already verified that - // the current buffer alone cannot satisfy). - assert(in_residual_buf(d, d->ptr)); - consumebytes(d, buf, avail); +static NOINLINE int32_t getbytes_slow(upb_pbdecoder *d, void *buf, + size_t bytes) { + const size_t avail = curbufleft(d); + consumebytes(d, buf, avail); + bytes -= avail; + assert(bytes > 0); + if (in_residual_buf(d, ptr(d))) { advancetobuf(d, d->buf_param, d->size_param); - consumebytes(d, buf + avail, bytes - avail); - d->userbuf_remaining = 0; + } + if (curbufleft(d) >= bytes) { + consumebytes(d, buf + avail, bytes); + return DECODE_OK; + } else if (d->data_end - d->buf == d->top->end_ofs - d->bufstart_ofs) { + seterr(d, "Submessage ended in the middle of a value"); + return upb_pbdecoder_suspend(d); } else { - // There is not enough remaining data, save residual bytes (if any) - // starting at the last committed checkpoint and exit. - if (in_buf(d->checkpoint, d->buf_param, d->buf_param + d->size_param)) { - // Checkpoint was in user buf; old residual bytes not needed. - d->ptr = d->checkpoint; - size_t save = bufleft(d); - assert(save <= sizeof(d->residual)); - memcpy(d->residual, d->ptr, save); - d->residual_end = d->residual + save; - d->bufstart_ofs = offset(d); - } else { - // Checkpoint was in residual buf; append user byte(s) to residual buf. - assert(d->checkpoint == d->residual); - assert((d->residual_end - d->residual) + d->size_param <= - sizeof(d->residual)); - if (!in_residual_buf(d, d->ptr)) { - d->bufstart_ofs -= (d->residual_end - d->residual); - } - memcpy(d->residual_end, d->buf_param, d->size_param); - d->residual_end += d->size_param; - } - suspendjmp(d); + return suspend_save(d); } } -FORCEINLINE void getbytes(upb_pbdecoder *d, void *buf, size_t bytes) { - if (bufleft(d) >= bytes) { +FORCEINLINE int32_t getbytes(upb_pbdecoder *d, void *buf, size_t bytes) { + if (curbufleft(d) >= bytes) { // Buffer has enough data to satisfy. consumebytes(d, buf, bytes); + return DECODE_OK; } else { - getbytes_slow(d, buf, bytes); + return getbytes_slow(d, buf, bytes); + } +} + +static NOINLINE size_t peekbytes_slow(upb_pbdecoder *d, void *buf, + size_t bytes) { + size_t ret = curbufleft(d); + memcpy(buf, ptr(d), ret); + if (in_residual_buf(d, ptr(d))) { + size_t copy = UPB_MIN(bytes - ret, d->size_param); + memcpy(buf + ret, d->buf_param, copy); + ret += copy; } + return ret; } -FORCEINLINE uint8_t getbyte(upb_pbdecoder *d) { - uint8_t byte; - getbytes(d, &byte, 1); - return byte; +FORCEINLINE size_t peekbytes(upb_pbdecoder *d, void *buf, size_t bytes) { + if (curbufleft(d) >= bytes) { + memcpy(buf, ptr(d), bytes); + return bytes; + } else { + return peekbytes_slow(d, buf, bytes); + } } /* Decoding of wire types *****************************************************/ -NOINLINE uint64_t decode_varint_slow(upb_pbdecoder *d) { +NOINLINE int32_t upb_pbdecoder_decode_varint_slow(upb_pbdecoder *d, + uint64_t *u64) { + *u64 = 0; uint8_t byte = 0x80; - uint64_t u64 = 0; int bitpos; for(bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) { - u64 |= ((uint64_t)((byte = getbyte(d)) & 0x7F)) << bitpos; + int32_t ret = getbytes(d, &byte, 1); + if (ret >= 0) return ret; + *u64 |= (uint64_t)(byte & 0x7F) << bitpos; } - if(bitpos == 70 && (byte & 0x80)) - abortjmp(d, "Unterminated varint"); - return u64; -} - -NOINLINE uint32_t decode_v32_slow(upb_pbdecoder *d) { - uint64_t u64 = decode_varint_slow(d); - if (u64 > UINT32_MAX) abortjmp(d, "Unterminated 32-bit varint"); - return (uint32_t)u64; -} - -// For tags and delimited lengths, which must be <=32bit and are usually small. -FORCEINLINE uint32_t decode_v32(upb_pbdecoder *d) { - // Nearly all will be either 1 byte (1-16) or 2 bytes (17-2048). - if (bufleft(d) >= 2) { - uint32_t ret = d->ptr[0] & 0x7f; - if ((d->ptr[0] & 0x80) == 0) { - advance(d, 1); - return ret; - } - ret |= (d->ptr[1] & 0x7f) << 7; - if ((d->ptr[1] & 0x80) == 0) { - advance(d, 2); - return ret; - } + if(bitpos == 70 && (byte & 0x80)) { + seterr(d, kUnterminatedVarint); + return upb_pbdecoder_suspend(d); } - return decode_v32_slow(d); + return DECODE_OK; } -FORCEINLINE uint64_t decode_varint(upb_pbdecoder *d) { - if (bufleft(d) >= 10) { +FORCEINLINE int32_t decode_varint(upb_pbdecoder *d, uint64_t *u64) { + if (curbufleft(d) > 0 && !(*ptr(d) & 0x80)) { + *u64 = *ptr(d); + advance(d, 1); + return DECODE_OK; + } else if (curbufleft(d) >= 10) { // Fast case. - upb_decoderet r = upb_vdecode_fast(d->ptr); - if (r.p == NULL) abortjmp(d, "Unterminated varint"); - advance(d, r.p - d->ptr); - return r.val; + upb_decoderet r = upb_vdecode_fast(ptr(d)); + if (r.p == NULL) { + seterr(d, kUnterminatedVarint); + return upb_pbdecoder_suspend(d); + } + advance(d, r.p - ptr(d)); + *u64 = r.val; + return DECODE_OK; } else { // Slow case -- varint spans buffer seam. - return decode_varint_slow(d); + return upb_pbdecoder_decode_varint_slow(d, u64); } } -FORCEINLINE uint32_t decode_fixed32(upb_pbdecoder *d) { - uint32_t u32; - getbytes(d, &u32, 4); - return u32; // TODO: proper byte swapping for big-endian machines. -} - -FORCEINLINE uint64_t decode_fixed64(upb_pbdecoder *d) { +FORCEINLINE int32_t decode_v32(upb_pbdecoder *d, uint32_t *u32) { uint64_t u64; - getbytes(d, &u64, 8); - return u64; // TODO: proper byte swapping for big-endian machines. + int32_t ret = decode_varint(d, &u64); + if (ret >= 0) return ret; + if (u64 > UINT32_MAX) { + seterr(d, "Unterminated 32-bit varint"); + return upb_pbdecoder_suspend(d); + } + *u32 = u64; + return DECODE_OK; } -static void push(upb_pbdecoder *d, const upb_fielddef *f, bool is_sequence, - bool is_packed, int32_t group_fieldnum, uint64_t end) { - frame *fr = d->top + 1; - if (fr >= d->limit) abortjmp(d, "Nesting too deep."); - fr->f = f; - fr->is_sequence = is_sequence; - fr->is_packed = is_packed; - fr->end_ofs = end; - fr->group_fieldnum = group_fieldnum; - d->top = fr; - set_delim_end(d); +// TODO: proper byte swapping for big-endian machines. +FORCEINLINE int32_t decode_fixed32(upb_pbdecoder *d, uint32_t *u32) { + return getbytes(d, u32, 4); } -static void push_msg(upb_pbdecoder *d, const upb_fielddef *f, uint64_t end) { - if (!upb_sink_startsubmsg(d->sink, getselector(f, UPB_HANDLER_STARTSUBMSG))) - abortjmp(d, "startsubmsg failed."); - int32_t group_fieldnum = (end == UPB_NONDELIMITED) ? - (int32_t)upb_fielddef_number(f) : -1; - push(d, f, false, false, group_fieldnum, end); +// TODO: proper byte swapping for big-endian machines. +FORCEINLINE int32_t decode_fixed64(upb_pbdecoder *d, uint64_t *u64) { + return getbytes(d, u64, 8); } -static void push_seq(upb_pbdecoder *d, const upb_fielddef *f, bool packed, - uint64_t end_ofs) { - if (!upb_sink_startseq(d->sink, getselector(f, UPB_HANDLER_STARTSEQ))) - abortjmp(d, "startseq failed."); - push(d, f, true, packed, -1, end_ofs); +int32_t upb_pbdecoder_decode_f32(upb_pbdecoder *d, uint32_t *u32) { + return decode_fixed32(d, u32); } -static void push_str(upb_pbdecoder *d, const upb_fielddef *f, size_t len, - uint64_t end) { - if (!upb_sink_startstr(d->sink, getselector(f, UPB_HANDLER_STARTSTR), len)) - abortjmp(d, "startseq failed."); - push(d, f, false, false, -1, end); +int32_t upb_pbdecoder_decode_f64(upb_pbdecoder *d, uint64_t *u64) { + return decode_fixed64(d, u64); } -static void pop_submsg(upb_pbdecoder *d) { - upb_sink_endsubmsg(d->sink, getselector(d->top->f, UPB_HANDLER_ENDSUBMSG)); - d->top--; - set_delim_end(d); -} +static double as_double(uint64_t n) { double d; memcpy(&d, &n, 8); return d; } +static float as_float(uint32_t n) { float f; memcpy(&f, &n, 4); return f; } -static void pop_seq(upb_pbdecoder *d) { - upb_sink_endseq(d->sink, getselector(d->top->f, UPB_HANDLER_ENDSEQ)); - d->top--; - set_delim_end(d); +static bool push(upb_pbdecoder *d, uint64_t end) { + upb_pbdecoder_frame *fr = d->top; + + if (end > fr->end_ofs) { + seterr(d, "Submessage end extends past enclosing submessage."); + return false; + } else if ((fr + 1) == d->limit) { + seterr(d, kPbDecoderStackOverflow); + return false; + } + + fr++; + fr->end_ofs = end; + fr->u.dispatch = NULL; + fr->groupnum = -1; + d->top = fr; + return true; } -static void pop_string(upb_pbdecoder *d) { - upb_sink_endstr(d->sink, getselector(d->top->f, UPB_HANDLER_ENDSTR)); - d->top--; - set_delim_end(d); +NOINLINE int32_t upb_pbdecoder_checktag_slow(upb_pbdecoder *d, + uint64_t expected) { + uint64_t data = 0; + size_t bytes = upb_value_size(expected); + size_t read = peekbytes(d, &data, bytes); + if (read == bytes && data == expected) { + // Advance past matched bytes. + int32_t ok = getbytes(d, &data, read); + UPB_ASSERT_VAR(ok, ok < 0); + return DECODE_OK; + } else if (read < bytes && memcmp(&data, &expected, read) == 0) { + return suspend_save(d); + } else { + return DECODE_MISMATCH; + } } -static void checkdelim(upb_pbdecoder *d) { - while (d->delim_end && d->ptr >= d->delim_end) { - // TODO(haberman): not sure what to do about this; if we detect this error - // we can possibly violate the promise that errors are always signaled by a - // short "parsed byte" count (because all bytes might have been successfully - // parsed prior to detecting this error). - // if (d->ptr > d->delim_end) abortjmp(d, "Bad submessage end"); - if (d->top->is_sequence) { - pop_seq(d); - } else { - pop_submsg(d); +int32_t upb_pbdecoder_skipunknown(upb_pbdecoder *d, uint32_t fieldnum, + uint8_t wire_type) { + if (fieldnum == 0 || fieldnum > UPB_MAX_FIELDNUMBER) { + seterr(d, "Invalid field number"); + return upb_pbdecoder_suspend(d); + } + + if (wire_type == UPB_WIRE_TYPE_END_GROUP) { + if (fieldnum != d->top->groupnum) { + seterr(d, "Unmatched ENDGROUP tag."); + return upb_pbdecoder_suspend(d); + } + return DECODE_ENDGROUP; + } + + // TODO: deliver to unknown field callback. + switch (wire_type) { + case UPB_WIRE_TYPE_VARINT: { + uint64_t u64; + return decode_varint(d, &u64); + } + case UPB_WIRE_TYPE_32BIT: + return skip(d, 4); + case UPB_WIRE_TYPE_64BIT: + return skip(d, 8); + case UPB_WIRE_TYPE_DELIMITED: { + uint32_t len; + CHECK_RETURN(decode_v32(d, &len)); + return skip(d, len); } + case UPB_WIRE_TYPE_START_GROUP: + seterr(d, "Can't handle unknown groups yet"); + return upb_pbdecoder_suspend(d); + case UPB_WIRE_TYPE_END_GROUP: + default: + seterr(d, "Invalid wire type"); + return upb_pbdecoder_suspend(d); } } +static int32_t dispatch(upb_pbdecoder *d) { + upb_inttable *dispatch = d->top->u.dispatch; + + // Decode tag. + uint32_t tag; + CHECK_RETURN(decode_v32(d, &tag)); + uint8_t wire_type = tag & 0x7; + uint32_t fieldnum = tag >> 3; + + // Lookup tag. Because of packed/non-packed compatibility, we have to + // check the wire type against two possibilities. + upb_value val; + if (upb_inttable_lookup32(dispatch, fieldnum, &val)) { + uint64_t v = upb_value_getuint64(val); + if (wire_type == (v & 0xff)) { + d->pc = d->top->base + (v >> 16); + return DECODE_OK; + } else if (wire_type == ((v >> 8) & 0xff)) { + bool found = + upb_inttable_lookup(dispatch, fieldnum + UPB_MAX_FIELDNUMBER, &val); + UPB_ASSERT_VAR(found, found); + d->pc = d->top->base + upb_value_getuint64(val); + return DECODE_OK; + } + } + + // Unknown field or ENDGROUP. + int32_t ret = upb_pbdecoder_skipunknown(d, fieldnum, wire_type); -/* Decoding of .proto types ***************************************************/ - -// Technically, we are losing data if we see a 32-bit varint that is not -// properly sign-extended. We could detect this and error about the data loss, -// but proto2 does not do this, so we pass. - -#define T(type, sel, wt, name, convfunc) \ - static void decode_ ## type(upb_pbdecoder *d, const upb_fielddef *f) { \ - upb_sink_put ## name(d->sink, getselector(f, UPB_HANDLER_ ## sel), \ - (convfunc)(decode_ ## wt(d))); \ - } \ - -static double upb_asdouble(uint64_t n) { double d; memcpy(&d, &n, 8); return d; } -static float upb_asfloat(uint32_t n) { float f; memcpy(&f, &n, 4); return f; } - -T(INT32, INT32, varint, int32, int32_t) -T(INT64, INT64, varint, int64, int64_t) -T(UINT32, UINT32, varint, uint32, uint32_t) -T(UINT64, UINT64, varint, uint64, uint64_t) -T(FIXED32, UINT32, fixed32, uint32, uint32_t) -T(FIXED64, UINT64, fixed64, uint64, uint64_t) -T(SFIXED32, INT32, fixed32, int32, int32_t) -T(SFIXED64, INT64, fixed64, int64, int64_t) -T(BOOL, BOOL, varint, bool, bool) -T(ENUM, INT32, varint, int32, int32_t) -T(DOUBLE, DOUBLE, fixed64, double, upb_asdouble) -T(FLOAT, FLOAT, fixed32, float, upb_asfloat) -T(SINT32, INT32, varint, int32, upb_zzdec_32) -T(SINT64, INT64, varint, int64, upb_zzdec_64) -#undef T - -static void decode_GROUP(upb_pbdecoder *d, const upb_fielddef *f) { - push_msg(d, f, UPB_NONDELIMITED); -} - -static void decode_MESSAGE(upb_pbdecoder *d, const upb_fielddef *f) { - uint32_t len = decode_v32(d); - push_msg(d, f, offset(d) + len); -} - -static void decode_STRING(upb_pbdecoder *d, const upb_fielddef *f) { - uint32_t strlen = decode_v32(d); - if (strlen <= bufleft(d)) { - upb_sink_startstr(d->sink, getselector(f, UPB_HANDLER_STARTSTR), strlen); - if (strlen) - upb_sink_putstring(d->sink, getselector(f, UPB_HANDLER_STRING), - d->ptr, strlen); - upb_sink_endstr(d->sink, getselector(f, UPB_HANDLER_ENDSTR)); - advance(d, strlen); + if (ret == DECODE_ENDGROUP) { + d->pc = d->top->base - 1; // Back to OP_ENDMSG. + return DECODE_OK; } else { - // Buffer ends in the middle of the string; need to push a decoder frame - // for it. - push_str(d, f, strlen, offset(d) + strlen); - if (bufleft(d)) { - upb_sink_putstring(d->sink, getselector(f, UPB_HANDLER_STRING), - d->ptr, bufleft(d)); - advance(d, bufleft(d)); - } - d->bufstart_ofs = offset(d); - d->residual_end = d->residual; - suspendjmp(d); + d->pc = d->last - 1; // Rewind to CHECKDELIM. + return ret; } } /* The main decoding loop *****************************************************/ -static const upb_fielddef *decode_tag(upb_pbdecoder *d) { - while (1) { - uint32_t tag = decode_v32(d); - uint8_t wire_type = tag & 0x7; - uint32_t fieldnum = tag >> 3; const upb_fielddef *f = NULL; - const upb_handlers *h = d->sink->top->h; // TODO(haberman): rm - f = upb_msgdef_itof(upb_handlers_msgdef(h), fieldnum); - bool packed = false; - - if (f) { - // Wire type check. - upb_descriptortype_t type = upb_fielddef_descriptortype(f); - if (wire_type == upb_decoder_types[type].native_wire_type) { - // Wire type is ok. - } else if ((wire_type == UPB_WIRE_TYPE_DELIMITED && - upb_decoder_types[type].is_numeric)) { - // Wire type is ok (and packed). - packed = true; - } else { - f = NULL; - } - } - - // There are no explicit "startseq" or "endseq" markers in protobuf - // streams, so we have to infer them by noticing when a repeated field - // starts or ends. - frame *fr = d->top; - if (fr->is_sequence && fr->f != f) { - pop_seq(d); - fr = d->top; - } +size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf, + size_t size) { + upb_pbdecoder *d = closure; + const upb_pbdecoderplan *p = hd; + assert(buf); + upb_pbdecoder_resume(d, NULL, buf, size); + UPB_UNUSED(p); - if (f && upb_fielddef_isseq(f) && !fr->is_sequence) { - if (packed) { - uint32_t len = decode_v32(d); - push_seq(d, f, true, offset(d) + len); - checkpoint(d); - } else { - push_seq(d, f, false, fr->end_ofs); - } - } +#define VMCASE(op, code) \ + case op: { code; if (consumes_input(op)) checkpoint(d); break; } +#define PRIMITIVE_OP(type, wt, name, convfunc, ctype) \ + VMCASE(OP_PARSE_ ## type, { \ + ctype val; \ + CHECK_RETURN(decode_ ## wt(d, &val)); \ + upb_sink_put ## name(d->sink, arg, (convfunc)(val)); \ + }) - if (f) return f; - - // Unknown field or ENDGROUP. - if (fieldnum == 0 || fieldnum > UPB_MAX_FIELDNUMBER) - abortjmp(d, "Invalid field number"); - switch (wire_type) { - case UPB_WIRE_TYPE_VARINT: decode_varint(d); break; - case UPB_WIRE_TYPE_32BIT: skip(d, 4); break; - case UPB_WIRE_TYPE_64BIT: skip(d, 8); break; - case UPB_WIRE_TYPE_DELIMITED: skip(d, decode_v32(d)); break; - case UPB_WIRE_TYPE_START_GROUP: - abortjmp(d, "Can't handle unknown groups yet"); - case UPB_WIRE_TYPE_END_GROUP: - if (fieldnum != fr->group_fieldnum) - abortjmp(d, "Unmatched ENDGROUP tag"); - pop_submsg(d); - break; - default: - abortjmp(d, "Invalid wire type"); + while(1) { + d->last = d->pc; + int32_t instruction = *d->pc++; + opcode op = getop(instruction); + uint32_t arg = instruction >> 8; + int32_t longofs = arg; + assert(ptr(d) != d->residual_end); +#ifdef UPB_DUMP_BYTECODE + fprintf(stderr, "s_ofs=%d buf_ofs=%d data_rem=%d buf_rem=%d delim_rem=%d " + "%x %s (%d)\n", + (int)offset(d), + (int)(ptr(d) - d->buf), + (int)(d->data_end - ptr(d)), + (int)(d->end - ptr(d)), + (int)((d->top->end_ofs - d->bufstart_ofs) - (ptr(d) - d->buf)), + (int)(d->pc - 1 - upb_pbdecoderplan_codebase(p)), + upb_pbdecoder_getopname(op), + arg); +#endif + switch (op) { + // Technically, we are losing data if we see a 32-bit varint that is not + // properly sign-extended. We could detect this and error about the data + // loss, but proto2 does not do this, so we pass. + PRIMITIVE_OP(INT32, varint, int32, int32_t, uint64_t) + PRIMITIVE_OP(INT64, varint, int64, int64_t, uint64_t) + PRIMITIVE_OP(UINT32, varint, uint32, uint32_t, uint64_t) + PRIMITIVE_OP(UINT64, varint, uint64, uint64_t, uint64_t) + PRIMITIVE_OP(FIXED32, fixed32, uint32, uint32_t, uint32_t) + PRIMITIVE_OP(FIXED64, fixed64, uint64, uint64_t, uint64_t) + PRIMITIVE_OP(SFIXED32, fixed32, int32, int32_t, uint32_t) + PRIMITIVE_OP(SFIXED64, fixed64, int64, int64_t, uint64_t) + PRIMITIVE_OP(BOOL, varint, bool, bool, uint64_t) + PRIMITIVE_OP(DOUBLE, fixed64, double, as_double, uint64_t) + PRIMITIVE_OP(FLOAT, fixed32, float, as_float, uint32_t) + PRIMITIVE_OP(SINT32, varint, int32, upb_zzdec_32, uint64_t) + PRIMITIVE_OP(SINT64, varint, int64, upb_zzdec_64, uint64_t) + + VMCASE(OP_SETDISPATCH, + d->top->base = d->pc - 1; + memcpy(&d->top->u.dispatch, d->pc, sizeof(void*)); + d->pc += sizeof(void*) / sizeof(uint32_t); + ) + VMCASE(OP_STARTMSG, + CHECK_SUSPEND(upb_sink_startmsg(d->sink)); + ) + VMCASE(OP_ENDMSG, + CHECK_SUSPEND(upb_sink_endmsg(d->sink)); + assert(d->call_len > 0); + d->pc = d->callstack[--d->call_len]; + ) + VMCASE(OP_STARTSEQ, + CHECK_SUSPEND(upb_sink_startseq(d->sink, arg)); + ) + VMCASE(OP_ENDSEQ, + CHECK_SUSPEND(upb_sink_endseq(d->sink, arg)); + ) + VMCASE(OP_STARTSUBMSG, + CHECK_SUSPEND(upb_sink_startsubmsg(d->sink, arg)); + ) + VMCASE(OP_ENDSUBMSG, + CHECK_SUSPEND(upb_sink_endsubmsg(d->sink, arg)); + ) + VMCASE(OP_STARTSTR, + uint32_t len = d->top->end_ofs - offset(d); + CHECK_SUSPEND(upb_sink_startstr(d->sink, arg, len)); + if (len == 0) { + d->pc++; // Skip OP_STRING. + } + ) + VMCASE(OP_STRING, + uint32_t len = curbufleft(d); + CHECK_SUSPEND(upb_sink_putstring(d->sink, arg, ptr(d), len)); + advance(d, len); + if (d->delim_end == NULL) { // String extends beyond this buf? + d->pc--; + d->bufstart_ofs += size; + d->residual_end = d->residual; + return size; + } + ) + VMCASE(OP_ENDSTR, + CHECK_SUSPEND(upb_sink_endstr(d->sink, arg)); + ) + VMCASE(OP_PUSHTAGDELIM, + CHECK_SUSPEND(push(d, d->top->end_ofs)); + ) + VMCASE(OP_POP, + assert(d->top > d->stack); + d->top--; + ) + VMCASE(OP_PUSHLENDELIM, + uint32_t len; + CHECK_RETURN(decode_v32(d, &len)); + CHECK_SUSPEND(push(d, offset(d) + len)); + set_delim_end(d); + ) + VMCASE(OP_SETDELIM, + set_delim_end(d); + ) + VMCASE(OP_SETGROUPNUM, + d->top->groupnum = arg; + ) + VMCASE(OP_SETBIGGROUPNUM, + d->top->groupnum = *d->pc++; + ) + VMCASE(OP_CHECKDELIM, + assert(!(d->delim_end && ptr(d) > d->delim_end)); + if (ptr(d) == d->delim_end) + d->pc += longofs; + ) + VMCASE(OP_CALL, + d->callstack[d->call_len++] = d->pc; + d->pc += longofs; + ) + VMCASE(OP_BRANCH, + d->pc += longofs; + ) + VMCASE(OP_TAG1, + CHECK_SUSPEND(curbufleft(d) > 0); + uint8_t expected = (arg >> 8) & 0xff; + if (*ptr(d) == expected) { + advance(d, 1); + } else { + int8_t shortofs; + badtag: + shortofs = arg; + if (shortofs == LABEL_DISPATCH) { + CHECK_RETURN(dispatch(d)); + } else { + d->pc += shortofs; + break; // Avoid checkpoint(). + } + } + ) + VMCASE(OP_TAG2, + CHECK_SUSPEND(curbufleft(d) > 0); + uint16_t expected = (arg >> 8) & 0xffff; + if (curbufleft(d) >= 2) { + uint16_t actual; + memcpy(&actual, ptr(d), 2); + if (expected == actual) { + advance(d, 2); + } else { + goto badtag; + } + } else { + int32_t result = upb_pbdecoder_checktag_slow(d, expected); + if (result == DECODE_MISMATCH) goto badtag; + if (result >= 0) return result; + } + ) + VMCASE(OP_TAGN, { + uint64_t expected; + memcpy(&expected, d->pc, 8); + d->pc += 2; + int32_t result = upb_pbdecoder_checktag_slow(d, expected); + if (result == DECODE_MISMATCH) goto badtag; + if (result >= 0) return result; + }) + VMCASE(OP_HALT, { + return size; + }) } - // TODO: deliver to unknown field callback. - checkpoint(d); - checkdelim(d); } } -void *start(void *closure, const void *handler_data, size_t size_hint) { - UPB_UNUSED(handler_data); +void *upb_pbdecoder_start(void *closure, const void *handler_data, + size_t size_hint) { UPB_UNUSED(size_hint); upb_pbdecoder *d = closure; + const upb_pbdecoderplan *plan = handler_data; + UPB_UNUSED(plan); + if (upb_pbdecoderplan_hasjitcode(plan)) { + d->top->u.closure = d->sink->top->closure; + d->call_len = 0; + } else { + d->call_len = 1; + d->pc = upb_pbdecoderplan_codebase(plan); + } assert(d); assert(d->sink); - upb_sink_startmsg(d->sink); + if (plan->topmethod->dest_handlers) { + assert(d->sink->top->h == plan->topmethod->dest_handlers); + } + d->status = &d->sink->pipeline_->status_; return d; } -bool end(void *closure, const void *handler_data) { - UPB_UNUSED(handler_data); +bool upb_pbdecoder_end(void *closure, const void *handler_data) { upb_pbdecoder *d = closure; + const upb_pbdecoderplan *plan = handler_data; if (d->residual_end > d->residual) { - // We have preserved bytes. - upb_status_seterrliteral(decoder_status(d), "Unexpected EOF"); - return false; - } - - // We may need to dispatch a top-level implicit frame. - if (d->top == d->stack + 1 && - d->top->is_sequence && - !d->top->is_packed) { - pop_seq(d); - } - if (d->top != d->stack) { - upb_status_seterrliteral( - decoder_status(d), "Ended inside delimited field."); + seterr(d, "Unexpected EOF"); return false; } - upb_sink_endmsg(d->sink); - return true; -} - -size_t decode(void *closure, const void *hd, const char *buf, size_t size) { - upb_pbdecoder *d = closure; - const decoderplan *plan = hd; - UPB_UNUSED(plan); - assert(d->sink->top->h == plan->dest_handlers); - - if (size == 0) return 0; - // Assume we'll consume the whole buffer unless this is overwritten. - d->ret = size; - d->buf_param = buf; - d->size_param = size; - - if (_setjmp(d->exitjmp)) { - // Hit end-of-buffer or error. - return d->ret; - } - - if (d->residual_end > d->residual) { - // We have residual bytes from the last buffer. - d->userbuf_remaining = d->size_param; - } else { - d->userbuf_remaining = 0; - advancetobuf(d, buf, d->size_param); - - if (d->top != d->stack && - upb_fielddef_isstring(d->top->f) && - !d->top->is_sequence) { - // Last buffer ended in the middle of a string; deliver more of it. - size_t len = d->top->end_ofs - offset(d); - if (d->size_param >= len) { - upb_sink_putstring(d->sink, getselector(d->top->f, UPB_HANDLER_STRING), - d->ptr, len); - advance(d, len); - pop_string(d); - } else { - upb_sink_putstring(d->sink, getselector(d->top->f, UPB_HANDLER_STRING), - d->ptr, d->size_param); - advance(d, d->size_param); - d->residual_end = d->residual; - advancetobuf(d, d->residual, 0); - return d->size_param; - } - } - } - checkpoint(d); - const upb_fielddef *f = d->top->f; - while(1) { + // Message ends here. + uint64_t end = offset(d); + d->top->end_ofs = end; + char dummy; + if (upb_pbdecoderplan_hasjitcode(plan)) { #ifdef UPB_USE_JIT_X64 - upb_decoder_enterjit(d, plan); - checkpoint(d); - set_delim_end(d); // JIT doesn't keep this current. + if (d->top != d->stack) + d->stack->end_ofs = 0; + upb_pbdecoderplan_jitcode(plan)(closure, handler_data, &dummy, 0); #endif - checkdelim(d); - if (!d->top->is_packed) { - f = decode_tag(d); + } else { + d->stack->end_ofs = end; + uint32_t *p = d->pc - 1; + if (getop(*p) == OP_CHECKDELIM) { + // Rewind from OP_TAG* to OP_CHECKDELIM. + assert(getop(*d->pc) == OP_TAG1 || + getop(*d->pc) == OP_TAG2 || + getop(*d->pc) == OP_TAGN); + d->pc = p; } + upb_pbdecoder_decode(closure, handler_data, &dummy, 0); + } - switch (upb_fielddef_descriptortype(f)) { - case UPB_DESCRIPTOR_TYPE_DOUBLE: decode_DOUBLE(d, f); break; - case UPB_DESCRIPTOR_TYPE_FLOAT: decode_FLOAT(d, f); break; - case UPB_DESCRIPTOR_TYPE_INT64: decode_INT64(d, f); break; - case UPB_DESCRIPTOR_TYPE_UINT64: decode_UINT64(d, f); break; - case UPB_DESCRIPTOR_TYPE_INT32: decode_INT32(d, f); break; - case UPB_DESCRIPTOR_TYPE_FIXED64: decode_FIXED64(d, f); break; - case UPB_DESCRIPTOR_TYPE_FIXED32: decode_FIXED32(d, f); break; - case UPB_DESCRIPTOR_TYPE_BOOL: decode_BOOL(d, f); break; - case UPB_DESCRIPTOR_TYPE_STRING: UPB_FALLTHROUGH_INTENDED; - case UPB_DESCRIPTOR_TYPE_BYTES: decode_STRING(d, f); break; - case UPB_DESCRIPTOR_TYPE_GROUP: decode_GROUP(d, f); break; - case UPB_DESCRIPTOR_TYPE_MESSAGE: decode_MESSAGE(d, f); break; - case UPB_DESCRIPTOR_TYPE_UINT32: decode_UINT32(d, f); break; - case UPB_DESCRIPTOR_TYPE_ENUM: decode_ENUM(d, f); break; - case UPB_DESCRIPTOR_TYPE_SFIXED32: decode_SFIXED32(d, f); break; - case UPB_DESCRIPTOR_TYPE_SFIXED64: decode_SFIXED64(d, f); break; - case UPB_DESCRIPTOR_TYPE_SINT32: decode_SINT32(d, f); break; - case UPB_DESCRIPTOR_TYPE_SINT64: decode_SINT64(d, f); break; - } - checkpoint(d); + if (d->call_len != 0) { + seterr(d, "Unexpected EOF"); + return false; } + + return upb_ok(&d->sink->pipeline_->status_); } void init(void *_d, upb_pipeline *p) { UPB_UNUSED(p); upb_pbdecoder *d = _d; - d->limit = &d->stack[UPB_MAX_NESTING]; + d->limit = &d->stack[UPB_DECODER_MAX_NESTING]; d->sink = NULL; + d->callstack[0] = &halt; // reset() must be called before decoding; this is guaranteed by assert() in // start(). } @@ -778,15 +735,13 @@ void init(void *_d, upb_pipeline *p) { void reset(void *_d) { upb_pbdecoder *d = _d; d->top = d->stack; - d->top->is_sequence = false; - d->top->is_packed = false; - d->top->group_fieldnum = UINT32_MAX; - d->top->end_ofs = UPB_NONDELIMITED; + d->top->end_ofs = UINT64_MAX; d->bufstart_ofs = 0; d->ptr = d->residual; d->buf = d->residual; d->end = d->residual; d->residual_end = d->residual; + d->call_len = 1; } bool upb_pbdecoder_resetsink(upb_pbdecoder *d, upb_sink* sink) { @@ -807,24 +762,3 @@ const upb_frametype upb_pbdecoder_frametype = { const upb_frametype *upb_pbdecoder_getframetype() { return &upb_pbdecoder_frametype; } - -const upb_handlers *upb_pbdecoder_gethandlers(const upb_handlers *dest, - bool allowjit, - const void *owner) { - UPB_UNUSED(allowjit); - decoderplan *p = malloc(sizeof(*p)); - assert(upb_handlers_isfrozen(dest)); - p->dest_handlers = dest; - upb_handlers_ref(dest, p); -#ifdef UPB_USE_JIT_X64 - p->jit_code = NULL; - if (allowjit) upb_decoderplan_makejit(p); -#endif - - upb_handlers *h = upb_handlers_new( - UPB_BYTESTREAM, &upb_pbdecoder_frametype, owner); - upb_handlers_setstartstr(h, UPB_BYTESTREAM_BYTES, start, NULL, NULL); - upb_handlers_setstring(h, UPB_BYTESTREAM_BYTES, decode, p, freeplan); - upb_handlers_setendstr(h, UPB_BYTESTREAM_BYTES, end, NULL, NULL); - return h; -} diff --git a/upb/pb/decoder.h b/upb/pb/decoder.h index c1b6cb3..c645688 100644 --- a/upb/pb/decoder.h +++ b/upb/pb/decoder.h @@ -14,6 +14,13 @@ #include "upb/sink.h" +// The maximum that any submessages can be nested. Matches proto2's limit. +// At the moment this specifies the size of several statically-sized arrays +// and therefore setting it high will cause more memory to be used. Will +// be replaced by a runtime-configurable limit and dynamically-resizing arrays. +// TODO: make this a runtime-settable property of Decoder. +#define UPB_DECODER_MAX_NESTING 64 + #ifdef __cplusplus namespace upb { namespace pb { diff --git a/upb/pb/decoder.int.h b/upb/pb/decoder.int.h new file mode 100644 index 0000000..8c8710c --- /dev/null +++ b/upb/pb/decoder.int.h @@ -0,0 +1,242 @@ + +#ifndef UPB_DECODER_INT_H_ +#define UPB_DECODER_INT_H_ + +#include <stdlib.h> +#include "upb/def.h" +#include "upb/handlers.h" +#include "upb/sink.h" +#include "upb/pb/decoder.h" + +// Opcode definitions. The canonical meaning of each opcode is its +// implementation in the interpreter (the JIT is written to match this). +// +// All instructions have the opcode in the low byte. +// Instruction format for most instructions is: +// +// +-------------------+--------+ +// | arg (24) | op (8) | +// +-------------------+--------+ +// +// Exceptions are indicated below. A few opcodes are multi-word. +typedef enum { + // Opcodes 1-8, 13, 15-18 parse their respective descriptor types. + // Arg for all of these is the upb selector for this field. +#define T(type) OP_PARSE_ ## type = UPB_DESCRIPTOR_TYPE_ ## type + T(DOUBLE), T(FLOAT), T(INT64), T(UINT64), T(INT32), T(FIXED64), T(FIXED32), + T(BOOL), T(UINT32), T(SFIXED32), T(SFIXED64), T(SINT32), T(SINT64), +#undef T + OP_STARTMSG = 9, // No arg. + OP_ENDMSG = 10, // No arg. + OP_STARTSEQ = 11, + OP_ENDSEQ = 12, + OP_STARTSUBMSG = 14, + OP_ENDSUBMSG = 19, + OP_STARTSTR = 20, + OP_STRING = 21, + OP_ENDSTR = 22, + + OP_PUSHTAGDELIM = 23, // No arg. + OP_PUSHLENDELIM = 24, // No arg. + OP_POP = 25, // No arg. + OP_SETDELIM = 26, // No arg. + OP_SETGROUPNUM = 27, + OP_SETBIGGROUPNUM = 28, // two words: | unused (24) | opc || groupnum (32) | + + // The arg for these opcodes is a local label reference. + OP_CHECKDELIM = 29, + OP_CALL = 30, + OP_BRANCH = 31, + + // Different opcodes depending on how many bytes expected. + OP_TAG1 = 32, // | expected tag (16) | jump target (8) | opc (8) | + OP_TAG2 = 33, // | expected tag (16) | jump target (8) | opc (8) | + OP_TAGN = 34, // three words: + // | unused (16) | jump target(8) | opc (8) | + // | expected tag 1 (32) | + // | expected tag 2 (32) | + + OP_SETDISPATCH = 35, // N words: + // | unused (24) | opc | + // | upb_inttable* (32 or 64) | + + OP_HALT = 36, // No arg. +} opcode; + +#define OP_MAX OP_HALT + +UPB_INLINE opcode getop(uint32_t instr) { return instr & 0xff; } + +const upb_frametype upb_pbdecoder_frametype; + +// Decoder entry points; used as handlers. +void *upb_pbdecoder_start(void *closure, const void *handler_data, + size_t size_hint); +size_t upb_pbdecoder_decode(void *closure, const void *hd, const char *buf, + size_t size); +bool upb_pbdecoder_end(void *closure, const void *handler_data); + +// Decoder-internal functions that the JIT calls to handle fallback paths. +void *upb_pbdecoder_resume(upb_pbdecoder *d, void *p, const char *buf, + size_t size); +size_t upb_pbdecoder_suspend(upb_pbdecoder *d); +int32_t upb_pbdecoder_skipunknown(upb_pbdecoder *d, uint32_t fieldnum, + uint8_t wire_type); +int32_t upb_pbdecoder_checktag_slow(upb_pbdecoder *d, uint64_t expected); +int32_t upb_pbdecoder_decode_varint_slow(upb_pbdecoder *d, uint64_t *u64); +int32_t upb_pbdecoder_decode_f32(upb_pbdecoder *d, uint32_t *u32); +int32_t upb_pbdecoder_decode_f64(upb_pbdecoder *d, uint64_t *u64); +void upb_pbdecoder_seterr(upb_pbdecoder *d, const char *msg); + +// Error messages that are shared between the bytecode and JIT decoders. +extern const char *kPbDecoderStackOverflow; + +typedef struct _upb_pbdecoderplan upb_pbdecoderplan; + +// Access to decoderplan members needed by the decoder. +bool upb_pbdecoderplan_hasjitcode(const upb_pbdecoderplan *p); +uint32_t *upb_pbdecoderplan_codebase(const upb_pbdecoderplan *p); +const char *upb_pbdecoder_getopname(unsigned int op); +upb_string_handler *upb_pbdecoderplan_jitcode(const upb_pbdecoderplan *p); + +// JIT entry point. +void upb_pbdecoder_jit(upb_pbdecoderplan *plan); +void upb_pbdecoder_freejit(upb_pbdecoderplan *plan); + + +// A special label that means "do field dispatch for this message and branch to +// wherever that takes you." +#define LABEL_DISPATCH 0 + +#define DECODE_OK -1 +#define DECODE_MISMATCH -2 // Used only from checktag_slow(). +#define DECODE_ENDGROUP -2 // Used only from checkunknown(). + +typedef struct { + // The absolute stream offset of the end-of-frame delimiter. + // Non-delimited frames (groups and non-packed repeated fields) reuse the + // delimiter of their parent, even though the frame may not end there. + // + // NOTE: the JIT stores a slightly different value here for non-top frames. + // It stores the value relative to the end of the enclosed message. But the + // innermost frame is still stored the same way, which is important for + // ensuring that calls from the JIT into C work correctly. + uint64_t end_ofs; + uint32_t *base; + uint32_t groupnum; + union { + upb_inttable *dispatch; // Not used by the JIT. + void *closure; // Only used by the JIT. + } u; +} upb_pbdecoder_frame; + +struct upb_pbdecoder { + // Where we push parsed data (not owned). + upb_sink *sink; + + size_t call_len; + uint32_t *pc, *last; + + // Current input buffer and its stream offset. + const char *buf, *ptr, *end, *checkpoint; + + // End of the delimited region, relative to ptr, or NULL if not in this buf. + const char *delim_end; + + // End of the delimited region, relative to ptr, or end if not in this buf. + const char *data_end; + + // Overall stream offset of "buf." + uint64_t bufstart_ofs; + + // How many bytes past the end of the user buffer we want to skip. + size_t skip; + + // Buffer for residual bytes not parsed from the previous buffer. + // The maximum number of residual bytes we require is 12; a five-byte + // unknown tag plus an eight-byte value, less one because the value + // is only a partial value. + char residual[12]; + char *residual_end; + + // Stores the user buffer passed to our decode function. + const char *buf_param; + size_t size_param; + +#ifdef UPB_USE_JIT_X64 + // Used momentarily by the generated code to store a value while a user + // function is called. + uint32_t tmp_len; + + const void *saved_rsp; +#endif + + upb_status *status; + + // Our internal stack. + upb_pbdecoder_frame *top, *limit; + upb_pbdecoder_frame stack[UPB_DECODER_MAX_NESTING]; + uint32_t *callstack[UPB_DECODER_MAX_NESTING * 2]; +}; + +// Data pertaining to a single decoding method/function. +// Each method contains code to parse a single message type. +// If may or may not be bound to a destination handlers object. +typedef struct { + // While compiling, the base is relative in "ofs", after compiling it is + // absolute in "ptr". + union { + uint32_t ofs; // PC offset of method. + const void *ptr; // Pointer to bytecode or machine code for this method. + } base; + + // Whether this method is native code or bytecode. + bool native_code; + + // The message type that this method is parsing. + const upb_msgdef *msg; + + // The destination handlers this method is bound to, or NULL if this method + // can be bound to a destination handlers instance at runtime. + // + // If non-NULL, we own a ref. + const upb_handlers *dest_handlers; + + // The dispatch table layout is: + // [field number] -> [ 48-bit offset ][ 8-bit wt2 ][ 8-bit wt1 ] + // + // If wt1 matches, jump to the 48-bit offset. If wt2 matches, lookup + // (UPB_MAX_FIELDNUMBER + fieldnum) and jump there. + // + // We need two wire types because of packed/non-packed compatibility. A + // primitive repeated field can use either wire type and be valid. While we + // could key the table on fieldnum+wiretype, the table would be 8x sparser. + // + // Storing two wire types in the primary value allows us to quickly rule out + // the second wire type without needing to do a separate lookup (this case is + // less common than an unknown field). + upb_inttable dispatch; +} upb_pbdecodermethod; + +struct _upb_pbdecoderplan { + // Pointer to bytecode. + uint32_t *code, *code_end; + + // Maps upb_msgdef*/upb_handlers* -> upb_pbdecodermethod + upb_inttable methods; + + // The method that starts parsing when we first call into the plan. + // Ideally we will remove the idea that any of the methods in the plan + // are special like this, so that any method can be the top-level one. + upb_pbdecodermethod *topmethod; + +#ifdef UPB_USE_JIT_X64 + // JIT-generated machine code (else NULL). + upb_string_handler *jit_code; + size_t jit_size; + char *debug_info; + void *dl; +#endif +}; + +#endif // UPB_DECODER_INT_H_ diff --git a/upb/pb/decoder_x64.dasc b/upb/pb/decoder_x64.dasc deleted file mode 100644 index dee063a..0000000 --- a/upb/pb/decoder_x64.dasc +++ /dev/null @@ -1,1086 +0,0 @@ -|// -|// upb - a minimalist implementation of protocol buffers. -|// -|// Copyright (c) 2011 Google Inc. See LICENSE for details. -|// Author: Josh Haberman <jhaberman@gmail.com> -|// -|// JIT compiler for upb_pbdecoder on x86. Given a decoderplan object (which -|// contains an embedded set of upb_handlers), generates code specialized to -|// parsing the specific message and calling specific handlers. -|// -|// Since the JIT can call other functions (the JIT'ted code is not a leaf -|// function) we must respect alignment rules. All x86-64 systems require -|// 16-byte stack alignment. - -#define _GNU_SOURCE -#include <stdio.h> -#include <sys/mman.h> -#include "dynasm/dasm_x86.h" -#include "upb/shim/shim.h" - -#ifndef MAP_ANONYMOUS -# define MAP_ANONYMOUS MAP_ANON -#endif - -// We map into the low 32 bits when we can, but if this is not available -// (like on OS X) we take what we can get. It's not required for correctness, -// it's just a performance thing that makes it more likely that our jumps -// can be rel32 (i.e. within 32-bits of our pc) instead of the longer -// sequence required for other jumps (see callp). -#ifndef MAP_32BIT -#define MAP_32BIT 0 -#endif - -// These are used to track jump targets for messages and fields. -enum { - STARTMSG = 0, - AFTER_STARTMSG = 1, - ENDOFBUF = 2, - ENDOFMSG = 3, - DYNDISPATCH = 4, - TOTAL_MSG_PCLABELS = 5, -}; - -enum { - FIELD = 0, - FIELD_NO_TYPECHECK = 1, - TOTAL_FIELD_PCLABELS = 2, -}; - -typedef struct { - uint32_t max_field_number; - // Currently keyed on field number. Could also try keying it - // on encoded or decoded tag, or on encoded field number. - void **tablearray; - // Pointer to the JIT code for parsing this message. - void *jit_func; -} upb_jitmsginfo; - -static uint32_t upb_getpclabel(decoderplan *plan, const void *obj, int n) { - upb_value v; - bool found = upb_inttable_lookupptr(&plan->pclabels, obj, &v); - UPB_ASSERT_VAR(found, found); - return upb_value_getuint32(v) + n; -} - -static upb_jitmsginfo *upb_getmsginfo(const decoderplan *plan, - const upb_handlers *h) { - upb_value v; - bool found = upb_inttable_lookupptr(&plan->msginfo, h, &v); - UPB_ASSERT_VAR(found, found); - return upb_value_getptr(v); -} - -// To debug JIT-ted code with GDB we need to tell GDB about the JIT-ted code -// at runtime. GDB 7.x+ has defined an interface for doing this, and these -// structure/function defintions are copied out of gdb/jit.h -// -// We need to give GDB an ELF file at runtime describing the symbols we have -// generated. To avoid implementing the ELF format, we generate an ELF file -// at compile-time and compile it in as a character string. We can replace -// a few key constants (address of JIT-ted function and its size) by looking -// for a few magic numbers and doing a dumb string replacement. - -#ifndef __APPLE__ -const unsigned char upb_jit_debug_elf_file[] = { -#include "upb/pb/jit_debug_elf_file.h" -}; - -typedef enum -{ - GDB_JIT_NOACTION = 0, - GDB_JIT_REGISTER, - GDB_JIT_UNREGISTER -} jit_actions_t; - -typedef struct gdb_jit_entry { - struct gdb_jit_entry *next_entry; - struct gdb_jit_entry *prev_entry; - const char *symfile_addr; - uint64_t symfile_size; -} gdb_jit_entry; - -typedef struct { - uint32_t version; - uint32_t action_flag; - gdb_jit_entry *relevant_entry; - gdb_jit_entry *first_entry; -} gdb_jit_descriptor; - -gdb_jit_descriptor __jit_debug_descriptor = {1, GDB_JIT_NOACTION, NULL, NULL}; - -void __attribute__((noinline)) __jit_debug_register_code() { - __asm__ __volatile__(""); -} - -void upb_reg_jit_gdb(decoderplan *plan) { - // Create debug info. - size_t elf_len = sizeof(upb_jit_debug_elf_file); - plan->debug_info = malloc(elf_len); - memcpy(plan->debug_info, upb_jit_debug_elf_file, elf_len); - uint64_t *p = (void*)plan->debug_info; - for (; (void*)(p+1) <= (void*)plan->debug_info + elf_len; ++p) { - if (*p == 0x12345678) { *p = (uintptr_t)plan->jit_code; } - if (*p == 0x321) { *p = plan->jit_size; } - } - - // Register the JIT-ted code with GDB. - gdb_jit_entry *e = malloc(sizeof(gdb_jit_entry)); - e->next_entry = __jit_debug_descriptor.first_entry; - e->prev_entry = NULL; - if (e->next_entry) e->next_entry->prev_entry = e; - e->symfile_addr = plan->debug_info; - e->symfile_size = elf_len; - __jit_debug_descriptor.first_entry = e; - __jit_debug_descriptor.relevant_entry = e; - __jit_debug_descriptor.action_flag = GDB_JIT_REGISTER; - __jit_debug_register_code(); -} - -#else - -void upb_reg_jit_gdb(decoderplan *plan) { - (void)plan; -} - -#endif - -// Has to be a separate function, otherwise GCC will complain about -// expressions like (&foo != NULL) because they will never evaluate -// to false. -static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; } - -|.arch x64 -|.actionlist upb_jit_actionlist -|.globals UPB_JIT_GLOBAL_ -|.globalnames upb_jit_globalnames -| -|// Calling conventions. Note -- this will need to be changed for -|// Windows, which uses a different calling convention! -|.define ARG1_64, rdi -|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi. -|.define ARG2_32, esi -|.define ARG2_64, rsi -|.define ARG3_32, edx -|.define ARG3_64, rdx -|.define ARG4_32, ecx -|.define ARG4_64, rcx -|.define XMMARG1, xmm0 - -| -|// Register allocation / type map. -|// ALL of the code in this file uses these register allocations. -|// When we "call" within this file, we do not use regular calling -|// conventions, but of course when calling to user callbacks we must. -|.define PTR, rbx // Writing this to DECODER->ptr commits our progress. -|.define CLOSURE, r12 -|.type SINKFRAME, upb_sinkframe, r13 -|.type FRAME, frame, r14 -|.type DECODER, upb_pbdecoder, r15 -|.type SINK, upb_sink -| -|.macro callp, addr -|| upb_assert_notnull(addr); -|// TODO(haberman): fix this. I believe the predicate we should actually be -|// testing is whether the jump distance is greater than INT32_MAX, not the -|// absolute address of the target. -|| if ((uintptr_t)addr < 0xffffffff) { - | call &addr -|| } else { - | mov64 rax, (uintptr_t)addr - | call rax -|| } -|.endmacro -| -|.macro loadarg2, val -||{ -|| uintptr_t data = (uintptr_t)val; -|| if (data > 0xffffffff) { -| mov64 ARG2_64, data -|| } else if (data) { -| mov ARG2_32, data -|| } else { -| xor ARG2_32, ARG2_32 -|| } -|| } -|.endmacro -| -|.macro load_handler_data, h, f, type -| loadarg2 gethandlerdata(h, f, type) -|.endmacro -| -|// Checkpoints our progress by writing PTR to DECODER, and -|// checks for end-of-buffer. -|.macro checkpoint, h -| mov DECODER->ptr, PTR -| cmp PTR, DECODER->effective_end -| jae =>upb_getpclabel(plan, h, ENDOFBUF) -|.endmacro -| -|.macro check_bool_ret -| test al, al -| jz ->exit_jit -|.endmacro -| -|.macro check_ptr_ret -| test rax, rax -| jz ->exit_jit -|.endmacro -| -|// Decodes varint into ARG2. -|// Inputs: -|// - ecx: first 4 bytes of varint -|// - offset: offset from PTR where varint begins -|// Outputs: -|// - ARG2: contains decoded varint -|// - rax: new PTR -|.macro decode_loaded_varint, offset -| // Check for <=2 bytes inline, otherwise jump to 2-10 byte decoder. -| lea rax, [PTR + offset + 1] -| mov ARG2_32, ecx -| and ARG2_32, 0x7f -| test cl, cl -| jns >9 -| lea rax, [PTR + offset + 2] -| movzx edx, ch -| and edx, 0x7f -| shl edx, 7 -| or ARG2_32, edx -| test cx, cx -| jns >9 -| mov ARG1_64, rax -|// XXX: I don't think this handles 64-bit values correctly. -|// Test with UINT64_MAX -| callp upb_vdecode_max8_fast -|// rax return from function will contain new pointer -| mov ARG2_64, rdx -| check_ptr_ret // Check for unterminated, >10-byte varint. -|9: -|.endmacro -| -|.macro decode_varint, offset -| mov ecx, dword [PTR + offset] -| decode_loaded_varint offset -| mov PTR, rax -|.endmacro -| -|// Table-based field dispatch. -|// Inputs: -|// - ecx: first 4 bytes of tag -|// Outputs: -|// - edx: field number -|// - esi: wire type -|// Could specialize this by avoiding the value masking: could just key the -|// table on the raw (length-masked) varint to save 3-4 cycles of latency. -|// Currently only support tables where all entries are in the array part. -|.macro dyndispatch_, h -|| asmlabel(plan, "_UPB_MCODE_DISPATCH_%s.%d", -|| upb_msgdef_fullname(upb_handlers_msgdef(h)), rand()); -|=>upb_getpclabel(plan, h, DYNDISPATCH): -| decode_loaded_varint, 0 -| mov ecx, esi -| shr ecx, 3 -| and esi, 0x7 // Note: this value is used in the FIELD pclabel below. -| cmp esi, UPB_WIRE_TYPE_END_GROUP -| je >1 -|| upb_jitmsginfo *mi = upb_getmsginfo(plan, h); -| cmp ecx, mi->max_field_number // Bounds-check the field. -| ja ->exit_jit // In the future; could be unknown label -|| if ((uintptr_t)mi->tablearray < 0xffffffff) { -| // TODO: support hybrid array/hash tables. -| mov rax, qword [rcx*8 + mi->tablearray] -|| } else { -| mov64 rax, (uintptr_t)mi->tablearray -| mov rax, qword [rax + rcx*8] -|| } -| jmp rax // Dispatch: unpredictable jump. -|1: -|// End group. -| cmp ecx, FRAME->group_fieldnum -| jne ->exit_jit // Unexpected END_GROUP tag. -| mov PTR, rax // rax came from decode_loaded_varint -| mov DECODER->ptr, PTR -| jmp =>upb_getpclabel(plan, h, ENDOFMSG) -|.endmacro -| -|.if 1 -| // Replicated dispatch: larger code, but better branch prediction. -| .define dyndispatch, dyndispatch_ -|.else -| // Single dispatch: smaller code, could be faster because of reduced -| // icache usage. We keep this around to allow for easy comparison between -| // the two. -| .macro dyndispatch, h -| jmp =>upb_getpclabel(plan, h, DYNDISPATCH) -| .endmacro -|.endif -| -|.macro pushsinkframe, handlers, field, endtype -| mov rax, DECODER->sink -| mov dword SINKFRAME->selector, getselector(field, endtype) -| lea rcx, [SINKFRAME + sizeof(upb_sinkframe)] // rcx for short addressing -| cmp rcx, SINK:rax->limit -| jae ->exit_jit // Frame stack overflow. -| mov64 r9, (uintptr_t)handlers -| mov SINKFRAME:rcx->h, r9 -| mov SINKFRAME:rcx->closure, CLOSURE -| mov SINK:rax->top, rcx -| mov SINKFRAME, rcx -|.endmacro -| -|.macro popsinkframe -| sub SINKFRAME, sizeof(upb_sinkframe) -| mov rax, DECODER->sink -| mov SINK:rax->top, SINKFRAME -| mov CLOSURE, SINKFRAME->closure -|.endmacro -| -|// Push a stack frame (not the CPU stack, the upb_pbdecoder stack). -|.macro pushframe, handlers, field, end_offset_, endtype -|// Decoder Frame. -| lea rax, [FRAME + sizeof(frame)] // rax for short addressing -| cmp rax, DECODER->limit -| jae ->exit_jit // Frame stack overflow. -| mov64 r10, (uintptr_t)field -| mov FRAME:rax->f, r10 -| mov qword FRAME:rax->end_ofs, end_offset_ -| mov byte FRAME:rax->is_sequence, (endtype == UPB_HANDLER_ENDSEQ) -| mov byte FRAME:rax->is_packed, 0 -|| if (upb_fielddef_istagdelim(field) && endtype == UPB_HANDLER_ENDSUBMSG) { -| mov dword FRAME:rax->group_fieldnum, upb_fielddef_number(field) -|| } else { -| mov dword FRAME:rax->group_fieldnum, 0xffffffff -|| } -| mov DECODER->top, rax -| mov FRAME, rax -| pushsinkframe handlers, field, endtype -|.endmacro -| -|.macro popframe -| sub FRAME, sizeof(frame) -| mov DECODER->top, FRAME -| popsinkframe -| setmsgend -|.endmacro -| -|.macro setmsgend -| mov rsi, DECODER->jit_end -| mov rax, qword FRAME->end_ofs // Will be UINT64_MAX for groups. -| sub rax, qword DECODER->bufstart_ofs -| add rax, qword DECODER->buf // rax = d->buf + f->end_ofs - d->bufstart_ofs -| jc >8 // If the addition overflowed, use jit_end -| cmp rax, rsi -| ja >8 // If jit_end is less, use jit_end -| mov rsi, rax // Use frame end. -|8: -| mov DECODER->effective_end, rsi -|.endmacro -| -|// rcx contains the tag, compare it against "tag", but since it is a varint -|// we must only compare as many bytes as actually have data. -|.macro checktag, tag -|| switch (upb_value_size(tag)) { -|| case 1: -| cmp cl, tag -|| break; -|| case 2: -| cmp cx, tag -|| break; -|| case 3: -| and ecx, 0xffffff // 3 bytes -| cmp rcx, tag -|| case 4: -| cmp ecx, tag -|| break; -|| case 5: -| mov64 rdx, 0xffffffffff // 5 bytes -| and rcx, rdx -| cmp rcx, tag -|| break; -|| default: abort(); -|| } -|.endmacro -| -|.macro sethas, reg, hasbit -|| if (hasbit >= 0) { -| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) -|| } -|.endmacro - - -#include <stdlib.h> -#include "upb/pb/varint.h" - -static upb_func *gethandler(const upb_handlers *h, const upb_fielddef *f, - upb_handlertype_t type) { - return upb_handlers_gethandler(h, getselector(f, type)); -} - -static uintptr_t gethandlerdata(const upb_handlers *h, const upb_fielddef *f, - upb_handlertype_t type) { - return (uintptr_t)upb_handlers_gethandlerdata(h, getselector(f, type)); -} - -static void asmlabel(decoderplan *plan, const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - char *str = NULL; - size_t size = 0; - upb_vrprintf(&str, &size, 0, fmt, ap); - va_end(ap); - uint32_t label = plan->pclabel_count++; - dasm_growpc(plan, plan->pclabel_count); - |=>label: - upb_inttable_insert(&plan->asmlabels, label, upb_value_ptr(str)); -} - -// Decodes the next val into ARG2, advances PTR. -static void upb_decoderplan_jit_decodefield(decoderplan *plan, - size_t tag_size, - const upb_handlers *h, - const upb_fielddef *f) { - // Decode the value into arg 3 for the callback. - asmlabel(plan, "UPB_MCODE_DECODE_FIELD_%s.%s", - upb_msgdef_fullname(upb_handlers_msgdef(h)), - upb_fielddef_name(f)); - switch (upb_fielddef_descriptortype(f)) { - case UPB_DESCRIPTOR_TYPE_DOUBLE: - | movsd XMMARG1, qword [PTR + tag_size] - | add PTR, 8 + tag_size - break; - - case UPB_DESCRIPTOR_TYPE_FIXED64: - case UPB_DESCRIPTOR_TYPE_SFIXED64: - | mov ARG2_64, qword [PTR + tag_size] - | add PTR, 8 + tag_size - break; - - case UPB_DESCRIPTOR_TYPE_FLOAT: - | movss XMMARG1, dword [PTR + tag_size] - | add PTR, 4 + tag_size - break; - - case UPB_DESCRIPTOR_TYPE_FIXED32: - case UPB_DESCRIPTOR_TYPE_SFIXED32: - | mov ARG2_32, dword [PTR + tag_size] - | add PTR, 4 + tag_size - break; - - case UPB_DESCRIPTOR_TYPE_BOOL: - // Can't assume it's one byte long, because bool must be wire-compatible - // with all of the varint integer types. - | decode_varint tag_size - | test ARG2_64, ARG2_64 - | setne al - | movzx ARG2_32, al - break; - - case UPB_DESCRIPTOR_TYPE_INT64: - case UPB_DESCRIPTOR_TYPE_UINT64: - case UPB_DESCRIPTOR_TYPE_INT32: - case UPB_DESCRIPTOR_TYPE_UINT32: - case UPB_DESCRIPTOR_TYPE_ENUM: - | decode_varint tag_size - break; - - case UPB_DESCRIPTOR_TYPE_SINT64: - // 64-bit zig-zag decoding. - | decode_varint tag_size - | mov rax, ARG2_64 - | shr ARG2_64, 1 - | and rax, 1 - | neg rax - | xor ARG2_64, rax - break; - - case UPB_DESCRIPTOR_TYPE_SINT32: - // 32-bit zig-zag decoding. - | decode_varint tag_size - | mov eax, ARG2_32 - | shr ARG2_32, 1 - | and eax, 1 - | neg eax - | xor ARG2_32, eax - break; - - case UPB_DESCRIPTOR_TYPE_STRING: - case UPB_DESCRIPTOR_TYPE_BYTES: { - // We only handle the case where the entire string is in our current - // buf, which sidesteps any security problems. The C path has more - // robust checks. - | mov ecx, dword [PTR + tag_size] - | decode_loaded_varint tag_size - | mov rdi, DECODER->end - | sub rdi, rax - | cmp ARG2_64, rdi // if (len > d->end - str) - | ja ->exit_jit // Can't deliver, whole string not in buf. - | mov PTR, rax - - upb_func *handler = gethandler(h, f, UPB_HANDLER_STARTSTR); - if (handler) { - // void* startstr(void *c, const void *hd, size_t hint) - | mov DECODER->tmp_len, ARG2_32 - | mov ARG1_64, CLOSURE - | mov ARG3_64, ARG2_64 - | load_handler_data h, f, UPB_HANDLER_STARTSTR - | callp handler - | check_ptr_ret - | mov ARG1_64, rax // sub-closure - | mov ARG4_32, DECODER->tmp_len - } else { - | mov ARG1_64, CLOSURE - | mov ARG4_64, ARG2_64 - } - - handler = gethandler(h, f, UPB_HANDLER_STRING); - if (handler) { - // size_t str(void *c, const void *hd, const char *buf, size_t len) - | load_handler_data h, f, UPB_HANDLER_STRING - | mov ARG3_64, PTR - | callp handler - // TODO: properly handle returns other than "n" (the whole string). - | add PTR, rax - } else { - | add PTR, ARG4_64 - } - - handler = gethandler(h, f, UPB_HANDLER_ENDSTR); - if (handler) { - // bool endstr(const upb_sinkframe *frame); - | mov ARG1_64, CLOSURE - | load_handler_data h, f, UPB_HANDLER_ENDSTR - | callp handler - | check_bool_ret - } - break; - } - - // Will dispatch callbacks and call submessage in a second. - case UPB_DESCRIPTOR_TYPE_MESSAGE: - | decode_varint tag_size - break; - case UPB_DESCRIPTOR_TYPE_GROUP: - | add PTR, tag_size - break; - - default: abort(); - } -} - -static void upb_decoderplan_jit_callcb(decoderplan *plan, - const upb_handlers *h, - const upb_fielddef *f) { - // Call callbacks. Specializing the append accessors didn't yield a speed - // increase in benchmarks. - asmlabel(plan, "UPB_MCODE_CALLCB_%s.%s", - upb_msgdef_fullname(upb_handlers_msgdef(h)), - upb_fielddef_name(f)); - if (upb_fielddef_issubmsg(f)) { - // Call startsubmsg handler (if any). - upb_func *startsubmsg = gethandler(h, f, UPB_HANDLER_STARTSUBMSG); - if (startsubmsg) { - // upb_sflow_t startsubmsg(const upb_sinkframe *frame) - | mov DECODER->tmp_len, ARG2_32 - | mov ARG1_64, CLOSURE - | load_handler_data h, f, UPB_HANDLER_STARTSUBMSG - | callp startsubmsg - | check_ptr_ret - | mov CLOSURE, rax - } - - const upb_handlers *sub_h = upb_handlers_getsubhandlers(h, f); - if (sub_h) { - if (upb_fielddef_istagdelim(f)) { - | mov rdx, UPB_NONDELIMITED - } else { - | mov esi, DECODER->tmp_len - | mov rdx, PTR - | sub rdx, DECODER->buf - | add rdx, DECODER->bufstart_ofs - | add rdx, rsi // = d->bufstart_ofs + (d->ptr - d->buf) + delim_len - } - | pushframe sub_h, f, rdx, UPB_HANDLER_ENDSUBMSG - | call =>upb_getpclabel(plan, sub_h, STARTMSG) - | popframe - } else { - if (upb_fielddef_istagdelim(f)) { - // Groups with no handlers not supported yet. - assert(false); - } else { - | mov esi, DECODER->tmp_len - | add PTR, rsi - } - } - - // Call endsubmsg handler (if any). - upb_func *endsubmsg = gethandler(h, f, UPB_HANDLER_ENDSUBMSG); - if (endsubmsg) { - // upb_flow_t endsubmsg(void *closure, upb_value fval); - | mov ARG1_64, CLOSURE - | load_handler_data h, f, UPB_HANDLER_ENDSUBMSG - | callp endsubmsg - | check_bool_ret - } - } else if (!upb_fielddef_isstring(f)) { - upb_handlertype_t handlertype = upb_handlers_getprimitivehandlertype(f); - upb_selector_t sel = getselector(f, handlertype); - upb_func *handler = gethandler(h, f, handlertype); - const upb_shim_data *data = upb_shim_getdata(h, sel); - if (data) { - switch (upb_fielddef_type(f)) { - case UPB_TYPE_INT64: - case UPB_TYPE_UINT64: - | mov [CLOSURE + data->offset], ARG2_64 - break; - case UPB_TYPE_INT32: - case UPB_TYPE_UINT32: - case UPB_TYPE_ENUM: - | mov [CLOSURE + data->offset], ARG2_32 - break; - case UPB_TYPE_DOUBLE: - | movsd qword [CLOSURE + data->offset], XMMARG1 - break; - case UPB_TYPE_FLOAT: - | movss dword [CLOSURE + data->offset], XMMARG1 - break; - case UPB_TYPE_BOOL: - | mov [CLOSURE + data->offset], ARG2_8 - break; - case UPB_TYPE_STRING: - case UPB_TYPE_BYTES: - case UPB_TYPE_MESSAGE: - assert(false); break; - } - | sethas CLOSURE, data->hasbit - } else if (handler) { - // bool value(const upb_sinkframe* frame, ctype val) - | mov ARG1_64, CLOSURE - | mov ARG3_64, ARG2_64 - | load_handler_data h, f, handlertype - | callp handler - | check_bool_ret - } - } -} - -static uint64_t upb_get_encoded_tag(const upb_fielddef *f) { - uint32_t tag = (upb_fielddef_number(f) << 3) | - upb_decoder_types[upb_fielddef_descriptortype(f)].native_wire_type; - uint64_t encoded_tag = upb_vencode32(tag); - // No tag should be greater than 5 bytes. - assert(encoded_tag <= 0xffffffffff); - return encoded_tag; -} - -static void upb_decoderplan_jit_endseq(decoderplan *plan, - const upb_handlers *h, - const upb_fielddef *f) { - | popframe - upb_func *endseq = gethandler(h, f, UPB_HANDLER_ENDSEQ); - if (endseq) { - | mov ARG1_64, CLOSURE - | load_handler_data h, f, UPB_HANDLER_ENDSEQ - | callp endseq - } -} - -// PTR should point to the beginning of the tag. -static void upb_decoderplan_jit_field(decoderplan *plan, - const upb_handlers *h, - const upb_fielddef *f, - const upb_fielddef *next_f) { - asmlabel(plan, "UPB_MCODE_FIELD_%s.%s", - upb_msgdef_fullname(upb_handlers_msgdef(h)), - upb_fielddef_name(f)); - uint64_t tag = upb_get_encoded_tag(f); - uint64_t next_tag = next_f ? upb_get_encoded_tag(next_f) : 0; - int tag_size = upb_value_size(tag); - - // PC-label for the dispatch table. - // We check the wire type (which must be loaded in edi) because the - // table is keyed on field number, not type. - |=>upb_getpclabel(plan, f, FIELD): - | cmp esi, (tag & 0x7) - | jne ->exit_jit // In the future: could be an unknown field or packed. - |=>upb_getpclabel(plan, f, FIELD_NO_TYPECHECK): - if (upb_fielddef_isseq(f)) { - upb_func *startseq = gethandler(h, f, UPB_HANDLER_STARTSEQ); - if (startseq) { - | mov ARG1_64, CLOSURE - | load_handler_data h, f, UPB_HANDLER_STARTSEQ - | callp startseq - | check_ptr_ret - | mov CLOSURE, rax - } - | mov rsi, FRAME->end_ofs - | pushframe h, f, rsi, UPB_HANDLER_ENDSEQ - } - - |1: // Label for repeating this field. - - upb_decoderplan_jit_decodefield(plan, tag_size, h, f); - upb_decoderplan_jit_callcb(plan, h, f); - - // This is kind of gross; future redesign should take into account how to - // make this work nicely. The difficult part is that the sequence can be - // broken either by end-of-message or by seeing a different field; in both - // cases we need to call the endseq handler, but what we do after that - // depends on which case triggered the end-of-sequence. - | mov DECODER->ptr, PTR - | cmp PTR, DECODER->jit_end - | jae ->exit_jit - | cmp PTR, DECODER->effective_end - | jb >2 - if (upb_fielddef_isseq(f)) { - upb_decoderplan_jit_endseq(plan, h, f); - } - | jmp =>upb_getpclabel(plan, h, ENDOFMSG) - |2: - | mov rcx, qword [PTR] - if (upb_fielddef_isseq(f)) { - | checktag tag - | je <1 - upb_decoderplan_jit_endseq(plan, h, f); - // Load next tag again (popframe/endseq clobbered it). - | mov rcx, qword [PTR] - } - - if (next_tag != 0) { - | checktag next_tag - | je =>upb_getpclabel(plan, next_f, FIELD_NO_TYPECHECK) - } - - // Fall back to dynamic dispatch. - | dyndispatch h -} - -static int upb_compare_uint32(const void *a, const void *b) { - return *(uint32_t*)a - *(uint32_t*)b; -} - -static void upb_decoderplan_jit_msg(decoderplan *plan, - const upb_handlers *h) { - asmlabel(plan, "UPB_MCODE_DECODEMSG_%s", - upb_msgdef_fullname(upb_handlers_msgdef(h))); - |=>upb_getpclabel(plan, h, AFTER_STARTMSG): - | push rbp - | mov rbp, rsp - | jmp >1 - - |=>upb_getpclabel(plan, h, STARTMSG): - | push rbp - | mov rbp, rsp - - // Call startmsg handler (if any): - upb_func *startmsg = upb_handlers_gethandler(h, UPB_STARTMSG_SELECTOR); - if (startmsg) { - // upb_flow_t startmsg(void *closure, const void *hd); - | mov ARG1_64, CLOSURE - | loadarg2 upb_handlers_gethandlerdata(h, UPB_STARTMSG_SELECTOR) - | callp startmsg - | check_bool_ret - } - - |1: - | setmsgend - | checkpoint h - | mov ecx, dword [PTR] - | dyndispatch_ h - - // --------- New code section (does not fall through) ------------------------ - - // Emit code for parsing each field (dynamic dispatch contains pointers to - // all of these). - - // Create an ordering over the fields in field number order. - // Parsing will theoretically be fastest if we emit code in the same - // order as field numbers are seen on-the-wire because of an optimization - // in the generated code that skips dynamic dispatch if the next field is - // as expected. - const upb_msgdef *md = upb_handlers_msgdef(h); - int num_keys = upb_msgdef_numfields(md); - uint32_t *keys = malloc(num_keys * sizeof(*keys)); - int idx = 0; - upb_msg_iter i; - for(upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) { - keys[idx++] = upb_fielddef_number(upb_msg_iter_field(&i)); - } - qsort(keys, num_keys, sizeof(uint32_t), &upb_compare_uint32); - - for(int i = 0; i < num_keys; i++) { - const upb_fielddef *f = upb_msgdef_itof(md, keys[i]); - const upb_fielddef *next_f = - (i + 1 < num_keys) ? upb_msgdef_itof(md, keys[i + 1]) : NULL; - upb_decoderplan_jit_field(plan, h, f, next_f); - } - - free(keys); - - // --------- New code section (does not fall through) ------------------------ - - // End-of-buf / end-of-message. - // We hit a buffer limit; either we hit jit_end or end-of-submessage. - |=>upb_getpclabel(plan, h, ENDOFBUF): - | cmp PTR, DECODER->jit_end - | jae ->exit_jit - - |=>upb_getpclabel(plan, h, ENDOFMSG): - // We are at end-of-submsg: call endmsg handler (if any): - upb_func *endmsg = upb_handlers_gethandler(h, UPB_ENDMSG_SELECTOR); - if (endmsg) { - // void endmsg(void *closure, const void *hd, upb_status *status) { - | mov ARG1_64, CLOSURE - | loadarg2 upb_handlers_gethandlerdata(h, UPB_ENDMSG_SELECTOR) - | mov ARG3_64, DECODER->sink - | mov ARG3_64, SINK:ARG3_64->pipeline_ - | add ARG3_64, offsetof(upb_pipeline, status_) - | callp endmsg - } - - | leave - | ret -} - -static void upb_decoderplan_jit(decoderplan *plan) { - // The JIT prologue/epilogue trampoline that is generated in this function - // does not depend on the handlers, so it will never vary. Ideally we would - // put it in an object file and just link it into upb so we could have only a - // single copy of it instead of one copy for each decoderplan. But our - // options for doing that are undesirable: GCC inline assembly is - // complicated, not portable to other compilers, and comes with subtle - // caveats about incorrect things what the optimizer might do if you eg. - // execute non-local jumps. Putting this code in a .s file would force us to - // calculate the structure offsets ourself instead of symbolically - // (ie. [r15 + 0xcd] instead of DECODER->ptr). So we tolerate a bit of - // unnecessary duplication/redundancy. - asmlabel(plan, "upb_jit_trampoline"); - | push rbp - | mov rbp, rsp - | push r15 - | push r14 - | push r13 - | push r12 - | push rbx - // Align stack. - | sub rsp, 8 - | mov DECODER, ARG1_64 - | mov DECODER->saved_rbp, rbp - | mov FRAME, DECODER:ARG1_64->top - | mov rax, DECODER:ARG1_64->sink - | mov SINKFRAME, SINK:rax->top - | mov CLOSURE, SINKFRAME->closure - | mov PTR, DECODER->ptr - - // TODO: push return addresses for re-entry (will be necessary for multiple - // buffer support). - | call ARG2_64 - asmlabel(plan, "exitjit"); - |->exit_jit: - | mov rbp, DECODER->saved_rbp - | lea rsp, [rbp - 48] - // Counter previous alignment. - | add rsp, 8 - | pop rbx - | pop r12 - | pop r13 - | pop r14 - | pop r15 - | leave - | ret - - upb_inttable_iter i; - upb_inttable_begin(&i, &plan->msginfo); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - const upb_handlers *h = (const upb_handlers*)upb_inttable_iter_key(&i); - upb_decoderplan_jit_msg(plan, h); - } -} - -static void upb_decoderplan_jit_assignpclabels(decoderplan *plan, - const upb_handlers *h) { - // Limit the DFS. - if (upb_inttable_lookupptr(&plan->pclabels, h, NULL)) return; - - upb_inttable_insertptr(&plan->pclabels, h, - upb_value_uint32(plan->pclabel_count)); - plan->pclabel_count += TOTAL_MSG_PCLABELS; - - upb_jitmsginfo *info = malloc(sizeof(*info)); - info->max_field_number = 0; - upb_inttable_insertptr(&plan->msginfo, h, upb_value_ptr(info)); - - upb_msg_iter i; - upb_msg_begin(&i, upb_handlers_msgdef(h)); - for(; !upb_msg_done(&i); upb_msg_next(&i)) { - const upb_fielddef *f = upb_msg_iter_field(&i); - info->max_field_number = - UPB_MAX(info->max_field_number, upb_fielddef_number(f)); - upb_inttable_insertptr(&plan->pclabels, f, - upb_value_uint32(plan->pclabel_count)); - plan->pclabel_count += TOTAL_FIELD_PCLABELS; - - // Discover the whole graph of handlers depth-first. We will probably - // revise this later to be more explicit about the list of handlers that - // the plan should include. - if (upb_fielddef_issubmsg(f)) { - const upb_handlers *subh = upb_handlers_getsubhandlers(h, f); - if (subh) upb_decoderplan_jit_assignpclabels(plan, subh); - } - } - // TODO: support large field numbers by either using a hash table or - // generating code for a binary search. For now large field numbers - // will just fall back to the table decoder. - info->max_field_number = UPB_MIN(info->max_field_number, 16000); - info->tablearray = malloc((info->max_field_number + 1) * sizeof(void*)); -} - -static void upb_decoderplan_makejit(decoderplan *plan) { - upb_inttable_init(&plan->msginfo, UPB_CTYPE_PTR); - plan->debug_info = NULL; - - // Assign pclabels. - plan->pclabel_count = 0; - upb_inttable_init(&plan->pclabels, UPB_CTYPE_UINT32); - upb_decoderplan_jit_assignpclabels(plan, plan->dest_handlers); - - upb_inttable_init(&plan->asmlabels, UPB_CTYPE_PTR); - - void **globals = malloc(UPB_JIT_GLOBAL__MAX * sizeof(*globals)); - dasm_init(plan, 1); - dasm_setupglobal(plan, globals, UPB_JIT_GLOBAL__MAX); - dasm_growpc(plan, plan->pclabel_count); - dasm_setup(plan, upb_jit_actionlist); - - upb_decoderplan_jit(plan); - - int dasm_status = dasm_link(plan, &plan->jit_size); - (void)dasm_status; - assert(dasm_status == DASM_S_OK); - - plan->jit_code = mmap(NULL, plan->jit_size, PROT_READ | PROT_WRITE, - MAP_32BIT | MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - - upb_reg_jit_gdb(plan); - - dasm_encode(plan, plan->jit_code); - - // Create dispatch tables. - upb_inttable_iter i; - upb_inttable_begin(&i, &plan->msginfo); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - const upb_handlers *h = (const upb_handlers*)upb_inttable_iter_key(&i); - upb_jitmsginfo *mi = upb_getmsginfo(plan, h); - // We jump to after the startmsg handler since it is called before entering - // the JIT (either by upb_pbdecoder or by a previous call to the JIT). - mi->jit_func = plan->jit_code + - dasm_getpclabel(plan, upb_getpclabel(plan, h, AFTER_STARTMSG)); - for (uint32_t j = 0; j <= mi->max_field_number; j++) { - const upb_fielddef *f = upb_msgdef_itof(upb_handlers_msgdef(h), j); - if (f) { - mi->tablearray[j] = plan->jit_code + - dasm_getpclabel(plan, upb_getpclabel(plan, f, FIELD)); - } else { - // TODO: extend the JIT to handle unknown fields. - // For the moment we exit the JIT for any unknown field. - mi->tablearray[j] = globals[UPB_JIT_GLOBAL_exit_jit]; - } - } - } - - upb_inttable_uninit(&plan->pclabels); - - mprotect(plan->jit_code, plan->jit_size, PROT_EXEC | PROT_READ); - -#ifndef NDEBUG - // Dump to a .o file in /tmp, for easy inspection. - - // Convert all asm labels from pclabel offsets to machine code offsets. - upb_inttable mclabels; - upb_inttable_init(&mclabels, UPB_CTYPE_PTR); - upb_inttable_begin(&i, &plan->asmlabels); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - upb_inttable_insert( - &mclabels, - dasm_getpclabel(plan, upb_inttable_iter_key(&i)), - upb_inttable_iter_value(&i)); - } - - FILE *f = fopen("/tmp/upb-jit-code.s", "w"); - if (f) { - fputs(" .text", f); - size_t linelen = 0; - for (size_t i = 0; i < plan->jit_size; i++) { - upb_value v; - if (upb_inttable_lookup(&mclabels, i, &v)) { - const char *label = upb_value_getptr(v); - fprintf(f, "\n\n_%s:\n", label); - fprintf(f, " .globl _%s", label); - linelen = 1000; - } - if (linelen >= 77) { - linelen = fprintf(f, "\n .byte %u", plan->jit_code[i]); - } else { - linelen += fprintf(f, ",%u", plan->jit_code[i]); - } - } - fputs("\n", f); - fclose(f); - } else { - fprintf(stderr, "Couldn't open /tmp/upb-jit-code.s for writing/\n"); - } - - upb_inttable_uninit(&mclabels); -#endif - - upb_inttable_begin(&i, &plan->asmlabels); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - free(upb_value_getptr(upb_inttable_iter_value(&i))); - } - upb_inttable_uninit(&plan->asmlabels); - - dasm_free(plan); - free(globals); -} - -static void upb_decoderplan_freejit(decoderplan *plan) { - upb_inttable_iter i; - upb_inttable_begin(&i, &plan->msginfo); - for(; !upb_inttable_done(&i); upb_inttable_next(&i)) { - upb_jitmsginfo *mi = upb_value_getptr(upb_inttable_iter_value(&i)); - free(mi->tablearray); - free(mi); - } - upb_inttable_uninit(&plan->msginfo); - munmap(plan->jit_code, plan->jit_size); - free(plan->debug_info); - // TODO: unregister -} - -static void upb_decoder_enterjit(upb_pbdecoder *d, const decoderplan *plan) { - if (plan->jit_code && - d->top == d->stack && - d->sink->top == d->sink->stack && - d->ptr && d->ptr < d->jit_end) { -#ifndef NDEBUG - register uint64_t rbx asm ("rbx") = 11; - register uint64_t r12 asm ("r12") = 12; - register uint64_t r13 asm ("r13") = 13; - register uint64_t r14 asm ("r14") = 14; - register uint64_t r15 asm ("r15") = 15; -#endif - // Decodes as many fields as possible, updating d->ptr appropriately, - // before falling through to the slow(er) path. - void (*upb_jit_decode)(upb_pbdecoder *d, void*) = (void*)plan->jit_code; - upb_jitmsginfo *mi = upb_getmsginfo(plan, plan->dest_handlers); - assert(mi); - upb_jit_decode(d, mi->jit_func); - assert(d->ptr <= d->end); - - // Test that callee-save registers were properly restored. - assert(rbx == 11); - assert(r12 == 12); - assert(r13 == 13); - assert(r14 == 14); - assert(r15 == 15); - } -} diff --git a/upb/pb/glue.c b/upb/pb/glue.c index 2c621f4..9027e0f 100644 --- a/upb/pb/glue.c +++ b/upb/pb/glue.c @@ -19,7 +19,7 @@ upb_def **upb_load_defs_from_descriptor(const char *str, size_t len, int *n, // Create handlers. const upb_handlers *reader_h = upb_descreader_gethandlers(&reader_h); const upb_handlers *decoder_h = - upb_pbdecoder_gethandlers(reader_h, false, &decoder_h); + upb_pbdecoder_gethandlers(reader_h, true, &decoder_h); // Create pipeline. upb_pipeline pipeline; @@ -68,8 +68,8 @@ char *upb_readfile(const char *filename, size_t *len) { long size = ftell(f); if(size < 0) goto error; if(fseek(f, 0, SEEK_SET) != 0) goto error; - char *buf = malloc(size); - if(fread(buf, size, 1, f) != 1) goto error; + char *buf = malloc(size + 1); + if(size && fread(buf, size, 1, f) != 1) goto error; fclose(f); if (len) *len = size; return buf; diff --git a/upb/pb/varint.c b/upb/pb/varint.c index d6d6161..ccd752d 100644 --- a/upb/pb/varint.c +++ b/upb/pb/varint.c @@ -5,7 +5,7 @@ * Author: Josh Haberman <jhaberman@gmail.com> */ -#include "upb/pb/varint.h" +#include "upb/pb/varint.int.h" // A basic branch-based decoder, uses 32-bit values to get good performance // on 32-bit architectures (but performs well on 64-bits also). diff --git a/upb/pb/varint.h b/upb/pb/varint.int.h index d33872d..d92fef9 100644 --- a/upb/pb/varint.h +++ b/upb/pb/varint.int.h @@ -11,6 +11,7 @@ #ifndef UPB_VARINT_DECODER_H_ #define UPB_VARINT_DECODER_H_ +#include <assert.h> #include <stdint.h> #include <string.h> #include "upb/upb.h" @@ -29,6 +30,8 @@ typedef enum { UPB_WIRE_TYPE_32BIT = 5, } upb_wiretype_t; +#define UPB_MAX_WIRE_TYPE 5 + // The maximum number of bytes that it takes to encode a 64-bit varint. // Note that with a better encoding this could be 9 (TODO: write up a // wiki document about this). @@ -87,7 +90,7 @@ UPB_VARINT_DECODER_CHECK2(massimino, upb_vdecode_max8_massimino); // favored best-performing implementations. UPB_INLINE upb_decoderet upb_vdecode_fast(const char *p) { if (sizeof(long) == 8) - return upb_vdecode_check2_massimino(p); + return upb_vdecode_check2_branch64(p); else return upb_vdecode_check2_branch32(p); } |