diff options
author | Josh Haberman <jhaberman@gmail.com> | 2015-01-07 18:02:09 -0800 |
---|---|---|
committer | Josh Haberman <jhaberman@gmail.com> | 2015-01-07 18:02:09 -0800 |
commit | 87fc2c516bff207f880c71526926842fd8dcc77e (patch) | |
tree | 8b9e8685170e227c927454af47f7535077295afd /upb/json/parser.rl | |
parent | d18475ae57dc62fc6dc8d9e32ffa0ce571df00bf (diff) |
Changes from Google-internal development.
* JSON parser expanded to handle split buffers.
* bugfix to the protobuf decoder.
Diffstat (limited to 'upb/json/parser.rl')
-rw-r--r-- | upb/json/parser.rl | 788 |
1 files changed, 552 insertions, 236 deletions
diff --git a/upb/json/parser.rl b/upb/json/parser.rl index abc76c8..b72bc10 100644 --- a/upb/json/parser.rl +++ b/upb/json/parser.rl @@ -33,6 +33,9 @@ #define PARSER_CHECK_RETURN(x) if (!(x)) return false +// Used to signal that a capture has been suspended. +static char suspend_capture; + static upb_selector_t getsel_for_handlertype(upb_json_parser *p, upb_handlertype_t type) { upb_selector_t sel; @@ -46,41 +49,6 @@ static upb_selector_t parser_getsel(upb_json_parser *p) { p, upb_handlers_getprimitivehandlertype(p->top->f)); } -static void start_member(upb_json_parser *p) { - assert(!p->top->f); - assert(!p->accumulated); - p->accumulated_len = 0; -} - -static bool end_member(upb_json_parser *p) { - // TODO(haberman): support keys that span buffers or have escape sequences. - assert(!p->top->f); - assert(p->accumulated); - const upb_fielddef *f = - upb_msgdef_ntof(p->top->m, p->accumulated, p->accumulated_len); - - if (!f) { - // TODO(haberman): Ignore unknown fields if requested/configured to do so. - upb_status_seterrf(p->status, "No such field: %.*s\n", - (int)p->accumulated_len, p->accumulated); - return false; - } - - p->top->f = f; - p->accumulated = NULL; - - return true; -} - -static void start_object(upb_json_parser *p) { - upb_sink_startmsg(&p->top->sink); -} - -static void end_object(upb_json_parser *p) { - upb_status status; - upb_sink_endmsg(&p->top->sink, &status); -} - static bool check_stack(upb_json_parser *p) { if ((p->top + 1) == p->limit) { upb_status_seterrmsg(p->status, "Nesting too deep"); @@ -90,83 +58,28 @@ static bool check_stack(upb_json_parser *p) { return true; } -static bool start_subobject(upb_json_parser *p) { - assert(p->top->f); - - if (!upb_fielddef_issubmsg(p->top->f)) { - upb_status_seterrf(p->status, - "Object specified for non-message/group field: %s", - upb_fielddef_name(p->top->f)); - return false; - } - - if (!check_stack(p)) return false; - - upb_jsonparser_frame *inner = p->top + 1; - - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); - upb_sink_startsubmsg(&p->top->sink, sel, &inner->sink); - inner->m = upb_fielddef_msgsubdef(p->top->f); - inner->f = NULL; - p->top = inner; +// There are GCC/Clang built-ins for overflow checking which we could start +// using if there was any performance benefit to it. +static bool checked_add(size_t a, size_t b, size_t *c) { + if (SIZE_MAX - a < b) return false; + *c = a + b; return true; } -static void end_subobject(upb_json_parser *p) { - p->top--; - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); - upb_sink_endsubmsg(&p->top->sink, sel); -} - -static bool start_array(upb_json_parser *p) { - assert(p->top->f); - - if (!upb_fielddef_isseq(p->top->f)) { - upb_status_seterrf(p->status, - "Array specified for non-repeated field: %s", - upb_fielddef_name(p->top->f)); - return false; +static size_t saturating_multiply(size_t a, size_t b) { + // size_t is unsigned, so this is defined behavior even on overflow. + size_t ret = a * b; + if (b != 0 && ret / b != a) { + ret = SIZE_MAX; } - - if (!check_stack(p)) return false; - - upb_jsonparser_frame *inner = p->top + 1; - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); - upb_sink_startseq(&p->top->sink, sel, &inner->sink); - inner->m = p->top->m; - inner->f = p->top->f; - p->top = inner; - - return true; + return ret; } -static void end_array(upb_json_parser *p) { - assert(p->top > p->stack); - p->top--; - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); - upb_sink_endseq(&p->top->sink, sel); -} +/* Base64 decoding ************************************************************/ -static void clear_member(upb_json_parser *p) { p->top->f = NULL; } - -static bool parser_putbool(upb_json_parser *p, bool val) { - if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL) { - upb_status_seterrf(p->status, - "Boolean value specified for non-bool field: %s", - upb_fielddef_name(p->top->f)); - return false; - } - - bool ok = upb_sink_putbool(&p->top->sink, parser_getsel(p), val); - UPB_ASSERT_VAR(ok, ok); - return true; -} - -static void start_text(upb_json_parser *p, const char *ptr) { - p->text_begin = ptr; -} +// TODO(haberman): make this streaming. static const signed char b64table[] = { -1, -1, -1, -1, -1, -1, -1, -1, @@ -286,89 +199,323 @@ badpadding: return false; } -static bool end_text(upb_json_parser *p, const char *ptr, bool is_num) { - assert(!p->accumulated); // TODO: handle this case. - p->accumulated = p->text_begin; - p->accumulated_len = ptr - p->text_begin; - if (p->top->f && upb_fielddef_isstring(p->top->f)) { - // This is a string field (as opposed to a member name). - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); - if (upb_fielddef_type(p->top->f) == UPB_TYPE_BYTES) { - PARSER_CHECK_RETURN(base64_push(p, sel, p->accumulated, - p->accumulated_len)); - } else { - upb_sink_putstring(&p->top->sink, sel, p->accumulated, p->accumulated_len, NULL); +/* Accumulate buffer **********************************************************/ + +// Functionality for accumulating a buffer. +// +// Some parts of the parser need an entire value as a contiguous string. For +// example, to look up a member name in a hash table, or to turn a string into +// a number, the relevant library routines need the input string to be in +// contiguous memory, even if the value spanned two or more buffers in the +// input. These routines handle that. +// +// In the common case we can just point to the input buffer to get this +// contiguous string and avoid any actual copy. So we optimistically begin +// this way. But there are a few cases where we must instead copy into a +// separate buffer: +// +// 1. The string was not contiguous in the input (it spanned buffers). +// +// 2. The string included escape sequences that need to be interpreted to get +// the true value in a contiguous buffer. + +static void assert_accumulate_empty(upb_json_parser *p) { + UPB_UNUSED(p); + assert(p->accumulated == NULL); + assert(p->accumulated_len == 0); +} + +static void accumulate_clear(upb_json_parser *p) { + p->accumulated = NULL; + p->accumulated_len = 0; +} + +// Used internally by accumulate_append(). +static bool accumulate_realloc(upb_json_parser *p, size_t need) { + size_t new_size = UPB_MAX(p->accumulate_buf_size, 128); + while (new_size < need) { + new_size = saturating_multiply(new_size, 2); + } + + void *mem = realloc(p->accumulate_buf, new_size); + if (!mem) { + upb_status_seterrmsg(p->status, "Out of memory allocating buffer."); + return false; + } + + p->accumulate_buf = mem; + p->accumulate_buf_size = new_size; + return true; +} + +// Logically appends the given data to the append buffer. +// If "can_alias" is true, we will try to avoid actually copying, but the buffer +// must be valid until the next accumulate_append() call (if any). +static bool accumulate_append(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + if (!p->accumulated && can_alias) { + p->accumulated = buf; + p->accumulated_len = len; + return true; + } + + if (p->accumulate_buf_size - p->accumulated_len < len) { + size_t need; + if (!checked_add(p->accumulated_len, len, &need)) { + upb_status_seterrmsg(p->status, "Integer overflow."); + return false; } - p->accumulated = NULL; - } else if (p->top->f && - upb_fielddef_type(p->top->f) == UPB_TYPE_ENUM && - !is_num) { - - // Enum case: resolve enum symbolic name to integer value. - const upb_enumdef *enumdef = - (const upb_enumdef*)upb_fielddef_subdef(p->top->f); - - int32_t int_val = 0; - if (upb_enumdef_ntoi(enumdef, p->accumulated, p->accumulated_len, - &int_val)) { - upb_selector_t sel = parser_getsel(p); - upb_sink_putint32(&p->top->sink, sel, int_val); - } else { - upb_status_seterrmsg(p->status, "Enum value name unknown"); + + if (!accumulate_realloc(p, need)) { return false; } - p->accumulated = NULL; } + if (p->accumulated != p->accumulate_buf) { + memcpy(p->accumulate_buf, p->accumulated, p->accumulated_len); + p->accumulated = p->accumulate_buf; + } + + memcpy(p->accumulate_buf + p->accumulated_len, buf, len); + p->accumulated_len += len; return true; } -static bool start_stringval(upb_json_parser *p) { - assert(p->top->f); +// Returns a pointer to the data accumulated since the last accumulate_clear() +// call, and writes the length to *len. This with point either to the input +// buffer or a temporary accumulate buffer. +static const char *accumulate_getptr(upb_json_parser *p, size_t *len) { + assert(p->accumulated); + *len = p->accumulated_len; + return p->accumulated; +} - if (upb_fielddef_isstring(p->top->f)) { - if (!check_stack(p)) return false; - // Start a new parser frame: parser frames correspond one-to-one with - // handler frames, and string events occur in a sub-frame. - upb_jsonparser_frame *inner = p->top + 1; - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); - upb_sink_startstr(&p->top->sink, sel, 0, &inner->sink); - inner->m = p->top->m; - inner->f = p->top->f; - p->top = inner; +/* Mult-part text data ********************************************************/ + +// When we have text data in the input, it can often come in multiple segments. +// For example, there may be some raw string data followed by an escape +// sequence. The two segments are processed with different logic. Also buffer +// seams in the input can cause multiple segments. +// +// As we see segments, there are two main cases for how we want to process them: +// +// 1. we want to push the captured input directly to string handlers. +// +// 2. we need to accumulate all the parts into a contiguous buffer for further +// processing (field name lookup, string->number conversion, etc). + +// This is the set of states for p->multipart_state. +enum { + // We are not currently processing multipart data. + MULTIPART_INACTIVE = 0, + + // We are processing multipart data by accumulating it into a contiguous + // buffer. + MULTIPART_ACCUMULATE = 1, + + // We are processing multipart data by pushing each part directly to the + // current string handlers. + MULTIPART_PUSHEAGERLY = 2 +}; - return true; - } else if (upb_fielddef_type(p->top->f) == UPB_TYPE_ENUM) { - // Do nothing -- symbolic enum names in quotes remain in the - // current parser frame. +// Start a multi-part text value where we accumulate the data for processing at +// the end. +static void multipart_startaccum(upb_json_parser *p) { + assert_accumulate_empty(p); + assert(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_ACCUMULATE; +} + +// Start a multi-part text value where we immediately push text data to a string +// value with the given selector. +static void multipart_start(upb_json_parser *p, upb_selector_t sel) { + assert_accumulate_empty(p); + assert(p->multipart_state == MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_PUSHEAGERLY; + p->string_selector = sel; +} + +static bool multipart_text(upb_json_parser *p, const char *buf, size_t len, + bool can_alias) { + switch (p->multipart_state) { + case MULTIPART_INACTIVE: + upb_status_seterrmsg( + p->status, "Internal error: unexpected state MULTIPART_INACTIVE"); + return false; + + case MULTIPART_ACCUMULATE: + if (!accumulate_append(p, buf, len, can_alias)) { + return false; + } + break; + + case MULTIPART_PUSHEAGERLY: { + const upb_bufhandle *handle = can_alias ? p->handle : NULL; + upb_sink_putstring(&p->top->sink, p->string_selector, buf, len, handle); + break; + } + } + + return true; +} + +// Note: this invalidates the accumulate buffer! Call only after reading its +// contents. +static void multipart_end(upb_json_parser *p) { + assert(p->multipart_state != MULTIPART_INACTIVE); + p->multipart_state = MULTIPART_INACTIVE; + accumulate_clear(p); +} + + +/* Input capture **************************************************************/ + +// Functionality for capturing a region of the input as text. Gracefully +// handles the case where a buffer seam occurs in the middle of the captured +// region. + +static void capture_begin(upb_json_parser *p, const char *ptr) { + assert(p->multipart_state != MULTIPART_INACTIVE); + assert(p->capture == NULL); + p->capture = ptr; +} + +static bool capture_end(upb_json_parser *p, const char *ptr) { + assert(p->capture); + if (multipart_text(p, p->capture, ptr - p->capture, true)) { + p->capture = NULL; return true; } else { - upb_status_seterrf(p->status, - "String specified for non-string/non-enum field: %s", - upb_fielddef_name(p->top->f)); return false; } +} +// This is called at the end of each input buffer (ie. when we have hit a +// buffer seam). If we are in the middle of capturing the input, this +// processes the unprocessed capture region. +static void capture_suspend(upb_json_parser *p, const char **ptr) { + if (!p->capture) return; + + if (multipart_text(p, p->capture, *ptr - p->capture, false)) { + // We use this as a signal that we were in the middle of capturing, and + // that capturing should resume at the beginning of the next buffer. + // + // We can't use *ptr here, because we have no guarantee that this pointer + // will be valid when we resume (if the underlying memory is freed, then + // using the pointer at all, even to compare to NULL, is likely undefined + // behavior). + p->capture = &suspend_capture; + } else { + // Need to back up the pointer to the beginning of the capture, since + // we were not able to actually preserve it. + *ptr = p->capture; + } } -static void end_stringval(upb_json_parser *p) { - if (upb_fielddef_isstring(p->top->f)) { - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); - upb_sink_endstr(&p->top->sink, sel); - p->top--; +static void capture_resume(upb_json_parser *p, const char *ptr) { + if (p->capture) { + assert(p->capture == &suspend_capture); + p->capture = ptr; + } +} + + +/* Callbacks from the parser **************************************************/ + +// These are the functions called directly from the parser itself. +// We define these in the same order as their declarations in the parser. + +static char escape_char(char in) { + switch (in) { + case 'r': return '\r'; + case 't': return '\t'; + case 'n': return '\n'; + case 'f': return '\f'; + case 'b': return '\b'; + case '/': return '/'; + case '"': return '"'; + case '\\': return '\\'; + default: + assert(0); + return 'x'; + } +} + +static bool escape(upb_json_parser *p, const char *ptr) { + char ch = escape_char(*ptr); + return multipart_text(p, &ch, 1, false); +} + +static void start_hex(upb_json_parser *p) { + p->digit = 0; +} + +static void hexdigit(upb_json_parser *p, const char *ptr) { + char ch = *ptr; + + p->digit <<= 4; + + if (ch >= '0' && ch <= '9') { + p->digit += (ch - '0'); + } else if (ch >= 'a' && ch <= 'f') { + p->digit += ((ch - 'a') + 10); + } else { + assert(ch >= 'A' && ch <= 'F'); + p->digit += ((ch - 'A') + 10); } } +static bool end_hex(upb_json_parser *p) { + uint32_t codepoint = p->digit; + + // emit the codepoint as UTF-8. + char utf8[3]; // support \u0000 -- \uFFFF -- need only three bytes. + int length = 0; + if (codepoint <= 0x7F) { + utf8[0] = codepoint; + length = 1; + } else if (codepoint <= 0x07FF) { + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x1F) | 0xC0; + length = 2; + } else /* codepoint <= 0xFFFF */ { + utf8[2] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[1] = (codepoint & 0x3F) | 0x80; + codepoint >>= 6; + utf8[0] = (codepoint & 0x0F) | 0xE0; + length = 3; + } + // TODO(haberman): Handle high surrogates: if codepoint is a high surrogate + // we have to wait for the next escape to get the full code point). + + return multipart_text(p, utf8, length, false); +} + +static void start_text(upb_json_parser *p, const char *ptr) { + capture_begin(p, ptr); +} + +static bool end_text(upb_json_parser *p, const char *ptr) { + return capture_end(p, ptr); +} + static void start_number(upb_json_parser *p, const char *ptr) { - start_text(p, ptr); - assert(p->accumulated == NULL); + multipart_startaccum(p); + capture_begin(p, ptr); } -static void end_number(upb_json_parser *p, const char *ptr) { - end_text(p, ptr, true); - const char *myend = p->accumulated + p->accumulated_len; +static bool end_number(upb_json_parser *p, const char *ptr) { + if (!capture_end(p, ptr)) { + return false; + } + + size_t len; + const char *buf = accumulate_getptr(p, &len); + const char *myend = buf + len; char *end; switch (upb_fielddef_type(p->top->f)) { @@ -376,7 +523,7 @@ static void end_number(upb_json_parser *p, const char *ptr) { case UPB_TYPE_INT32: { long val = strtol(p->accumulated, &end, 0); if (val > INT32_MAX || val < INT32_MIN || errno == ERANGE || end != myend) - assert(false); + goto err; else upb_sink_putint32(&p->top->sink, parser_getsel(p), val); break; @@ -384,7 +531,7 @@ static void end_number(upb_json_parser *p, const char *ptr) { case UPB_TYPE_INT64: { long long val = strtoll(p->accumulated, &end, 0); if (val > INT64_MAX || val < INT64_MIN || errno == ERANGE || end != myend) - assert(false); + goto err; else upb_sink_putint64(&p->top->sink, parser_getsel(p), val); break; @@ -392,7 +539,7 @@ static void end_number(upb_json_parser *p, const char *ptr) { case UPB_TYPE_UINT32: { unsigned long val = strtoul(p->accumulated, &end, 0); if (val > UINT32_MAX || errno == ERANGE || end != myend) - assert(false); + goto err; else upb_sink_putuint32(&p->top->sink, parser_getsel(p), val); break; @@ -400,7 +547,7 @@ static void end_number(upb_json_parser *p, const char *ptr) { case UPB_TYPE_UINT64: { unsigned long long val = strtoull(p->accumulated, &end, 0); if (val > UINT64_MAX || errno == ERANGE || end != myend) - assert(false); + goto err; else upb_sink_putuint64(&p->top->sink, parser_getsel(p), val); break; @@ -408,7 +555,7 @@ static void end_number(upb_json_parser *p, const char *ptr) { case UPB_TYPE_DOUBLE: { double val = strtod(p->accumulated, &end); if (errno == ERANGE || end != myend) - assert(false); + goto err; else upb_sink_putdouble(&p->top->sink, parser_getsel(p), val); break; @@ -416,7 +563,7 @@ static void end_number(upb_json_parser *p, const char *ptr) { case UPB_TYPE_FLOAT: { float val = strtof(p->accumulated, &end); if (errno == ERANGE || end != myend) - assert(false); + goto err; else upb_sink_putfloat(&p->top->sink, parser_getsel(p), val); break; @@ -425,84 +572,236 @@ static void end_number(upb_json_parser *p, const char *ptr) { assert(false); } - p->accumulated = NULL; + multipart_end(p); + return true; + +err: + upb_status_seterrf(p->status, "error parsing number: %.*s", buf, len); + multipart_end(p); + return false; } -static char escape_char(char in) { - switch (in) { - case 'r': return '\r'; - case 't': return '\t'; - case 'n': return '\n'; - case 'f': return '\f'; - case 'b': return '\b'; - case '/': return '/'; - case '"': return '"'; - case '\\': return '\\'; +static bool parser_putbool(upb_json_parser *p, bool val) { + if (upb_fielddef_type(p->top->f) != UPB_TYPE_BOOL) { + upb_status_seterrf(p->status, + "Boolean value specified for non-bool field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + bool ok = upb_sink_putbool(&p->top->sink, parser_getsel(p), val); + UPB_ASSERT_VAR(ok, ok); + return true; +} + +static bool start_stringval(upb_json_parser *p) { + assert(p->top->f); + + if (upb_fielddef_isstring(p->top->f)) { + if (!check_stack(p)) return false; + + // Start a new parser frame: parser frames correspond one-to-one with + // handler frames, and string events occur in a sub-frame. + upb_jsonparser_frame *inner = p->top + 1; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSTR); + upb_sink_startstr(&p->top->sink, sel, 0, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + if (upb_fielddef_type(p->top->f) == UPB_TYPE_STRING) { + // For STRING fields we push data directly to the handlers as it is + // parsed. We don't do this yet for BYTES fields, because our base64 + // decoder is not streaming. + // + // TODO(haberman): make base64 decoding streaming also. + multipart_start(p, getsel_for_handlertype(p, UPB_HANDLER_STRING)); + return true; + } else { + multipart_startaccum(p); + return true; + } + } else if (upb_fielddef_type(p->top->f) == UPB_TYPE_ENUM) { + // No need to push a frame -- symbolic enum names in quotes remain in the + // current parser frame. + // + // Enum string values must accumulate so we can look up the value in a table + // once it is complete. + multipart_startaccum(p); + return true; + } else { + upb_status_seterrf(p->status, + "String specified for non-string/non-enum field: %s", + upb_fielddef_name(p->top->f)); + return false; + } +} + +static bool end_stringval(upb_json_parser *p) { + bool ok = true; + + switch (upb_fielddef_type(p->top->f)) { + case UPB_TYPE_BYTES: + if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), + p->accumulated, p->accumulated_len)) { + return false; + } + // Fall through. + + case UPB_TYPE_STRING: { + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); + upb_sink_endstr(&p->top->sink, sel); + p->top--; + break; + } + + case UPB_TYPE_ENUM: { + // Resolve enum symbolic name to integer value. + const upb_enumdef *enumdef = + (const upb_enumdef*)upb_fielddef_subdef(p->top->f); + + size_t len; + const char *buf = accumulate_getptr(p, &len); + + int32_t int_val = 0; + ok = upb_enumdef_ntoi(enumdef, buf, len, &int_val); + + if (ok) { + upb_selector_t sel = parser_getsel(p); + upb_sink_putint32(&p->top->sink, sel, int_val); + } else { + upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf); + } + + break; + } + default: - assert(0); - return 'x'; + assert(false); + upb_status_seterrmsg(p->status, "Internal error in JSON decoder"); + ok = false; + break; } + + multipart_end(p); + return ok; } -static void escape(upb_json_parser *p, const char *ptr) { - char ch = escape_char(*ptr); - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); - upb_sink_putstring(&p->top->sink, sel, &ch, 1, NULL); +static void start_member(upb_json_parser *p) { + assert(!p->top->f); + multipart_startaccum(p); } -static uint8_t hexdigit(char ch) { - if (ch >= '0' && ch <= '9') { - return ch - '0'; - } else if (ch >= 'a' && ch <= 'f') { - return ch - 'a' + 10; - } else { - assert(ch >= 'A' && ch <= 'F'); - return ch - 'A' + 10; +static bool end_member(upb_json_parser *p) { + assert(!p->top->f); + size_t len; + const char *buf = accumulate_getptr(p, &len); + + const upb_fielddef *f = upb_msgdef_ntof(p->top->m, buf, len); + + if (!f) { + // TODO(haberman): Ignore unknown fields if requested/configured to do so. + upb_status_seterrf(p->status, "No such field: %.*s\n", (int)len, buf); + return false; } + + p->top->f = f; + multipart_end(p); + + return true; } -static void start_hex(upb_json_parser *p, const char *ptr) { - start_text(p, ptr); +static void clear_member(upb_json_parser *p) { p->top->f = NULL; } + +static bool start_subobject(upb_json_parser *p) { + assert(p->top->f); + + if (!upb_fielddef_issubmsg(p->top->f)) { + upb_status_seterrf(p->status, + "Object specified for non-message/group field: %s", + upb_fielddef_name(p->top->f)); + return false; + } + + if (!check_stack(p)) return false; + + upb_jsonparser_frame *inner = p->top + 1; + + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSUBMSG); + upb_sink_startsubmsg(&p->top->sink, sel, &inner->sink); + inner->m = upb_fielddef_msgsubdef(p->top->f); + inner->f = NULL; + p->top = inner; + + return true; } -static void hex(upb_json_parser *p, const char *end) { - const char *start = p->text_begin; - UPB_ASSERT_VAR(end, end - start == 4); - uint16_t codepoint = - (hexdigit(start[0]) << 12) | - (hexdigit(start[1]) << 8) | - (hexdigit(start[2]) << 4) | - hexdigit(start[3]); - // emit the codepoint as UTF-8. - char utf8[3]; // support \u0000 -- \uFFFF -- need only three bytes. - int length = 0; - if (codepoint <= 0x7F) { - utf8[0] = codepoint; - length = 1; - } else if (codepoint <= 0x07FF) { - utf8[1] = (codepoint & 0x3F) | 0x80; - codepoint >>= 6; - utf8[0] = (codepoint & 0x1F) | 0xC0; - length = 2; - } else /* codepoint <= 0xFFFF */ { - utf8[2] = (codepoint & 0x3F) | 0x80; - codepoint >>= 6; - utf8[1] = (codepoint & 0x3F) | 0x80; - codepoint >>= 6; - utf8[0] = (codepoint & 0x0F) | 0xE0; - length = 3; +static void end_subobject(upb_json_parser *p) { + p->top--; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSUBMSG); + upb_sink_endsubmsg(&p->top->sink, sel); +} + +static bool start_array(upb_json_parser *p) { + assert(p->top->f); + + if (!upb_fielddef_isseq(p->top->f)) { + upb_status_seterrf(p->status, + "Array specified for non-repeated field: %s", + upb_fielddef_name(p->top->f)); + return false; } - // TODO(haberman): Handle high surrogates: if codepoint is a high surrogate - // we have to wait for the next escape to get the full code point). - upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); - upb_sink_putstring(&p->top->sink, sel, utf8, length, NULL); + if (!check_stack(p)) return false; + + upb_jsonparser_frame *inner = p->top + 1; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_STARTSEQ); + upb_sink_startseq(&p->top->sink, sel, &inner->sink); + inner->m = p->top->m; + inner->f = p->top->f; + p->top = inner; + + return true; +} + +static void end_array(upb_json_parser *p) { + assert(p->top > p->stack); + + p->top--; + upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSEQ); + upb_sink_endseq(&p->top->sink, sel); +} + +static void start_object(upb_json_parser *p) { + upb_sink_startmsg(&p->top->sink); +} + +static void end_object(upb_json_parser *p) { + upb_status status; + upb_sink_endmsg(&p->top->sink, &status); } + #define CHECK_RETURN_TOP(x) if (!(x)) goto error + +/* The actual parser **********************************************************/ + // What follows is the Ragel parser itself. The language is specified in Ragel // and the actions call our C functions above. +// +// Ragel has an extensive set of functionality, and we use only a small part of +// it. There are many action types but we only use a few: +// +// ">" -- transition into a machine +// "%" -- transition out of a machine +// "@" -- transition into a final state of a machine. +// +// "@" transitions are tricky because a machine can transition into a final +// state repeatedly. But in some cases we know this can't happen, for example +// a string which is delimited by a final '"' can only transition into its +// final state once, when the closing '"' is seen. + %%{ machine json; @@ -520,24 +819,30 @@ static void hex(upb_json_parser *p, const char *end) { text = /[^\\"]/+ >{ start_text(parser, p); } - %{ CHECK_RETURN_TOP(end_text(parser, p, false)); } + %{ CHECK_RETURN_TOP(end_text(parser, p)); } ; unicode_char = "\\u" /[0-9A-Fa-f]/{4} - >{ start_hex(parser, p); } - %{ hex(parser, p); } + >{ start_hex(parser); } + ${ hexdigit(parser, p); } + %{ CHECK_RETURN_TOP(end_hex(parser)); } ; escape_char = "\\" /[rtbfn"\/\\]/ - >{ escape(parser, p); } + >{ CHECK_RETURN_TOP(escape(parser, p)); } + ; + + string_machine := + (text | unicode_char | escape_char)** + '"' + @{ fhold; fret; } ; - string_machine := (text | unicode_char | escape_char)** '"' @{ fret; } ; - string = '"' @{ fcall string_machine; }; + string = '"' @{ fcall string_machine; } '"'; value2 = ^(space | "]" | "}") >{ fhold; fcall value_machine; } ; @@ -545,7 +850,7 @@ static void hex(upb_json_parser *p, const char *end) { ws string >{ start_member(parser); } - %{ CHECK_RETURN_TOP(end_member(parser)); } + @{ CHECK_RETURN_TOP(end_member(parser)); } ws ":" ws value2 %{ clear_member(parser); } @@ -573,10 +878,10 @@ static void hex(upb_json_parser *p, const char *end) { value = number >{ start_number(parser, p); } - %{ end_number(parser, p); } + %{ CHECK_RETURN_TOP(end_number(parser, p)); } | string >{ CHECK_RETURN_TOP(start_stringval(parser)); } - %{ end_stringval(parser); } + @{ CHECK_RETURN_TOP(end_stringval(parser)); } | "true" %{ CHECK_RETURN_TOP(parser_putbool(parser, true)); } | "false" @@ -602,6 +907,7 @@ size_t parse(void *closure, const void *hd, const char *buf, size_t size, UPB_UNUSED(hd); UPB_UNUSED(handle); upb_json_parser *parser = closure; + parser->handle = handle; // Variables used by Ragel's generated code. int cs = parser->current_state; @@ -611,10 +917,14 @@ size_t parse(void *closure, const void *hd, const char *buf, size_t size, const char *p = buf; const char *pe = buf + size; + capture_resume(parser, buf); + %% write exec; if (p != pe) { upb_status_seterrf(parser->status, "Parse error at %s\n", p); + } else { + capture_suspend(parser, &p); } error: @@ -631,8 +941,13 @@ bool end(void *closure, const void *hd) { return true; } + +/* Public API *****************************************************************/ + void upb_json_parser_init(upb_json_parser *p, upb_status *status) { p->limit = p->stack + UPB_JSON_MAX_DEPTH; + p->accumulate_buf = NULL; + p->accumulate_buf_size = 0; upb_byteshandler_init(&p->input_handler_); upb_byteshandler_setstring(&p->input_handler_, parse, NULL); upb_byteshandler_setendstr(&p->input_handler_, end, NULL); @@ -642,6 +957,7 @@ void upb_json_parser_init(upb_json_parser *p, upb_status *status) { void upb_json_parser_uninit(upb_json_parser *p) { upb_byteshandler_uninit(&p->input_handler_); + free(p->accumulate_buf); } void upb_json_parser_reset(upb_json_parser *p) { @@ -654,9 +970,9 @@ void upb_json_parser_reset(upb_json_parser *p) { %% write init; p->current_state = cs; p->parser_top = top; - p->text_begin = NULL; - p->accumulated = NULL; - p->accumulated_len = 0; + accumulate_clear(p); + p->multipart_state = MULTIPART_INACTIVE; + p->capture = NULL; } void upb_json_parser_resetoutput(upb_json_parser *p, upb_sink *sink) { |