From 87fc2c516bff207f880c71526926842fd8dcc77e Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Wed, 7 Jan 2015 18:02:09 -0800 Subject: Changes from Google-internal development. * JSON parser expanded to handle split buffers. * bugfix to the protobuf decoder. --- tests/json/test_json.cc | 84 +++++++++++++++++++----------- tests/pb/test_decoder.cc | 98 +++++++++++++++++------------------ tests/test_table.cc | 17 ++++--- tests/test_util.h | 130 ++++++++++++++++++++++++++++++----------------- 4 files changed, 196 insertions(+), 133 deletions(-) (limited to 'tests') diff --git a/tests/json/test_json.cc b/tests/json/test_json.cc index 1444081..f1e2304 100644 --- a/tests/json/test_json.cc +++ b/tests/json/test_json.cc @@ -6,6 +6,7 @@ * A set of tests for JSON parsing and serialization. */ +#include "tests/test_util.h" #include "tests/upb_test.h" #include "upb/handlers.h" #include "upb/symtab.h" @@ -27,6 +28,8 @@ struct TestCase { const char* expected; }; +bool verbose = false; + static TestCase kTestRoundtripMessages[] = { // Test most fields here. { @@ -190,6 +193,51 @@ class StringSink { std::string s_; }; +void test_json_roundtrip_message(const char* json_src, + const char* json_expected, + const upb::Handlers* serialize_handlers, + int seam) { + upb::Status st; + upb::json::Parser parser(&st); + upb::json::Printer printer(serialize_handlers); + StringSink data_sink; + + parser.ResetOutput(printer.input()); + printer.ResetOutput(data_sink.Sink()); + + upb::BytesSink* input = parser.input(); + void *sub; + size_t len = strlen(json_src); + size_t ofs = 0; + + bool ok = input->Start(0, &sub) && + parse_buffer(input, sub, json_src, 0, seam, &ofs, &st, verbose) && + parse_buffer(input, sub, json_src, seam, len, &ofs, &st, verbose) && + ofs == len; + + if (ok) { + if (verbose) { + fprintf(stderr, "calling end()\n"); + } + ok = input->End(); + } + + if (!ok) { + fprintf(stderr, "upb parse error: %s\n", st.error_message()); + } + ASSERT(ok); + + if (memcmp(json_expected, + data_sink.Data().data(), + data_sink.Data().size())) { + fprintf(stderr, + "JSON parse/serialize roundtrip result differs:\n" + "Original:\n%s\nParsed/Serialized:\n%s\n", + json_src, data_sink.Data().c_str()); + abort(); + } +} + // Starts with a message in JSON format, parses and directly serializes again, // and compares the result. void test_json_roundtrip() { @@ -200,36 +248,14 @@ void test_json_roundtrip() { for (const TestCase* test_case = kTestRoundtripMessages; test_case->input != NULL; test_case++) { + const char *expected = + (test_case->expected == EXPECT_SAME) ? + test_case->input : + test_case->expected; - const char *json_src = test_case->input; - const char *json_expected = test_case->expected; - if (json_expected == EXPECT_SAME) { - json_expected = json_src; - } - - upb::Status st; - upb::json::Parser parser(&st); - upb::json::Printer printer(serialize_handlers.get()); - StringSink data_sink; - - parser.ResetOutput(printer.input()); - printer.ResetOutput(data_sink.Sink()); - - bool ok = upb::BufferSource::PutBuffer(json_src, strlen(json_src), - parser.input()); - if (!ok) { - fprintf(stderr, "upb parse error: %s\n", st.error_message()); - } - ASSERT(ok); - - if (memcmp(json_expected, - data_sink.Data().data(), - data_sink.Data().size())) { - fprintf(stderr, - "JSON parse/serialize roundtrip result differs:\n" - "Original:\n%s\nParsed/Serialized:\n%s\n", - json_src, data_sink.Data().c_str()); - abort(); + for (int i = 0; i < strlen(test_case->input); i++) { + test_json_roundtrip_message(test_case->input, expected, + serialize_handlers.get(), i); } } } diff --git a/tests/pb/test_decoder.cc b/tests/pb/test_decoder.cc index 313ce02..3aea777 100644 --- a/tests/pb/test_decoder.cc +++ b/tests/pb/test_decoder.cc @@ -36,11 +36,17 @@ #include #include +#include "tests/test_util.h" #include "tests/upb_test.h" + +#ifdef AMALGAMATED +#include "upb.h" +#else // AMALGAMATED #include "upb/handlers.h" #include "upb/pb/decoder.h" #include "upb/pb/varint.int.h" #include "upb/upb.h" +#endif // !AMALGAMATED #undef PRINT_FAILURE #define PRINT_FAILURE(expr) \ @@ -62,7 +68,6 @@ uint32_t filter_hash = 0; double completed; double total; double *count; -upb::BufferHandle global_handle; enum TestMode { COUNT_ONLY = 1, @@ -525,55 +530,16 @@ void CheckBytesParsed(const upb::pb::Decoder& decoder, size_t ofs) { ASSERT(ofs <= (decoder.BytesParsed() + MAX_BUFFERED)); } -bool parse(upb::pb::Decoder* decoder, void* subc, const char* buf, - size_t start, size_t end, size_t* ofs, upb::Status* status) { +static bool parse(upb::pb::Decoder* decoder, void* subc, const char* buf, + size_t start, size_t end, size_t* ofs, upb::Status* status) { CheckBytesParsed(*decoder, *ofs); - upb::BytesSink* s = decoder->input(); - start = UPB_MAX(start, *ofs); - if (start <= end) { - size_t len = end - start; - if (filter_hash) { - fprintf(stderr, "Calling parse(%zu) for bytes %zu-%zu of the input\n", - len, start, end); - } - size_t parsed = s->PutBuffer(subc, buf + start, len, &global_handle); - if (filter_hash) { - if (parsed == len) { - fprintf(stderr, - "parse(%zu) = %zu, complete byte count indicates success\n", - len, len); - } else if (parsed > len) { - fprintf(stderr, - "parse(%zu) = %zu, long byte count indicates success and skip" - "of the next %zu bytes\n", - len, parsed, parsed - len); - } else { - fprintf(stderr, - "parse(%zu) = %zu, short byte count indicates failure; " - "last %zu bytes were not consumed\n", - len, parsed, len - parsed); - } - } - if (status->ok() != (parsed >= len)) { - if (status->ok()) { - fprintf(stderr, - "Error: decode function returned short byte count but set no " - "error status\n"); - } else { - fprintf(stderr, - "Error: decode function returned complete byte count but set " - "error status\n"); - } - fprintf(stderr, "Status: %s, parsed=%zu, len=%zu\n", - status->error_message(), parsed, len); - ASSERT(false); - } - if (!status->ok()) - return false; - *ofs += parsed; + bool ret = parse_buffer(decoder->input(), subc, buf, start, end, ofs, status, + filter_hash != 0); + if (ret) { CheckBytesParsed(*decoder, *ofs); } - return true; + + return ret; } #define LINE(x) x "\n" @@ -1148,7 +1114,41 @@ void test_emptyhandlers(bool allowjit) { upb::reffed_ptr h(upb::Handlers::New(md.get())); bool ok = h->Freeze(NULL); ASSERT(ok); - NewMethod(h.get(), allowjit); +upb::reffed_ptr method = + NewMethod(h.get(), allowjit); + ASSERT(method.get()); + + // TODO: also test the case where a message has fields, but the fields are + // submessage fields and have no handlers. This also results in a decoder + // method with no field-handling code. + + // Ensure that the method can run with empty and non-empty input. + string test_unknown_field_msg = + cat(tag(1, UPB_WIRE_TYPE_VARINT), varint(42), + tag(2, UPB_WIRE_TYPE_DELIMITED), delim("My test data")); + const struct { + const char* data; + size_t length; + } testdata[] = { + { "", 0 }, + { test_unknown_field_msg.data(), test_unknown_field_msg.size() }, + { NULL, 0 }, + }; + for (int i = 0; testdata[i].data; i++) { + upb::Status status; + upb::pb::Decoder decoder(method.get(), &status); + upb::Sink sink(global_handlers, &closures[0]); + decoder.ResetOutput(&sink); + upb::BytesSink* input = decoder.input(); + void* subc; + ASSERT(input->Start(0, &subc)); + size_t ofs = 0; + ASSERT(parse_buffer(input, subc, + testdata[i].data, 0, testdata[i].length, + &ofs, &status, false)); + ASSERT(ofs == testdata[i].length); + ASSERT(input->End()); + } } void run_tests(bool use_jit) { @@ -1166,7 +1166,7 @@ void run_tests(bool use_jit) { test_invalid(); test_valid(); - test_emptyhandlers(false); + test_emptyhandlers(use_jit); } void run_test_suite() { diff --git a/tests/test_table.cc b/tests/test_table.cc index d51138b..9cc98b1 100644 --- a/tests/test_table.cc +++ b/tests/test_table.cc @@ -15,7 +15,6 @@ #include #include #include -#include "tests/test_util.h" #include "tests/upb_test.h" #include "upb/table.int.h" @@ -214,7 +213,8 @@ void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { x += (uintptr_t)ok; } double total = get_usertime() - before; - printf("%s/s\n", eng(i/total, 3, false)); + printf("%ld/s\n", (long)(i/total)); + double upb_seq_i = i / 100; // For later percentage calcuation. printf("upb_inttable(rand): "); fflush(stdout); @@ -227,7 +227,8 @@ void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { x += (uintptr_t)ok; } total = get_usertime() - before; - printf("%s/s\n", eng(i/total, 3, false)); + printf("%ld/s\n", (long)(i/total)); + double upb_rand_i = i / 100; // For later percentage calculation. printf("std::map(seq): "); fflush(stdout); @@ -238,7 +239,7 @@ void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { x += m[key]; } total = get_usertime() - before; - printf("%s/s\n", eng(i/total, 3, false)); + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_seq_i); printf("std::map(rand): "); fflush(stdout); @@ -249,7 +250,7 @@ void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { x += m[key]; } total = get_usertime() - before; - printf("%s/s\n", eng(i/total, 3, false)); + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_rand_i); printf("__gnu_cxx::hash_map(seq): "); fflush(stdout); @@ -260,7 +261,7 @@ void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { x += hm[key]; } total = get_usertime() - before; - printf("%s/s\n", eng(i/total, 3, false)); + printf("%ld/s (%0.1f%% of upb)\n", (long)(i/total), i / upb_seq_i); printf("__gnu_cxx::hash_map(rand): "); fflush(stdout); @@ -272,7 +273,7 @@ void test_inttable(int32_t *keys, uint16_t num_entries, const char *desc) { } total = get_usertime() - before; if (x == INT_MAX) abort(); - printf("%s/s\n\n", eng(i/total, 3, false)); + printf("%ld/s (%0.1f%% of upb)\n\n", (long)(i/total), i / upb_rand_i); upb_inttable_uninit(&table); delete rand_order; } @@ -308,7 +309,7 @@ extern "C" { int run_tests(int argc, char *argv[]) { for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--benchmark") == 0) benchmark = true; + if (strcmp(argv[i], "benchmark") == 0) benchmark = true; } vector keys; diff --git a/tests/test_util.h b/tests/test_util.h index a998a19..27c2bb3 100644 --- a/tests/test_util.h +++ b/tests/test_util.h @@ -1,53 +1,89 @@ -/* Function for printing numbers using si prefixes (k, M, G, etc.). - * From http://www.cs.tut.fi/~jkorpela/c/eng.html */ +/* + * upb - a minimalist implementation of protocol buffers. + * + * Copyright (c) 2014 Google Inc. See LICENSE for details. + * + * Common functionality for tests. + */ -#define PREFIX_START (-24) -/* Smallest power of then for which there is a prefix defined. - If the set of prefixes will be extended, change this constant - and update the table "prefix". */ +#ifndef UPB_TEST_UTIL_H_ +#define UPB_TEST_UTIL_H_ #include #include +#include "tests/upb_test.h" +#include "upb/sink.h" -static char *eng(double value, int digits, int numeric) -{ - static const char *prefix[] = { - "y", "z", "a", "f", "p", "n", "u", "m", "", - "k", "M", "G", "T", "P", "E", "Z", "Y" - }; -#define PREFIX_END (PREFIX_START+\ -(int)((sizeof(prefix)/sizeof(char *)-1)*3)) - - int expof10; - static char result[100]; - char *res = result; - - if (value < 0.) - { - *res++ = '-'; - value = -value; - } - - expof10 = (int) log10(value); - if(expof10 > 0) - expof10 = (expof10/3)*3; - else - expof10 = (-expof10+3)/3*(-3); - - value *= pow(10,-expof10); - - if (value >= 1000.) - { value /= 1000.0; expof10 += 3; } - else if(value >= 100.0) - digits -= 2; - else if(value >= 10.0) - digits -= 1; - - if(numeric || (expof10 < PREFIX_START) || - (expof10 > PREFIX_END)) - sprintf(res, "%.*fe%d", digits-1, value, expof10); - else - sprintf(res, "%.*f %s", digits-1, value, - prefix[(expof10-PREFIX_START)/3]); - return result; +upb::BufferHandle global_handle; + +// Puts a region of the given buffer [start, end) into the given sink (which +// probably represents a parser. Can gracefully handle the case where the +// parser returns a "parsed" length that is less or greater than the input +// buffer length, and tracks the overall parse offset in *ofs. +// +// Pass verbose=true to print detailed diagnostics to stderr. +bool parse_buffer(upb::BytesSink* sink, void* subc, const char* buf, + size_t start, size_t end, size_t* ofs, + upb::Status* status, bool verbose) { + start = UPB_MAX(start, *ofs); + + if (start <= end) { + size_t len = end - start; + + // Copy buffer into a separate, temporary buffer. + // This is necessary to verify that the parser is not erroneously + // reading outside the specified bounds. + char *buf2 = (char*)malloc(len); + assert(buf2); + memcpy(buf2, buf + start, len); + + if (verbose) { + fprintf(stderr, "Calling parse(%zu) for bytes %zu-%zu of the input\n", + len, start, end); + } + + size_t parsed = sink->PutBuffer(subc, buf2, len, &global_handle); + free(buf2); + + if (verbose) { + if (parsed == len) { + fprintf(stderr, + "parse(%zu) = %zu, complete byte count indicates success\n", + len, len); + } else if (parsed > len) { + fprintf(stderr, + "parse(%zu) = %zu, long byte count indicates success and skip" + "of the next %zu bytes\n", + len, parsed, parsed - len); + } else { + fprintf(stderr, + "parse(%zu) = %zu, short byte count indicates failure; " + "last %zu bytes were not consumed\n", + len, parsed, len - parsed); + } + } + + if (status->ok() != (parsed >= len)) { + if (status->ok()) { + fprintf(stderr, + "Error: decode function returned short byte count but set no " + "error status\n"); + } else { + fprintf(stderr, + "Error: decode function returned complete byte count but set " + "error status\n"); + } + fprintf(stderr, "Status: %s, parsed=%zu, len=%zu\n", + status->error_message(), parsed, len); + ASSERT(false); + } + + if (!status->ok()) + return false; + + *ofs += parsed; + } + return true; } + +#endif -- cgit v1.2.3