Sync from internal Google development.

Many improvements, too many to mention.  One significant
perf regression warrants investigation:

omitfp.parsetoproto2_googlemessage1.upb_jit: 343 -> 252 (-26.53)
plain.parsetoproto2_googlemessage1.upb_jit: 334 -> 251 (-24.85)

25% regression for this benchmark is bad, but since I don't think
there's any fundamental design issue that caused it I'm going to
go ahead with the commit anyway.  Can investigate and fix later.

Other benchmarks were neutral or showed slight improvement.

1 files changed, 72 insertions, 222 deletions

diff --git a/tests/test_vs_proto2.cc b/tests/test_vs_proto2.cc
index 53b2498..020dca5 100644
--- a/tests/test_vs_proto2.cc
+++ b/tests/test_vs_proto2.cc
@@ -1,7 +1,7 @@
 /*
  * upb - a minimalist implementation of protocol buffers.
  *
- * Copyright (c) 2011 Google Inc.  See LICENSE for details.
+ * Copyright (c) 2011-2012 Google Inc.  See LICENSE for details.
  *
  * A test that verifies that our results are identical to proto2 for a
  * given proto type and input protobuf.
@@ -9,230 +9,87 @@
 
 #define __STDC_LIMIT_MACROS  // So we get UINT32_MAX
 #include <assert.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/wire_format_lite.h>
 #include <inttypes.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/wire_format_lite.h>
 #include "benchmarks/google_messages.pb.h"
-#include "upb/def.h"
-#include "upb/msg.h"
+#include "upb/def.hpp"
+#include "upb/handlers.hpp"
+#include "upb/msg.hpp"
+#include "upb/pb/decoder.hpp"
 #include "upb/pb/glue.h"
 #include "upb/pb/varint.h"
+#include "upb/proto2_bridge.hpp"
 #include "upb_test.h"
 
-size_t string_size;
-
-void compare(const google::protobuf::Message& proto2_msg,
-             void *upb_msg, const upb_msgdef *upb_md);
-
-void compare_arrays(const google::protobuf::Reflection *r,
-                    const google::protobuf::Message& proto2_msg,
-                    const google::protobuf::FieldDescriptor *proto2_f,
-                    void *upb_msg, upb_fielddef *upb_f)
-{
-  ASSERT(upb_msg_has(upb_msg, upb_f));
-  ASSERT(upb_isseq(upb_f));
-  const void *arr = upb_value_getptr(upb_msg_getseq(upb_msg, upb_f));
-  const void *iter = upb_seq_begin(arr, upb_f);
-  for(int i = 0;
-      i < r->FieldSize(proto2_msg, proto2_f);
-      i++, iter = upb_seq_next(arr, iter, upb_f)) {
-    ASSERT(!upb_seq_done(iter));
-    upb_value v = upb_seq_get(iter, upb_f);
-    switch(upb_f->type) {
-      default:
-        ASSERT(false);
-      case UPB_TYPE(DOUBLE):
-        ASSERT(r->GetRepeatedDouble(proto2_msg, proto2_f, i) == upb_value_getdouble(v));
-        break;
-      case UPB_TYPE(FLOAT):
-        ASSERT(r->GetRepeatedFloat(proto2_msg, proto2_f, i) == upb_value_getfloat(v));
-        break;
-      case UPB_TYPE(INT64):
-      case UPB_TYPE(SINT64):
-      case UPB_TYPE(SFIXED64):
-        ASSERT(r->GetRepeatedInt64(proto2_msg, proto2_f, i) == upb_value_getint64(v));
-        break;
-      case UPB_TYPE(UINT64):
-      case UPB_TYPE(FIXED64):
-        ASSERT(r->GetRepeatedUInt64(proto2_msg, proto2_f, i) == upb_value_getuint64(v));
-        break;
-      case UPB_TYPE(SFIXED32):
-      case UPB_TYPE(SINT32):
-      case UPB_TYPE(INT32):
-      case UPB_TYPE(ENUM):
-        ASSERT(r->GetRepeatedInt32(proto2_msg, proto2_f, i) == upb_value_getint32(v));
-        break;
-      case UPB_TYPE(FIXED32):
-      case UPB_TYPE(UINT32):
-        ASSERT(r->GetRepeatedUInt32(proto2_msg, proto2_f, i) == upb_value_getuint32(v));
-        break;
-      case UPB_TYPE(BOOL):
-        ASSERT(r->GetRepeatedBool(proto2_msg, proto2_f, i) == upb_value_getbool(v));
-        break;
-      case UPB_TYPE(STRING):
-      case UPB_TYPE(BYTES): {
-        std::string str = r->GetRepeatedString(proto2_msg, proto2_f, i);
-        upb_stdarray *upbstr = (upb_stdarray*)upb_value_getptr(v);
-        std::string str2(upbstr->ptr, upbstr->len);
-        string_size += upbstr->len;
-        ASSERT(str == str2);
-        break;
-      }
-      case UPB_TYPE(GROUP):
-      case UPB_TYPE(MESSAGE):
-        ASSERT(upb_dyncast_msgdef(upb_f->def) != NULL);
-        compare(r->GetRepeatedMessage(proto2_msg, proto2_f, i),
-                upb_value_getptr(v), upb_downcast_msgdef(upb_f->def));
-    }
-  }
-  ASSERT(upb_seq_done(iter));
-}
-
-void compare_values(const google::protobuf::Reflection *r,
-                    const google::protobuf::Message& proto2_msg,
-                    const google::protobuf::FieldDescriptor *proto2_f,
-                    void *upb_msg, upb_fielddef *upb_f)
-{
-  upb_value v = upb_msg_get(upb_msg, upb_f);
-  switch(upb_f->type) {
-    default:
-      ASSERT(false);
-    case UPB_TYPE(DOUBLE):
-      ASSERT(r->GetDouble(proto2_msg, proto2_f) == upb_value_getdouble(v));
-      break;
-    case UPB_TYPE(FLOAT):
-      ASSERT(r->GetFloat(proto2_msg, proto2_f) == upb_value_getfloat(v));
-      break;
-    case UPB_TYPE(INT64):
-    case UPB_TYPE(SINT64):
-    case UPB_TYPE(SFIXED64):
-      ASSERT(r->GetInt64(proto2_msg, proto2_f) == upb_value_getint64(v));
-      break;
-    case UPB_TYPE(UINT64):
-    case UPB_TYPE(FIXED64):
-      ASSERT(r->GetUInt64(proto2_msg, proto2_f) == upb_value_getuint64(v));
-      break;
-    case UPB_TYPE(SFIXED32):
-    case UPB_TYPE(SINT32):
-    case UPB_TYPE(INT32):
-    case UPB_TYPE(ENUM):
-      ASSERT(r->GetInt32(proto2_msg, proto2_f) == upb_value_getint32(v));
-      break;
-    case UPB_TYPE(FIXED32):
-    case UPB_TYPE(UINT32):
-      ASSERT(r->GetUInt32(proto2_msg, proto2_f) == upb_value_getuint32(v));
-      break;
-    case UPB_TYPE(BOOL):
-      ASSERT(r->GetBool(proto2_msg, proto2_f) == upb_value_getbool(v));
-      break;
-    case UPB_TYPE(STRING):
-    case UPB_TYPE(BYTES): {
-      std::string str = r->GetString(proto2_msg, proto2_f);
-      upb_stdarray *upbstr = (upb_stdarray*)upb_value_getptr(v);
-      std::string str2(upbstr->ptr, upbstr->len);
-      string_size += upbstr->len;
-      ASSERT(str == str2);
-      break;
-    }
-    case UPB_TYPE(GROUP):
-    case UPB_TYPE(MESSAGE):
-      // XXX: getstr
-      compare(r->GetMessage(proto2_msg, proto2_f),
-              upb_value_getptr(v), upb_downcast_msgdef(upb_f->def));
-  }
-}
-
-void compare(const google::protobuf::Message& proto2_msg,
-             void *upb_msg, const upb_msgdef *upb_md)
-{
-  const google::protobuf::Reflection *r = proto2_msg.GetReflection();
-  const google::protobuf::Descriptor *d = proto2_msg.GetDescriptor();
-
-  ASSERT(d->field_count() == upb_msgdef_numfields(upb_md));
-  upb_msg_iter i;
-  for(i = upb_msg_begin(upb_md); !upb_msg_done(i); i = upb_msg_next(upb_md, i)) {
-    upb_fielddef *upb_f = upb_msg_iter_field(i);
+void compare_metadata(const google::protobuf::Descriptor* d,
+                      const upb::MessageDef *upb_md) {
+  ASSERT(d->field_count() == upb_md->field_count());
+  for (upb::MessageDef::ConstIterator i(upb_md); !i.Done(); i.Next()) {
+    const upb::FieldDef* upb_f = i.field();
     const google::protobuf::FieldDescriptor *proto2_f =
-        d->FindFieldByNumber(upb_f->number);
-    // Make sure the definitions are equal.
+        d->FindFieldByNumber(upb_f->number());
     ASSERT(upb_f);
     ASSERT(proto2_f);
-    ASSERT(upb_f->number == proto2_f->number());
-    ASSERT(std::string(upb_f->name) == proto2_f->name());
-    ASSERT(upb_f->type == proto2_f->type());
-    ASSERT(upb_isseq(upb_f) == proto2_f->is_repeated());
-
-    if(!upb_msg_has(upb_msg, upb_f)) {
-      if(upb_isseq(upb_f))
-        ASSERT(r->FieldSize(proto2_msg, proto2_f) == 0);
-      else
-        ASSERT(r->HasField(proto2_msg, proto2_f) == false);
-    } else {
-      if(upb_isseq(upb_f)) {
-        compare_arrays(r, proto2_msg, proto2_f, upb_msg, upb_f);
-      } else {
-        ASSERT(r->HasField(proto2_msg, proto2_f) == true);
-        compare_values(r, proto2_msg, proto2_f, upb_msg, upb_f);
-      }
-    }
+    ASSERT(upb_f->number() == proto2_f->number());
+    ASSERT(std::string(upb_f->name()) == proto2_f->name());
+    ASSERT(upb_f->type() == static_cast<upb::FieldType>(proto2_f->type()));
+    ASSERT(upb_f->IsSequence() == proto2_f->is_repeated());
   }
 }
 
-void parse_and_compare(MESSAGE_CIDENT *proto2_msg,
-                       void *upb_msg, const upb_msgdef *upb_md,
-                       const char *str, size_t len, bool allow_jit)
-{
+void parse_and_compare(MESSAGE_CIDENT *msg1, MESSAGE_CIDENT *msg2,
+                       const upb::MessageDef *upb_md,
+                       const char *str, size_t len, bool allow_jit) {
   // Parse to both proto2 and upb.
-  ASSERT(proto2_msg->ParseFromArray(str, len));
-  upb_status status = UPB_STATUS_INIT;
-  upb_msg_clear(upb_msg, upb_md);
-  upb_strtomsg(str, len, upb_msg, upb_md, allow_jit, &status);
-  if (!upb_ok(&status)) {
-    fprintf(stderr, "Error parsing protobuf: %s", upb_status_getstr(&status));
-    exit(1);
-  }
-  string_size = 0;
-  compare(*proto2_msg, upb_msg, upb_md);
-  printf("Total size: %zd, string size: %zd (%0.2f%%)\n", len,
-         string_size, (double)string_size / len * 100);
-  upb_status_uninit(&status);
+  ASSERT(msg1->ParseFromArray(str, len));
+
+  upb::Handlers* handlers = upb::Handlers::New();
+  upb::RegisterWriteHandlers(handlers, upb_md);
+  upb::DecoderPlan* plan = upb::DecoderPlan::New(handlers, allow_jit);
+  upb::StringSource src(str, len);
+  upb::Decoder decoder;
+  decoder.ResetPlan(plan, 0);
+  decoder.ResetInput(src.AllBytes(), msg2);
+  msg2->Clear();
+  ASSERT(decoder.Decode() == UPB_OK);
+  plan->Unref();
+  handlers->Unref();
+
+  // Would like to just compare the message objects themselves,  but
+  // unfortunately MessageDifferencer is not part of the open-source release of
+  // proto2, so we compare their serialized strings, which we expect will be
+  // equivalent.
+  std::string str1;
+  std::string str2;
+  msg1->SerializeToString(&str1);
+  msg2->SerializeToString(&str2);
+  ASSERT(str1 == str2);
+  ASSERT(std::string(str, len) == str2);
 }
 
-int main(int argc, char *argv[])
-{
-  if (argc < 3) {
-    fprintf(stderr, "Usage: test_vs_proto2 <descriptor file> <message file>\n");
-    return 1;
+void test_zig_zag() {
+  for (uint64_t num = 5; num * 1.5 > num; num *= 1.5) {
+    ASSERT(upb_zzenc_64(num) ==
+           google::protobuf::internal::WireFormatLite::ZigZagEncode64(num));
+    if (num < UINT32_MAX) {
+      ASSERT(upb_zzenc_32(num) ==
+             google::protobuf::internal::WireFormatLite::ZigZagEncode32(num));
+    }
   }
-  const char *descriptor_file = argv[1];
-  const char *message_file = argv[2];
 
-  // Initialize upb state, parse descriptor.
-  upb_status status = UPB_STATUS_INIT;
-  upb_symtab *symtab = upb_symtab_new();
-  size_t fds_len;
-  const char *fds = upb_readfile(descriptor_file, &fds_len);
-  if(fds == NULL) {
-    fprintf(stderr, "Couldn't read %s.\n", descriptor_file);
-    return 1;
-  }
-  upb_load_descriptor_into_symtab(symtab, fds, fds_len, &status);
-  if(!upb_ok(&status)) {
-    fprintf(stderr, "Error importing %s: %s", descriptor_file,
-            upb_status_getstr(&status));
-    return 1;
-  }
-  free((void*)fds);
+}
 
-  const upb_def *def = upb_symtab_lookup(symtab, MESSAGE_NAME);
-  const upb_msgdef *msgdef;
-  if(!def || !(msgdef = upb_dyncast_msgdef_const(def))) {
-    fprintf(stderr, "Error finding symbol '%s'.\n", MESSAGE_NAME);
+int main(int argc, char *argv[])
+{
+  if (argc < 2) {
+    fprintf(stderr, "Usage: test_vs_proto2 <message file>\n");
     return 1;
   }
+  const char *message_file = argv[1];
 
   // Read the message data itself.
   size_t len;
@@ -242,32 +99,25 @@ int main(int argc, char *argv[])
     return 1;
   }
 
+  MESSAGE_CIDENT msg1;
+  MESSAGE_CIDENT msg2;
+
+  const upb::MessageDef* m = upb::proto2_bridge::NewFinalMessageDef(msg1, &m);
+
+  compare_metadata(msg1.GetDescriptor(), m);
+
   // Run twice to test proper object reuse.
-  MESSAGE_CIDENT proto2_msg;
-  void *upb_msg = upb_stdmsg_new(msgdef);
-  parse_and_compare(&proto2_msg, upb_msg, msgdef, str, len, true);
-  parse_and_compare(&proto2_msg, upb_msg, msgdef, str, len, false);
-  parse_and_compare(&proto2_msg, upb_msg, msgdef, str, len, true);
-  parse_and_compare(&proto2_msg, upb_msg, msgdef, str, len, false);
+  parse_and_compare(&msg1, &msg2, m, str, len, true);
+  parse_and_compare(&msg1, &msg2, m, str, len, false);
+  parse_and_compare(&msg1, &msg2, m, str, len, true);
+  parse_and_compare(&msg1, &msg2, m, str, len, false);
   printf("All tests passed, %d assertions.\n", num_assertions);
 
-  upb_stdmsg_free(upb_msg, msgdef);
-  upb_def_unref(UPB_UPCAST(msgdef));
+  m->Unref(&m);
   free((void*)str);
-  upb_symtab_unref(symtab);
-  upb_status_uninit(&status);
 
-  // Test Zig-Zag encoding/decoding.
-  for (uint64_t num = 5; num * 1.5 > num; num *= 1.5) {
-    ASSERT(upb_zzenc_64(num) ==
-           google::protobuf::internal::WireFormatLite::ZigZagEncode64(num));
-    if (num < UINT32_MAX) {
-      ASSERT(upb_zzenc_32(num) ==
-             google::protobuf::internal::WireFormatLite::ZigZagEncode32(num));
-    }
-  }
+  test_zig_zag();
 
   google::protobuf::ShutdownProtobufLibrary();
-
   return 0;
 }