Sync from Google development.

2 files changed, 339 insertions, 217 deletions

diff --git a/tests/pb/test_decoder.cc b/tests/pb/test_decoder.cc
index eb95580..1660436 100644
--- a/tests/pb/test_decoder.cc
+++ b/tests/pb/test_decoder.cc
@@ -37,22 +37,25 @@
 #include <string.h>
 
 #include "tests/upb_test.h"
-#include "third_party/upb/tests/pb/test_decoder_schema.upb.h"
 #include "upb/handlers.h"
 #include "upb/pb/decoder.h"
 #include "upb/pb/varint.int.h"
 #include "upb/upb.h"
 
 #undef PRINT_FAILURE
-#define PRINT_FAILURE(expr) \
-  fprintf(stderr, "Assertion failed: %s:%d\n", __FILE__, __LINE__); \
-  fprintf(stderr, "expr: %s\n", #expr); \
-  if (testhash) { \
-    fprintf(stderr, "assertion failed running test %x.  " \
-                    "Run with the arg %x to run only this test.\n", \
-                    testhash, testhash); \
-    fprintf(stderr, "Failed at %02.2f%% through tests.\n", \
-                    (float)completed * 100 / total); \
+#define PRINT_FAILURE(expr)                                           \
+  fprintf(stderr, "Assertion failed: %s:%d\n", __FILE__, __LINE__);   \
+  fprintf(stderr, "expr: %s\n", #expr);                               \
+  if (testhash) {                                                     \
+    fprintf(stderr, "assertion failed running test %x.\n", testhash); \
+    if (!filter_hash) {                                               \
+      fprintf(stderr,                                                 \
+              "Run with the arg %x to run only this test. "           \
+              "(This will also turn on extra debugging output)\n",    \
+              testhash);                                              \
+    }                                                                 \
+    fprintf(stderr, "Failed at %02.2f%% through tests.\n",            \
+            (float)completed * 100 / total);                          \
   }
 
 uint32_t filter_hash = 0;
@@ -90,106 +93,101 @@ static const upb_decoder_typeinfo upb_decoder_types[] = {
   {UPB_WIRE_TYPE_VARINT,      true},   // SINT64
 };
 
+#ifndef USE_GOOGLE
+using std::string;
+#endif
 
-class buffer {
- public:
-  buffer(const void *data, size_t len) : len_(0) { append(data, len); }
-  explicit buffer(const char *data) : len_(0) { append(data); }
-  explicit buffer(size_t len) : len_(len) { memset(buf_, 0, len); }
-  buffer(const buffer& buf) : len_(0) { append(buf); }
-  buffer() : len_(0) {}
-
-  void append(const void *data, size_t len) {
-    ASSERT_NOCOUNT(len + len_ < sizeof(buf_));
-    memcpy(buf_ + len_, data, len);
-    len_ += len;
-    buf_[len_] = NULL;
-  }
-
-  void append(const buffer& buf) {
-    append(buf.buf_, buf.len_);
-  }
-
-  void append(const char *str) {
-    append(str, strlen(str));
-  }
-
-  void vappendf(const char *fmt, va_list args) {
-    size_t avail = sizeof(buf_) - len_;
-    size_t size = vsnprintf(buf_ + len_, avail, fmt, args);
-    ASSERT_NOCOUNT(avail > size);
-    len_ += size;
-  }
+void vappendf(string* str, const char *format, va_list args) {
+  va_list copy;
+  va_copy(copy, args);
 
-  void appendf(const char *fmt, ...) {
-    va_list args;
-    va_start(args, fmt);
-    vappendf(fmt, args);
-    va_end(args);
+  int count = vsnprintf(NULL, 0, format, args);
+  if (count >= 0)
+  {
+    assert(count < 32768);
+    char *buffer = new char[count + 1];
+    assert(buffer);
+    count = vsnprintf(buffer, count + 1, format, copy);
+    assert(count >= 0);
+    str->append(buffer, count);
+    delete [] buffer;
   }
+  va_end(copy);
+}
 
-  void assign(const buffer& buf) {
-    clear();
-    append(buf);
-  }
+void appendf(string* str, const char *fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  vappendf(str, fmt, args);
+  va_end(args);
+}
 
-  bool eql(const buffer& other) const {
-    return len_ == other.len_ && memcmp(buf_, other.buf_, len_) == 0;
+void PrintBinary(const string& str) {
+  for (size_t i = 0; i < str.size(); i++) {
+    if (isprint(str[i])) {
+      fprintf(stderr, "%c", str[i]);
+    } else {
+      fprintf(stderr, "\\x%02x", str[i]);
+    }
   }
-
-  void clear() { len_ = 0; }
-  size_t len() const { return len_; }
-  const char *buf() const { return buf_; }
-
- private:
-  // Has to be big enough for the largest string used in the test.
-  char buf_[32768];
-  size_t len_;
-};
-
+}
 
 /* Routines for building arbitrary protos *************************************/
 
-const buffer empty;
-
-buffer cat(const buffer& a, const buffer& b,
-           const buffer& c = empty,
-           const buffer& d = empty,
-           const buffer& e = empty,
-           const buffer& f = empty) {
-  buffer ret;
+const string empty;
+
+string cat(const string& a, const string& b,
+           const string& c = empty,
+           const string& d = empty,
+           const string& e = empty,
+           const string& f = empty,
+           const string& g = empty,
+           const string& h = empty,
+           const string& i = empty,
+           const string& j = empty,
+           const string& k = empty,
+           const string& l = empty) {
+  string ret;
+  ret.reserve(a.size() + b.size() + c.size() + d.size() + e.size() + f.size() +
+              g.size() + h.size() + i.size() + j.size() + k.size() + l.size());
   ret.append(a);
   ret.append(b);
   ret.append(c);
   ret.append(d);
   ret.append(e);
   ret.append(f);
+  ret.append(g);
+  ret.append(h);
+  ret.append(i);
+  ret.append(j);
+  ret.append(k);
+  ret.append(l);
   return ret;
 }
 
-buffer varint(uint64_t x) {
+string varint(uint64_t x) {
   char buf[UPB_PB_VARINT_MAX_LEN];
   size_t len = upb_vencode64(x, buf);
-  return buffer(buf, len);
+  return string(buf, len);
 }
 
 // TODO: proper byte-swapping for big-endian machines.
-buffer fixed32(void *data) { return buffer(data, 4); }
-buffer fixed64(void *data) { return buffer(data, 8); }
-
-buffer delim(const buffer& buf) { return cat(varint(buf.len()), buf); }
-buffer uint32(uint32_t u32) { return fixed32(&u32); }
-buffer uint64(uint64_t u64) { return fixed64(&u64); }
-buffer flt(float f) { return fixed32(&f); }
-buffer dbl(double d) { return fixed64(&d); }
-buffer zz32(int32_t x) { return varint(upb_zzenc_32(x)); }
-buffer zz64(int64_t x) { return varint(upb_zzenc_64(x)); }
-
-buffer tag(uint32_t fieldnum, char wire_type) {
+string fixed32(void *data) { return string(static_cast<char*>(data), 4); }
+string fixed64(void *data) { return string(static_cast<char*>(data), 8); }
+
+string delim(const string& buf) { return cat(varint(buf.size()), buf); }
+string uint32(uint32_t u32) { return fixed32(&u32); }
+string uint64(uint64_t u64) { return fixed64(&u64); }
+string flt(float f) { return fixed32(&f); }
+string dbl(double d) { return fixed64(&d); }
+string zz32(int32_t x) { return varint(upb_zzenc_32(x)); }
+string zz64(int64_t x) { return varint(upb_zzenc_64(x)); }
+
+string tag(uint32_t fieldnum, char wire_type) {
   return varint((fieldnum << 3) | wire_type);
 }
 
-buffer submsg(uint32_t fn, const buffer& buf) {
+string submsg(uint32_t fn, const string& buf) {
   return cat( tag(fn, UPB_WIRE_TYPE_DELIMITED), delim(buf) );
 }
 
@@ -203,18 +201,17 @@ buffer submsg(uint32_t fn, const buffer& buf) {
 // handled.
 
 int closures[UPB_DECODER_MAX_NESTING];
-buffer output;
+string output;
 
-void indentbuf(buffer *buf, int depth) {
-  for (int i = 0; i < depth; i++)
-    buf->append("  ", 2);
+void indentbuf(string *buf, int depth) {
+  buf->append(2 * depth, ' ');
 }
 
-#define NUMERIC_VALUE_HANDLER(member, ctype, fmt) \
-  bool value_ ## member(int* depth, const uint32_t* num, ctype val) {          \
-    indentbuf(&output, *depth);                                                \
-    output.appendf("%" PRIu32 ":%" fmt "\n", *num, val);                       \
-    return true;                                                               \
+#define NUMERIC_VALUE_HANDLER(member, ctype, fmt)                   \
+  bool value_##member(int* depth, const uint32_t* num, ctype val) { \
+    indentbuf(&output, *depth);                                     \
+    appendf(&output, "%" PRIu32 ":%" fmt "\n", *num, val);          \
+    return true;                                                    \
   }
 
 NUMERIC_VALUE_HANDLER(uint32, uint32_t, PRIu32)
@@ -226,19 +223,20 @@ NUMERIC_VALUE_HANDLER(double, double,   "g")
 
 bool value_bool(int* depth, const uint32_t* num, bool val) {
   indentbuf(&output, *depth);
-  output.appendf("%" PRIu32 ":%s\n", *num, val ? "true" : "false");
+  appendf(&output, "%" PRIu32 ":%s\n", *num, val ? "true" : "false");
   return true;
 }
 
 int* startstr(int* depth, const uint32_t* num, size_t size_hint) {
   indentbuf(&output, *depth);
-  output.appendf("%" PRIu32 ":(%zu)\"", *num, size_hint);
+  appendf(&output, "%" PRIu32 ":(%zu)\"", *num, size_hint);
   return depth + 1;
 }
 
 size_t value_string(int* depth, const uint32_t* num, const char* buf,
                     size_t n, const upb::BufferHandle* handle) {
   UPB_UNUSED(num);
+  UPB_UNUSED(depth);
   output.append(buf, n);
   ASSERT(handle == &global_handle);
   return n;
@@ -253,7 +251,7 @@ bool endstr(int* depth, const uint32_t* num) {
 
 int* startsubmsg(int* depth, const uint32_t* num) {
   indentbuf(&output, *depth);
-  output.appendf("%" PRIu32 ":{\n", *num);
+  appendf(&output, "%" PRIu32 ":{\n", *num);
   return depth + 1;
 }
 
@@ -266,7 +264,7 @@ bool endsubmsg(int* depth, const uint32_t* num) {
 
 int* startseq(int* depth, const uint32_t* num) {
   indentbuf(&output, *depth);
-  output.appendf("%" PRIu32 ":[\n", *num);
+  appendf(&output, "%" PRIu32 ":[\n", *num);
   return depth + 1;
 }
 
@@ -284,6 +282,7 @@ bool startmsg(int* depth) {
 }
 
 bool endmsg(int* depth, upb_status* status) {
+  UPB_UNUSED(status);
   indentbuf(&output, *depth);
   output.append(">\n");
   return true;
@@ -298,7 +297,7 @@ template<class T, bool F(int*, const uint32_t*, T)>
 void doreg(upb_handlers *h, uint32_t num) {
   const upb_fielddef *f = upb_msgdef_itof(upb_handlers_msgdef(h), num);
   ASSERT(f);
-  ASSERT(h->SetValueHandler<T>(f, UpbBindT(F, new uint32_t(num))));
+  ASSERT(h->SetValueHandler<T>(f, UpbBindT(&F, new uint32_t(num))));
   if (f->IsSequence()) {
     ASSERT(h->SetStartSequenceHandler(f, UpbBind(startseq, new uint32_t(num))));
     ASSERT(h->SetEndSequenceHandler(f, UpbBind(endseq, new uint32_t(num))));
@@ -347,9 +346,93 @@ void reg_str(upb_handlers *h, uint32_t num) {
   ASSERT(h->SetStringHandler(f, UpbBind(value_string, new uint32_t(num))));
 }
 
+void AddField(upb_descriptortype_t descriptor_type, const std::string& name,
+              uint32_t fn, bool repeated, upb::MessageDef* md) {
+  // TODO: Fluent interface?  ie.
+  //   ASSERT(md->AddField(upb::BuildFieldDef()
+  //       .SetName("f_message")
+  //       .SetNumber(UPB_DESCRIPTOR_TYPE_MESSAGE)
+  //       .SetDescriptorType(UPB_DESCRIPTOR_TYPE_MESSAGE)
+  //       .SetMessageSubdef(md.get())));
+  upb::reffed_ptr<upb::FieldDef> f = upb::FieldDef::New();
+  ASSERT(f->set_name(name, NULL));
+  ASSERT(f->set_number(fn, NULL));
+  f->set_label(repeated ? UPB_LABEL_REPEATED : UPB_LABEL_OPTIONAL);
+  f->set_descriptor_type(descriptor_type);
+  ASSERT(md->AddField(f.get(), NULL));
+}
+
+void AddFieldsForType(upb_descriptortype_t descriptor_type,
+                      const char* basename, upb::MessageDef* md) {
+  const upb_descriptortype_t t = descriptor_type;
+  AddField(t, std::string("f_") + basename, t, false, md);
+  AddField(t, std::string("r_") + basename, rep_fn(t), true, md);
+}
+
+upb::reffed_ptr<const upb::MessageDef> NewMessageDef() {
+  upb::reffed_ptr<upb::MessageDef> md = upb::MessageDef::New();
+
+  md->set_full_name("DecoderTest", NULL);
+
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_DOUBLE, "double", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_FLOAT, "float", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_INT64, "int64", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_UINT64, "uint64", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_INT32, "int32", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_FIXED64, "fixed64", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_FIXED32, "fixed32", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_BOOL, "bool", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_STRING, "string", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_BYTES, "bytes", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_UINT32, "uint32", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_SFIXED32, "sfixed32", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_SFIXED64, "sfixed64", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_SINT32, "sint32", md.get());
+  AddFieldsForType(UPB_DESCRIPTOR_TYPE_SINT64, "sint64", md.get());
+
+  AddField(UPB_DESCRIPTOR_TYPE_STRING, "nop_field", 40, false, md.get());
+
+  upb::reffed_ptr<upb::FieldDef> f = upb::FieldDef::New();
+  ASSERT(f->set_name("f_message", NULL));
+  ASSERT(f->set_number(UPB_DESCRIPTOR_TYPE_MESSAGE, NULL));
+  f->set_descriptor_type(UPB_DESCRIPTOR_TYPE_MESSAGE);
+  ASSERT(f->set_message_subdef(md.get(), NULL));
+  ASSERT(md->AddField(f.get(), NULL));
+
+  f = upb::FieldDef::New();
+  ASSERT(f->set_name("r_message", NULL));
+  ASSERT(f->set_number(rep_fn(UPB_DESCRIPTOR_TYPE_MESSAGE), NULL));
+  f->set_label(UPB_LABEL_REPEATED);
+  f->set_descriptor_type(UPB_DESCRIPTOR_TYPE_MESSAGE);
+  ASSERT(f->set_message_subdef(md.get(), NULL));
+  ASSERT(md->AddField(f.get(), NULL));
+
+  upb::reffed_ptr<upb::EnumDef> e = upb::EnumDef::New();
+  ASSERT(e->AddValue("FOO", 1, NULL));
+  ASSERT(e->Freeze(NULL));
+
+  f = upb::FieldDef::New();
+  ASSERT(f->set_name("f_enum", NULL));
+  ASSERT(f->set_number(UPB_DESCRIPTOR_TYPE_ENUM, NULL));
+  f->set_descriptor_type(UPB_DESCRIPTOR_TYPE_ENUM);
+  ASSERT(f->set_enum_subdef(e.get(), NULL));
+  ASSERT(md->AddField(f.get(), NULL));
+
+  f = upb::FieldDef::New();
+  ASSERT(f->set_name("r_enum", NULL));
+  ASSERT(f->set_number(rep_fn(UPB_DESCRIPTOR_TYPE_ENUM), NULL));
+  f->set_label(UPB_LABEL_REPEATED);
+  f->set_descriptor_type(UPB_DESCRIPTOR_TYPE_ENUM);
+  ASSERT(f->set_enum_subdef(e.get(), NULL));
+  ASSERT(md->AddField(f.get(), NULL));
+
+  ASSERT(md->Freeze(NULL));
+
+  return md;
+}
+
 upb::reffed_ptr<const upb::Handlers> NewHandlers() {
-  upb::reffed_ptr<upb::Handlers> h(
-      upb::Handlers::New(UPB_TEST_DECODER_DECODERTEST));
+  upb::reffed_ptr<upb::Handlers> h(upb::Handlers::New(NewMessageDef().get()));
 
   h->SetStartMessageHandler(UpbMakeHandler(startmsg));
   h->SetEndMessageHandler(UpbMakeHandler(endmsg));
@@ -395,11 +478,11 @@ upb::reffed_ptr<const upb::Handlers> NewHandlers() {
 const upb::Handlers *global_handlers;
 const upb::pb::DecoderMethod *global_method;
 
-uint32_t Hash(const buffer& proto, const buffer* expected_output, size_t seam1,
+uint32_t Hash(const string& proto, const string* expected_output, size_t seam1,
               size_t seam2) {
-  uint32_t hash = MurmurHash2(proto.buf(), proto.len(), 0);
+  uint32_t hash = MurmurHash2(proto.c_str(), proto.size(), 0);
   if (expected_output)
-    hash = MurmurHash2(expected_output->buf(), expected_output->len(), hash);
+    hash = MurmurHash2(expected_output->c_str(), expected_output->size(), hash);
   hash = MurmurHash2(&seam1, sizeof(seam1), hash);
   hash = MurmurHash2(&seam2, sizeof(seam2), hash);
   return hash;
@@ -410,8 +493,38 @@ bool parse(upb::BytesSink* s, void* subc, const char* buf, size_t start,
   start = UPB_MAX(start, *ofs);
   if (start <= end) {
     size_t len = end - start;
+    if (filter_hash) {
+      fprintf(stderr, "Calling parse(%zu) for bytes %zu-%zu of the input\n",
+              len, start, end);
+    }
     size_t parsed = s->PutBuffer(subc, buf + start, len, &global_handle);
+    if (filter_hash) {
+      if (parsed == len) {
+        fprintf(stderr,
+                "parse(%zu) = %zu, complete byte count indicates success\n",
+                len, len);
+      } else if (parsed > len) {
+        fprintf(stderr,
+                "parse(%zu) = %zu, long byte count indicates success and skip"
+                "of the next %zu bytes\n",
+                len, parsed, parsed - len);
+      } else {
+        fprintf(stderr,
+                "parse(%zu) = %zu, short byte count indicates failure; "
+                "last %zu bytes were not consumed\n",
+                len, parsed, len - parsed);
+      }
+    }
     if (status->ok() != (parsed >= len)) {
+      if (status->ok()) {
+        fprintf(stderr,
+                "Error: decode function returned short byte count but set no "
+                "error status\n");
+      } else {
+        fprintf(stderr,
+                "Error: decode function returned complete byte count but set "
+                "error status\n");
+      }
       fprintf(stderr, "Status: %s, parsed=%zu, len=%zu\n",
               status->error_message(), parsed, len);
       ASSERT(false);
@@ -424,13 +537,13 @@ bool parse(upb::BytesSink* s, void* subc, const char* buf, size_t start,
 }
 
 #define LINE(x) x "\n"
-void run_decoder(const buffer& proto, const buffer* expected_output) {
+void run_decoder(const string& proto, const string* expected_output) {
   upb::Status status;
   upb::pb::Decoder decoder(global_method, &status);
   upb::Sink sink(global_handlers, &closures[0]);
   decoder.ResetOutput(&sink);
-  for (size_t i = 0; i < proto.len(); i++) {
-    for (size_t j = i; j < UPB_MIN(proto.len(), i + 5); j++) {
+  for (size_t i = 0; i < proto.size(); i++) {
+    for (size_t j = i; j < UPB_MIN(proto.size(), i + 5); j++) {
       testhash = Hash(proto, expected_output, i, j);
       if (filter_hash && testhash != filter_hash) continue;
       if (!count_only) {
@@ -440,27 +553,50 @@ void run_decoder(const buffer& proto, const buffer* expected_output) {
         size_t ofs = 0;
         upb::BytesSink* input = decoder.input();
         void *sub;
+
+        if (filter_hash) {
+          fprintf(stderr, "RUNNING TEST CASE, hash=%x\n", testhash);
+          fprintf(stderr, "JIT on: %s\n",
+                  global_method->is_native() ? "true" : "false");
+          fprintf(stderr, "Input (len=%zu): ", proto.size());
+          PrintBinary(proto);
+          fprintf(stderr, "\n");
+          if (expected_output) {
+            fprintf(stderr, "Expected output: %s\n", expected_output->c_str());
+          } else {
+            fprintf(stderr, "Expected to FAIL\n");
+          }
+          fprintf(stderr, "Calling start()\n");
+        }
+
         bool ok =
-            input->Start(proto.len(), &sub) &&
-            parse(input, sub, proto.buf(), 0, i, &ofs, &status) &&
-            parse(input, sub, proto.buf(), i, j, &ofs, &status) &&
-            parse(input, sub, proto.buf(), j, proto.len(), &ofs, &status) &&
-            ofs == proto.len() &&
-            input->End();
+            input->Start(proto.size(), &sub) &&
+            parse(input, sub, proto.c_str(), 0, i, &ofs, &status) &&
+            parse(input, sub, proto.c_str(), i, j, &ofs, &status) &&
+            parse(input, sub, proto.c_str(), j, proto.size(), &ofs, &status) &&
+            ofs == proto.size();
+
+        if (ok) {
+          if (filter_hash) {
+            fprintf(stderr, "calling end()\n");
+          }
+          ok = input->End();
+        }
+
         if (expected_output) {
-          if (!output.eql(*expected_output)) {
+          if (output != *expected_output) {
             fprintf(stderr, "Text mismatch: '%s' vs '%s'\n",
-                    output.buf(), expected_output->buf());
+                    output.c_str(), expected_output->c_str());
           }
           if (!ok) {
             fprintf(stderr, "Failed: %s\n", status.error_message());
           }
           ASSERT(ok);
-          ASSERT(output.eql(*expected_output));
+          ASSERT(output == *expected_output);
         } else {
           if (ok) {
             fprintf(stderr, "Didn't expect ok result, but got output: '%s'\n",
-                    output.buf());
+                    output.c_str());
           }
           ASSERT(!ok);
         }
@@ -471,15 +607,15 @@ void run_decoder(const buffer& proto, const buffer* expected_output) {
   testhash = 0;
 }
 
-const static buffer thirty_byte_nop = buffer(cat(
-    tag(NOP_FIELD, UPB_WIRE_TYPE_DELIMITED), delim(buffer(30)) ));
+const static string thirty_byte_nop = cat(
+    tag(NOP_FIELD, UPB_WIRE_TYPE_DELIMITED), delim(string(30, 'X')) );
 
-void assert_successful_parse(const buffer& proto,
+void assert_successful_parse(const string& proto,
                              const char *expected_fmt, ...) {
-  buffer expected_text;
+  string expected_text;
   va_list args;
   va_start(args, expected_fmt);
-  expected_text.vappendf(expected_fmt, args);
+  vappendf(&expected_text, expected_fmt, args);
   va_end(args);
   // To test both middle-of-buffer and end-of-buffer code paths,
   // repeat once with no-op padding data at the end of buffer.
@@ -487,11 +623,11 @@ void assert_successful_parse(const buffer& proto,
   run_decoder(cat( proto, thirty_byte_nop ), &expected_text);
 }
 
-void assert_does_not_parse_at_eof(const buffer& proto) {
+void assert_does_not_parse_at_eof(const string& proto) {
   run_decoder(proto, NULL);
 }
 
-void assert_does_not_parse(const buffer& proto) {
+void assert_does_not_parse(const string& proto) {
   // Test that the error is caught both at end-of-buffer and middle-of-buffer.
   assert_does_not_parse_at_eof(proto);
   assert_does_not_parse_at_eof(cat( proto, thirty_byte_nop ));
@@ -502,19 +638,19 @@ void assert_does_not_parse(const buffer& proto) {
 
 void test_premature_eof_for_type(upb_descriptortype_t type) {
   // Incomplete values for each wire type.
-  static const buffer incompletes[6] = {
-    buffer("\x80"),     // UPB_WIRE_TYPE_VARINT
-    buffer("abcdefg"),  // UPB_WIRE_TYPE_64BIT
-    buffer("\x80"),     // UPB_WIRE_TYPE_DELIMITED (partial length)
-    buffer(),           // UPB_WIRE_TYPE_START_GROUP (no value required)
-    buffer(),           // UPB_WIRE_TYPE_END_GROUP (no value required)
-    buffer("abc")       // UPB_WIRE_TYPE_32BIT
+  static const string incompletes[6] = {
+    string("\x80"),     // UPB_WIRE_TYPE_VARINT
+    string("abcdefg"),  // UPB_WIRE_TYPE_64BIT
+    string("\x80"),     // UPB_WIRE_TYPE_DELIMITED (partial length)
+    string(),           // UPB_WIRE_TYPE_START_GROUP (no value required)
+    string(),           // UPB_WIRE_TYPE_END_GROUP (no value required)
+    string("abc")       // UPB_WIRE_TYPE_32BIT
   };
 
   uint32_t fieldnum = type;
   uint32_t rep_fieldnum = rep_fn(type);
   int wire_type = upb_decoder_types[type].native_wire_type;
-  const buffer& incomplete = incompletes[wire_type];
+  const string& incomplete = incompletes[wire_type];
 
   // EOF before a known non-repeated value.
   assert_does_not_parse_at_eof(tag(fieldnum, wire_type));
@@ -552,19 +688,19 @@ void test_premature_eof_for_type(upb_descriptortype_t type) {
 
     if (type == UPB_DESCRIPTOR_TYPE_MESSAGE) {
       // Submessage ends in the middle of a value.
-      buffer incomplete_submsg =
+      string incomplete_submsg =
           cat ( tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT),
                 incompletes[UPB_WIRE_TYPE_VARINT] );
       assert_does_not_parse(
           cat( tag(fieldnum, UPB_WIRE_TYPE_DELIMITED),
-               varint(incomplete_submsg.len()),
+               varint(incomplete_submsg.size()),
                incomplete_submsg ));
     }
   } else {
     // Packed region ends in the middle of a value.
     assert_does_not_parse(
         cat( tag(rep_fieldnum, UPB_WIRE_TYPE_DELIMITED),
-             varint(incomplete.len()),
+             varint(incomplete.size()),
              incomplete ));
 
     // EOF in the middle of packed region.
@@ -576,7 +712,7 @@ void test_premature_eof_for_type(upb_descriptortype_t type) {
 // "33" and "66" are just two random values that all numeric types can
 // represent.
 void test_valid_data_for_type(upb_descriptortype_t type,
-                              const buffer& enc33, const buffer& enc66) {
+                              const string& enc33, const string& enc66) {
   uint32_t fieldnum = type;
   uint32_t rep_fieldnum = rep_fn(type);
   int wire_type = upb_decoder_types[type].native_wire_type;
@@ -614,7 +750,7 @@ void test_valid_data_for_type(upb_descriptortype_t type,
 }
 
 void test_valid_data_for_signed_type(upb_descriptortype_t type,
-                                     const buffer& enc33, const buffer& enc66) {
+                                     const string& enc33, const string& enc66) {
   uint32_t fieldnum = type;
   uint32_t rep_fieldnum = rep_fn(type);
   int wire_type = upb_decoder_types[type].native_wire_type;
@@ -672,15 +808,14 @@ void test_invalid() {
   test_premature_eof_for_type(UPB_DESCRIPTOR_TYPE_SINT64);
 
   // EOF inside a tag's varint.
-  assert_does_not_parse_at_eof( buffer("\x80") );
+  assert_does_not_parse_at_eof( string("\x80") );
 
   // EOF inside a known group.
   // TODO(haberman): add group to decoder test schema.
   //assert_does_not_parse_at_eof( tag(4, UPB_WIRE_TYPE_START_GROUP) );
 
   // EOF inside an unknown group.
-  // TODO(haberman): unknown groups not supported yet.
-  //assert_does_not_parse_at_eof( tag(UNKNOWN_FIELD, UPB_WIRE_TYPE_START_GROUP) );
+  assert_does_not_parse_at_eof( tag(UNKNOWN_FIELD, UPB_WIRE_TYPE_START_GROUP) );
 
   // End group that we are not currently in.
   assert_does_not_parse( tag(4, UPB_WIRE_TYPE_END_GROUP) );
@@ -701,7 +836,7 @@ void test_invalid() {
             tag(UPB_DESCRIPTOR_TYPE_GROUP, UPB_WIRE_TYPE_END_GROUP)));
 
   // Test exceeding the resource limit of stack depth.
-  buffer buf;
+  string buf;
   for (int i = 0; i <= UPB_DECODER_MAX_NESTING; i++) {
     buf.assign(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, buf));
   }
@@ -710,11 +845,16 @@ void test_invalid() {
 
 void test_valid() {
   // Empty protobuf.
-  assert_successful_parse(buffer(""), "<\n>\n");
+  assert_successful_parse(string(""), "<\n>\n");
 
   // Empty protobuf where we never call PutString between
   // StartString/EndString.
-  {
+
+  // Randomly generated hash for this test, hope it doesn't conflict with others
+  // by chance.
+  const uint32_t emptyhash = 0x5709be8e;
+  if (!filter_hash || filter_hash == testhash) {
+    testhash = emptyhash;
     upb::Status status;
     upb::pb::Decoder decoder(global_method, &status);
     upb::Sink sink(global_handlers, &closures[0]);
@@ -723,7 +863,7 @@ void test_valid() {
     bool ok = upb::BufferSource::PutBuffer("", 0, decoder.input());
     ASSERT(ok);
     ASSERT(status.ok());
-    ASSERT(output.eql(buffer("<\n>\n")));
+    ASSERT(output == string("<\n>\n"));
   }
 
   test_valid_data_for_signed_type(UPB_DESCRIPTOR_TYPE_DOUBLE,
@@ -770,7 +910,7 @@ void test_valid() {
       cat( tag(12345, UPB_WIRE_TYPE_64BIT), uint64(2345678) ),
       "<\n>\n");
   assert_successful_parse(
-      submsg(12345, buffer("                ")),
+      submsg(12345, string("                ")),
       "<\n>\n");
 
   assert_successful_parse(
@@ -818,7 +958,7 @@ void test_valid() {
   // Submessage tests.
   uint32_t msg_fn = UPB_DESCRIPTOR_TYPE_MESSAGE;
   assert_successful_parse(
-      submsg(msg_fn, submsg(msg_fn, submsg(msg_fn, buffer()))),
+      submsg(msg_fn, submsg(msg_fn, submsg(msg_fn, string()))),
       LINE("<")
       LINE("%u:{")
       LINE("  <")
@@ -836,7 +976,7 @@ void test_valid() {
 
   uint32_t repm_fn = rep_fn(UPB_DESCRIPTOR_TYPE_MESSAGE);
   assert_successful_parse(
-      submsg(repm_fn, submsg(repm_fn, buffer())),
+      submsg(repm_fn, submsg(repm_fn, string())),
       LINE("<")
       LINE("%u:[")
       LINE("  %u:{")
@@ -852,16 +992,59 @@ void test_valid() {
       LINE("]")
       LINE(">"), repm_fn, repm_fn, repm_fn, repm_fn);
 
+  // Test unknown group.
+  uint32_t unknown_group_fn = 12321;
+  assert_successful_parse(
+      cat( tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP),
+           tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) ),
+      LINE("<")
+      LINE(">")
+  );
+
+  // Test some unknown fields inside an unknown group.
+  const string unknown_group_with_data =
+      cat(
+          tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP),
+          tag(12345, UPB_WIRE_TYPE_VARINT), varint(2345678),
+          tag(123456789, UPB_WIRE_TYPE_32BIT), uint32(2345678),
+          tag(123477, UPB_WIRE_TYPE_64BIT), uint64(2345678),
+          tag(123, UPB_WIRE_TYPE_DELIMITED), varint(0),
+          tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP)
+         );
+
+  // Nested unknown group with data.
+  assert_successful_parse(
+      cat(
+           tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP),
+           unknown_group_with_data,
+           tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP),
+           tag(UPB_DESCRIPTOR_TYPE_INT32, UPB_WIRE_TYPE_VARINT), varint(1)
+         ),
+      LINE("<")
+      LINE("%u:1")
+      LINE(">"),
+      UPB_DESCRIPTOR_TYPE_INT32
+  );
+
+  assert_successful_parse(
+      cat( tag(unknown_group_fn, UPB_WIRE_TYPE_START_GROUP),
+           tag(unknown_group_fn + 1, UPB_WIRE_TYPE_START_GROUP),
+           tag(unknown_group_fn + 1, UPB_WIRE_TYPE_END_GROUP),
+           tag(unknown_group_fn, UPB_WIRE_TYPE_END_GROUP) ),
+      LINE("<")
+      LINE(">")
+  );
+
   // Staying within the stack limit should work properly.
-  buffer buf;
-  buffer textbuf;
+  string buf;
+  string textbuf;
   int total = UPB_DECODER_MAX_NESTING - 1;
   for (int i = 0; i < total; i++) {
     buf.assign(submsg(UPB_DESCRIPTOR_TYPE_MESSAGE, buf));
     indentbuf(&textbuf, i);
     textbuf.append("<\n");
     indentbuf(&textbuf, i);
-    textbuf.appendf("%u:{\n", UPB_DESCRIPTOR_TYPE_MESSAGE);
+    appendf(&textbuf, "%u:{\n", UPB_DESCRIPTOR_TYPE_MESSAGE);
   }
   indentbuf(&textbuf, total);
   textbuf.append("<\n");
@@ -873,7 +1056,7 @@ void test_valid() {
     indentbuf(&textbuf, total - i - 1);
     textbuf.append(">\n");
   }
-  assert_successful_parse(buf, "%s", textbuf.buf());
+  assert_successful_parse(buf, "%s", textbuf.c_str());
 }
 
 void run_tests() {
@@ -885,14 +1068,17 @@ upb::reffed_ptr<const upb::pb::DecoderMethod> NewMethod(
     const upb::Handlers* dest_handlers, bool allow_jit) {
   upb::pb::CodeCache cache;
   cache.set_allow_jit(allow_jit);
-  return cache.GetDecoderMethodForDestHandlers(dest_handlers);
+  return cache.GetDecoderMethod(upb::pb::DecoderMethodOptions(dest_handlers));
 }
 
 void test_emptyhandlers(bool allowjit) {
   // Create an empty handlers to make sure that the decoder can handle empty
   // messages.
-  upb::reffed_ptr<upb::Handlers> h(
-      upb::Handlers::New(UPB_TEST_DECODER_EMPTYMESSAGE));
+  upb::reffed_ptr<upb::MessageDef> md = upb::MessageDef::New();
+  ASSERT(md->set_full_name("Empty", NULL));
+  ASSERT(md->Freeze(NULL));
+
+  upb::reffed_ptr<upb::Handlers> h(upb::Handlers::New(md.get()));
   bool ok = h->Freeze(NULL);
   ASSERT(ok);
   NewMethod(h.get(), allowjit);
diff --git a/tests/pb/test_decoder_schema.proto b/tests/pb/test_decoder_schema.proto
deleted file mode 100644
index 50bfca9..0000000
--- a/tests/pb/test_decoder_schema.proto
+++ /dev/null
@@ -1,64 +0,0 @@
-//
-// upb - a minimalist implementation of protocol buffers.
-//
-// Copyright (c) 2012 Google Inc.  See LICENSE for details.
-// Author: Josh Haberman <jhaberman@gmail.com>
-//
-// Schema used in test_decoder.cc.  It contains two fields (one optional
-// and one repeated) for each type.
-
-package upb.test_decoder;
-
-message M {
-  optional M m = 1;
-}
-
-enum E {
-  FOO = 1;
-}
-
-message EmptyMessage {}
-
-message DecoderTest {
-  optional double   f_double    = 1;
-  optional float    f_float     = 2;
-  optional int64    f_int64     = 3;
-  optional uint64   f_uint64    = 4;
-  optional int32    f_int32     = 5;
-  optional fixed64  f_fixed64   = 6;
-  optional fixed32  f_fixed32   = 7;
-  optional bool     f_bool      = 8;
-  optional string   f_string    = 9;
-  optional bytes    f_bytes     = 12;
-  optional uint32   f_uint32    = 13;
-  optional sfixed32 f_sfixed32  = 15;
-  optional sfixed64 f_sfixed64  = 16;
-  optional sint32   f_sint32    = 17;
-  optional sint64   f_sint64    = 18;
-
-  optional DecoderTest f_message  = 11;
-  optional E f_enum     = 14;
-
-
-  repeated double   r_double    = 536869912;
-  repeated float    r_float     = 536869913;
-  repeated int64    r_int64     = 536869914;
-  repeated uint64   r_uint64    = 536869915;
-  repeated int32    r_int32     = 536869916;
-  repeated fixed64  r_fixed64   = 536869917;
-  repeated fixed32  r_fixed32   = 536869918;
-  repeated bool     r_bool      = 536869919;
-  repeated string   r_string    = 536869920;
-  repeated bytes    r_bytes     = 536869923;
-  repeated uint32   r_uint32    = 536869924;
-  repeated sfixed32 r_sfixed32  = 536869926;
-  repeated sfixed64 r_sfixed64  = 536869927;
-  repeated sint32   r_sint32    = 536869928;
-  repeated sint64   r_sint64    = 536869929;
-
-  repeated DecoderTest r_message  = 536869922;
-  repeated E r_enum     = 536869925;
-
-  // To allow arbitrary padding.
-  optional string nop_field = 40;
-}