Merge branch 'src-refactoring'

6 files changed, 369 insertions, 26 deletions

diff --git a/tests/test_decoder.c b/tests/test_decoder.c
new file mode 100644
index 0000000..f48472d
--- /dev/null
+++ b/tests/test_decoder.c
@@ -0,0 +1,42 @@
+
+#include "upb_decoder.h"
+#include "upb_textprinter.h"
+#include "upb_stdio.h"
+
+int main() {
+  upb_symtab *symtab = upb_symtab_new();
+  upb_symtab_add_descriptorproto(symtab);
+  upb_def *fds = upb_symtab_lookup(
+      symtab, UPB_STRLIT("google.protobuf.FileDescriptorSet"));
+
+  upb_stdio *in = upb_stdio_new();
+  upb_stdio_reset(in, stdin);
+  upb_stdio *out = upb_stdio_new();
+  upb_stdio_reset(out, stdout);
+  upb_decoder *d = upb_decoder_new(upb_downcast_msgdef(fds));
+  upb_decoder_reset(d, upb_stdio_bytesrc(in));
+  upb_textprinter *p = upb_textprinter_new();
+  upb_handlers handlers;
+  upb_handlers_init(&handlers);
+  upb_textprinter_reset(p, &handlers, upb_stdio_bytesink(out), false);
+  upb_src *src = upb_decoder_src(d);
+  upb_src_sethandlers(src, &handlers);
+
+  upb_status status = UPB_STATUS_INIT;
+  upb_src_run(src, &status);
+
+  assert(upb_ok(&status));
+
+  upb_status_uninit(&status);
+  upb_stdio_free(in);
+  upb_stdio_free(out);
+  upb_decoder_free(d);
+  upb_textprinter_free(p);
+  upb_def_unref(fds);
+  upb_symtab_unref(symtab);
+
+  // Prevent C library from holding buffers open, so Valgrind doesn't see
+  // memory leaks.
+  fclose(stdin);
+  fclose(stdout);
+}
diff --git a/tests/test_def.c b/tests/test_def.c
new file mode 100644
index 0000000..2d2658f
--- /dev/null
+++ b/tests/test_def.c
@@ -0,0 +1,25 @@
+
+#undef NDEBUG  /* ensure tests always assert. */
+#include "upb_def.h"
+#include <stdlib.h>
+
+int main() {
+  upb_symtab *s = upb_symtab_new();
+  upb_symtab_add_descriptorproto(s);
+
+  int count;
+  upb_def **defs = upb_symtab_getdefs(s, &count, UPB_DEF_ANY);
+  for (int i = 0; i < count; i++) {
+    upb_def_unref(defs[i]);
+  }
+  free(defs);
+
+  upb_string *str = upb_strdupc("google.protobuf.FileDescriptorSet");
+  upb_def *fds = upb_symtab_lookup(s, str);
+  assert(fds != NULL);
+  assert(upb_dyncast_msgdef(fds) != NULL);
+  upb_def_unref(fds);
+  upb_string_unref(str);
+  upb_symtab_unref(s);
+  return 0;
+}
diff --git a/tests/test_stream.c b/tests/test_stream.c
new file mode 100644
index 0000000..b6d511c
--- /dev/null
+++ b/tests/test_stream.c
@@ -0,0 +1,127 @@
+
+#undef NDEBUG  /* ensure tests always assert. */
+#include "upb_stream.h"
+#include "upb_string.h"
+
+typedef struct {
+  upb_string *str;
+  bool should_delegate;
+} test_data;
+
+extern upb_handlerset test_handlers;
+
+static void strappendf(upb_string *s, const char *format, ...) {
+  upb_string *str = upb_string_new();
+  va_list args;
+  va_start(args, format);
+  upb_string_vprintf(str, format, args);
+  va_end(args);
+  upb_strcat(s, str);
+  upb_string_unref(str);
+}
+
+static upb_flow_t startmsg(void *closure) {
+  test_data *d = closure;
+  strappendf(d->str, "startmsg\n");
+  return UPB_CONTINUE;
+}
+
+static upb_flow_t endmsg(void *closure) {
+  test_data *d = closure;
+  strappendf(d->str, "endmsg\n");
+  return UPB_CONTINUE;
+}
+
+static upb_flow_t value(void *closure, struct _upb_fielddef *f, upb_value val) {
+  (void)f;
+  test_data *d = closure;
+  strappendf(d->str, "value, %lld\n", upb_value_getint64(val));
+  return UPB_CONTINUE;
+}
+
+static upb_flow_t startsubmsg(void *closure, struct _upb_fielddef *f,
+                              upb_handlers *delegate_to) {
+  (void)f;
+  test_data *d = closure;
+  strappendf(d->str, "startsubmsg\n");
+  if (d->should_delegate) {
+    upb_register_handlerset(delegate_to, &test_handlers);
+    upb_set_handler_closure(delegate_to, closure, NULL);
+    return UPB_DELEGATE;
+  } else {
+    return UPB_CONTINUE;
+  }
+}
+
+static upb_flow_t endsubmsg(void *closure) {
+  test_data *d = closure;
+  strappendf(d->str, "endsubmsg\n");
+  return UPB_CONTINUE;
+}
+
+static upb_flow_t unknownval(void *closure, upb_field_number_t fieldnum,
+                             upb_value val) {
+  (void)val;
+  test_data *d = closure;
+  strappendf(d->str, "unknownval, %d\n", fieldnum);
+  return UPB_CONTINUE;
+}
+
+upb_handlerset test_handlers = {
+  &startmsg,
+  &endmsg,
+  &value,
+  &startsubmsg,
+  &endsubmsg,
+  &unknownval,
+};
+
+static void test_dispatcher() {
+  test_data data;
+  data.should_delegate = false;
+  data.str = upb_string_new();
+  upb_handlers h;
+  upb_handlers_init(&h);
+  upb_handlers_reset(&h);
+  upb_register_handlerset(&h, &test_handlers);
+  upb_set_handler_closure(&h, &data, NULL);
+  upb_dispatcher d;
+  upb_dispatcher_init(&d);
+  upb_dispatcher_reset(&d, &h);
+
+  upb_dispatch_startmsg(&d);
+  upb_value val;
+  upb_value_setint64(&val, 5);
+  upb_dispatch_value(&d, NULL, val);
+  upb_dispatch_startsubmsg(&d, NULL);
+  data.should_delegate = true;
+  upb_dispatch_startsubmsg(&d, NULL);
+  data.should_delegate = false;
+  upb_dispatch_startsubmsg(&d, NULL);
+  upb_dispatch_value(&d, NULL, val);
+  upb_dispatch_endsubmsg(&d);
+  upb_dispatch_endsubmsg(&d);
+  upb_dispatch_endsubmsg(&d);
+  upb_dispatch_endmsg(&d);
+
+  upb_string expected = UPB_STACK_STRING(
+      "startmsg\n"
+      "value, 5\n"
+      "startsubmsg\n"
+      "startsubmsg\n"
+      "startmsg\n"  // Because of the delegation.
+      "startsubmsg\n"
+      "value, 5\n"
+      "endsubmsg\n"
+      "endmsg\n"   // Because of the delegation.
+      "endsubmsg\n"
+      "endsubmsg\n"
+      "endmsg\n");
+  assert(upb_streql(data.str, &expected));
+  upb_string_unref(data.str);
+}
+
+int main() {
+  test_dispatcher();
+  return 0;
+}
diff --git a/tests/test_string.c b/tests/test_string.c
new file mode 100644
index 0000000..ef0e2a9
--- /dev/null
+++ b/tests/test_string.c
@@ -0,0 +1,126 @@
+
+#undef NDEBUG  /* ensure tests always assert. */
+#include "upb_string.h"
+
+char static_str[] = "Static string.";
+upb_string static_upbstr = UPB_STATIC_STRING(static_str);
+
+static void test_static() {
+  // Static string is initialized appropriately.
+  assert(upb_streql(&static_upbstr, UPB_STRLIT("Static string.")));
+
+  // Taking a ref on a static string returns the same string, and repeated
+  // refs don't get the string in a confused state.
+  assert(upb_string_getref(&static_upbstr) == &static_upbstr);
+  assert(upb_string_getref(&static_upbstr) == &static_upbstr);
+  assert(upb_string_getref(&static_upbstr) == &static_upbstr);
+
+  // Unreffing a static string does nothing (is not harmful).
+  upb_string_unref(&static_upbstr);
+  upb_string_unref(&static_upbstr);
+  upb_string_unref(&static_upbstr);
+  upb_string_unref(&static_upbstr);
+  upb_string_unref(&static_upbstr);
+
+  // Recycling a static string returns a new string (that can be modified).
+  upb_string *str = &static_upbstr;
+  upb_string_recycle(&str);
+  assert(str != &static_upbstr);
+
+  upb_string_unref(str);
+}
+
+static void test_dynamic() {
+  upb_string *str = upb_string_new();
+  assert(str != NULL);
+  upb_string_unref(str);
+
+  // Can also create a string by recycle(NULL).
+  str = NULL;
+  upb_string_recycle(&str);
+  assert(str != NULL);
+
+  // Take a ref and recycle; should create a new string and release a ref
+  // on the old one.
+  upb_string *strcp = upb_string_getref(str);
+  assert(strcp == str);
+  assert(upb_atomic_read(&str->refcount) == 2);
+  upb_string_recycle(&str);
+  assert(strcp != str);
+  assert(upb_atomic_read(&str->refcount) == 1);
+  assert(upb_atomic_read(&strcp->refcount) == 1);
+  upb_string_unref(strcp);
+
+  upb_strcpyc(str, static_str);
+  assert(upb_string_len(str) == (sizeof(static_str) - 1));
+  const char *robuf = upb_string_getrobuf(str);
+  assert(robuf != NULL);
+  assert(upb_streqlc(str, static_str));
+  upb_string_endread(str);
+
+  upb_string *str2 = str;
+  upb_string_recycle(&str2);
+  // No other referents, so should return the same string.
+  assert(str2 == str);
+
+  // Write a shorter string, the same memory should be reused.
+  upb_strcpyc(str, "XX");
+  const char *robuf2 = upb_string_getrobuf(str);
+  assert(robuf2 == robuf);
+  assert(upb_streqlc(str, "XX"));
+  assert(upb_streql(str, UPB_STRLIT("XX")));
+
+  // Make string alias part of another string.
+  str2 = upb_strdupc("WXYZ");
+  upb_string_recycle(&str);
+  upb_string_substr(str, str2, 1, 2);
+  assert(upb_string_len(str) == 2);
+  assert(upb_string_len(str2) == 4);
+  // The two string should be aliasing the same data.
+  const char *robuf3 = upb_string_getrobuf(str);
+  const char *robuf4 = upb_string_getrobuf(str2);
+  assert(robuf3 == robuf4 + 1);
+  // The aliased string should have an extra ref.
+  assert(upb_atomic_read(&str2->refcount) == 2);
+
+  // Recycling str should eliminate the extra ref.
+  upb_string_recycle(&str);
+  assert(upb_atomic_read(&str2->refcount) == 1);
+
+  // Resetting str should reuse its old data.
+  upb_strcpyc(str, "XX");
+  const char *robuf5 = upb_string_getrobuf(str);
+  assert(robuf5 == robuf);
+
+  // Resetting str to something very long should require new data to be
+  // allocated.
+  upb_string_recycle(&str);
+  const char longstring[] = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
+  upb_strcpyc(str, longstring);
+  const char *robuf6 = upb_string_getrobuf(str);
+  assert(robuf6 != robuf);
+  assert(upb_streqlc(str, longstring));
+
+  // Test printf.
+  upb_string_recycle(&str);
+  upb_string_printf(str, "Number: %d, String: %s", 5, "YO!");
+  assert(upb_streqlc(str, "Number: 5, String: YO!"));
+
+  // Test asprintf
+  upb_string *str3 = upb_string_asprintf("Yo %s: " UPB_STRFMT "\n",
+                                         "Josh", UPB_STRARG(str));
+  const char expected[] = "Yo Josh: Number: 5, String: YO!\n";
+  assert(upb_streqlc(str3, expected));
+
+  upb_string_unref(str);
+  upb_string_unref(str2);
+  upb_string_unref(str3);
+
+  // Unref of NULL is harmless.
+  upb_string_unref(NULL);
+}
+
+int main() {
+  test_static();
+  test_dynamic();
+}
diff --git a/tests/test_table.cc b/tests/test_table.cc
index 47d806c..47d5e57 100644
--- a/tests/test_table.cc
+++ b/tests/test_table.cc
@@ -1,7 +1,7 @@
 
 #undef NDEBUG  /* ensure tests always assert. */
 #include "upb_table.h"
-#include "upb_data.h"
+#include "upb_string.h"
 #include "test_util.h"
 #include <assert.h>
 #include <map>
@@ -12,6 +12,8 @@
 #include <sys/resource.h>
 #include <iostream>
 
+bool benchmark = false;
+
 using std::string;
 using std::vector;
 
@@ -45,7 +47,7 @@ void test_strtable(const vector<string>& keys, uint32_t num_to_insert)
     all.insert(key);
     strtable_entry e;
     e.value = key[0];
-    upb_strptr str = upb_strduplen(key.c_str(), key.size());
+    upb_string *str = upb_strduplen(key.c_str(), key.size());
     e.e.key = str;
     upb_strtable_insert(&table, &e.e);
     upb_string_unref(str);  // The table still owns a ref.
@@ -55,7 +57,7 @@ void test_strtable(const vector<string>& keys, uint32_t num_to_insert)
   /* Test correctness. */
   for(uint32_t i = 0; i < keys.size(); i++) {
     const string& key = keys[i];
-    upb_strptr str = upb_strduplen(key.c_str(), key.size());
+    upb_string *str = upb_strduplen(key.c_str(), key.size());
     strtable_entry *e = (strtable_entry*)upb_strtable_lookup(&table, str);
     if(m.find(key) != m.end()) { /* Assume map implementation is correct. */
       assert(e);
@@ -71,7 +73,7 @@ void test_strtable(const vector<string>& keys, uint32_t num_to_insert)
   strtable_entry *e;
   for(e = (strtable_entry*)upb_strtable_begin(&table); e;
       e = (strtable_entry*)upb_strtable_next(&table, &e->e)) {
-    string tmp(upb_string_getrobuf(e->e.key), upb_strlen(e->e.key));
+    string tmp(upb_string_getrobuf(e->e.key), upb_string_len(e->e.key));
     std::set<string>::iterator i = all.find(tmp);
     assert(i != all.end());
     all.erase(i);
@@ -116,6 +118,11 @@ void test_inttable(int32_t *keys, size_t num_entries)
     }
   }
 
+  if(!benchmark) {
+    upb_inttable_free(&table);
+    return;
+  }
+
   /* Test performance. We only test lookups for keys that are known to exist. */
   uintptr_t x = 0;
   const unsigned int iterations = 0xFFFFFF;
@@ -219,8 +226,12 @@ int32_t *get_contiguous_keys(int32_t num)
   return buf;
 }
 
-int main()
+int main(int argc, char *argv[])
 {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--benchmark") == 0) benchmark = true;
+  }
+
   vector<string> keys;
   keys.push_back("google.protobuf.FileDescriptorSet");
   keys.push_back("google.protobuf.FileDescriptorProto");
diff --git a/tests/test_vs_proto2.cc b/tests/test_vs_proto2.cc
index 9083788..9446b8f 100644
--- a/tests/test_vs_proto2.cc
+++ b/tests/test_vs_proto2.cc
@@ -4,9 +4,10 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <google/protobuf/descriptor.h>
-#include "upb_data.h"
+#include "upb_msg.h"
 #include "upb_def.h"
 #include "upb_decoder.h"
+#include "upb_strstream.h"
 
 int num_assertions = 0;
 #define ASSERT(expr) do { \
@@ -25,7 +26,7 @@ void compare_arrays(const google::protobuf::Reflection *r,
                     upb_msg *upb_msg, upb_fielddef *upb_f)
 {
   ASSERT(upb_msg_has(upb_msg, upb_f));
-  upb_arrayptr arr = upb_msg_get(upb_msg, upb_f).arr;
+  upb_array *arr = upb_msg_get(upb_msg, upb_f).arr;
   ASSERT(upb_array_len(arr) == (upb_arraylen_t)r->FieldSize(proto2_msg, proto2_f));
   for(upb_arraylen_t i = 0; i < upb_array_len(arr); i++) {
     upb_value v = upb_array_get(arr, upb_f, i);
@@ -63,7 +64,7 @@ void compare_arrays(const google::protobuf::Reflection *r,
       case UPB_TYPE(STRING):
       case UPB_TYPE(BYTES): {
         std::string str = r->GetRepeatedString(proto2_msg, proto2_f, i);
-        std::string str2(upb_string_getrobuf(v.str), upb_strlen(v.str));
+        std::string str2(upb_string_getrobuf(v.str), upb_string_len(v.str));
         ASSERT(str == str2);
         break;
       }
@@ -116,7 +117,7 @@ void compare_values(const google::protobuf::Reflection *r,
     case UPB_TYPE(STRING):
     case UPB_TYPE(BYTES): {
       std::string str = r->GetString(proto2_msg, proto2_f);
-      std::string str2(upb_string_getrobuf(v.str), upb_strlen(v.str));
+      std::string str2(upb_string_getrobuf(v.str), upb_string_len(v.str));
       ASSERT(str == str2);
       break;
     }
@@ -133,9 +134,10 @@ void compare(const google::protobuf::Message& proto2_msg,
   const google::protobuf::Reflection *r = proto2_msg.GetReflection();
   const google::protobuf::Descriptor *d = proto2_msg.GetDescriptor();
 
-  ASSERT((upb_field_count_t)d->field_count() == upb_md->num_fields);
-  for(upb_field_count_t i = 0; i < upb_md->num_fields; i++) {
-    upb_fielddef *upb_f = &upb_md->fields[i];
+  ASSERT((upb_field_count_t)d->field_count() == upb_msgdef_numfields(upb_md));
+  upb_msg_iter i;
+  for(i = upb_msg_begin(upb_md); !upb_msg_done(i); i = upb_msg_next(upb_md, i)) {
+    upb_fielddef *upb_f = upb_msg_iter_field(i);
     const google::protobuf::FieldDescriptor *proto2_f =
         d->FindFieldByNumber(upb_f->number);
     // Make sure the definitions are equal.
@@ -143,7 +145,7 @@ void compare(const google::protobuf::Message& proto2_msg,
     ASSERT(proto2_f);
     ASSERT(upb_f->number == proto2_f->number());
     ASSERT(std::string(upb_string_getrobuf(upb_f->name),
-                       upb_strlen(upb_f->name)) ==
+                       upb_string_len(upb_f->name)) ==
            proto2_f->name());
     ASSERT(upb_f->type == proto2_f->type());
     ASSERT(upb_isarray(upb_f) == proto2_f->is_repeated());
@@ -166,10 +168,10 @@ void compare(const google::protobuf::Message& proto2_msg,
 
 void parse_and_compare(MESSAGE_CIDENT *proto2_msg,
                        upb_msg *upb_msg, upb_msgdef *upb_md,
-                       upb_strptr str)
+                       upb_string *str)
 {
   // Parse to both proto2 and upb.
-  ASSERT(proto2_msg->ParseFromArray(upb_string_getrobuf(str), upb_strlen(str)));
+  ASSERT(proto2_msg->ParseFromArray(upb_string_getrobuf(str), upb_string_len(str)));
   upb_status status = UPB_STATUS_INIT;
   upb_msg_decodestr(upb_msg, upb_md, str, &status);
   ASSERT(upb_ok(&status));
@@ -194,22 +196,32 @@ int main(int argc, char *argv[])
 
   // Initialize upb state, parse descriptor.
   upb_status status = UPB_STATUS_INIT;
-  upb_symtab *c = upb_symtab_new();
-  upb_strptr fds = upb_strreadfile(MESSAGE_DESCRIPTOR_FILE);
-  if(upb_string_isnull(fds)) {
+  upb_symtab *symtab = upb_symtab_new();
+  upb_string *fds = upb_strreadfile(MESSAGE_DESCRIPTOR_FILE);
+  if(fds == NULL) {
     fprintf(stderr, "Couldn't read " MESSAGE_DESCRIPTOR_FILE ".\n");
     return 1;
   }
-  upb_symtab_add_desc(c, fds, &status);
+  upb_symtab_add_descriptorproto(symtab);
+  upb_def *fds_msgdef = upb_symtab_lookup(
+      symtab, UPB_STRLIT("google.protobuf.FileDescriptorSet"));
+
+  upb_stringsrc *ssrc = upb_stringsrc_new();
+  upb_stringsrc_reset(ssrc, fds);
+  upb_decoder *decoder = upb_decoder_new(upb_downcast_msgdef(fds_msgdef));
+  upb_decoder_reset(decoder, upb_stringsrc_bytesrc(ssrc));
+  upb_symtab_addfds(symtab, upb_decoder_src(decoder), &status);
   if(!upb_ok(&status)) {
-    fprintf(stderr, "Error importing " MESSAGE_DESCRIPTOR_FILE ": %s.\n",
-            status.msg);
+    fprintf(stderr, "Error importing " MESSAGE_DESCRIPTOR_FILE ": ");
+    upb_printerr(&status);
     return 1;
   }
   upb_string_unref(fds);
+  upb_decoder_free(decoder);
+  upb_stringsrc_free(ssrc);
 
-  upb_strptr proto_name = upb_strdupc(MESSAGE_NAME);
-  upb_msgdef *def = upb_downcast_msgdef(upb_symtab_lookup(c, proto_name));
+  upb_string *proto_name = upb_strdupc(MESSAGE_NAME);
+  upb_msgdef *def = upb_downcast_msgdef(upb_symtab_lookup(symtab, proto_name));
   if(!def) {
     fprintf(stderr, "Error finding symbol '" UPB_STRFMT "'.\n",
             UPB_STRARG(proto_name));
@@ -218,8 +230,8 @@ int main(int argc, char *argv[])
   upb_string_unref(proto_name);
 
   // Read the message data itself.
-  upb_strptr str = upb_strreadfile(MESSAGE_FILE);
-  if(upb_string_isnull(str)) {
+  upb_string *str = upb_strreadfile(MESSAGE_FILE);
+  if(str == NULL) {
     fprintf(stderr, "Error reading " MESSAGE_FILE "\n");
     return 1;
   }
@@ -234,7 +246,7 @@ int main(int argc, char *argv[])
   upb_msg_unref(upb_msg, def);
   upb_def_unref(UPB_UPCAST(def));
   upb_string_unref(str);
-  upb_symtab_unref(c);
+  upb_symtab_unref(symtab);
 
   return 0;
 }