Google-internal development.

6 files changed, 99 insertions, 52 deletions

diff --git a/upb/bindings/googlepb/bridge.cc b/upb/bindings/googlepb/bridge.cc
index 6ae8868..2a2bbbf 100644
--- a/upb/bindings/googlepb/bridge.cc
+++ b/upb/bindings/googlepb/bridge.cc
@@ -246,7 +246,8 @@ const Handlers* CodeCache::GetMaybeUnfrozenWriteHandlers(
   to_freeze_.push_back(h);
   const goog::Descriptor* d = m.GetDescriptor();
 
-  for (upb::MessageDef::const_iterator i = md->begin(); i != md->end(); ++i) {
+  for (upb::MessageDef::const_field_iterator i = md->field_begin();
+       i != md->field_end(); ++i) {
     const FieldDef* upb_f = *i;
 
     const goog::FieldDescriptor* proto2_f =
diff --git a/upb/bindings/googlepb/proto2.cc b/upb/bindings/googlepb/proto2.cc
index 3911172..87c13b6 100644
--- a/upb/bindings/googlepb/proto2.cc
+++ b/upb/bindings/googlepb/proto2.cc
@@ -946,14 +946,14 @@ case goog::FieldDescriptor::cpptype:                                           \
    public:
     typedef goog::Message Type;
 #ifdef GOOGLE_PROTOBUF_HAS_ARENAS
-    static ::proto2::Arena* GetArena(Type* t) {
+    static goog::Arena* GetArena(Type* t) {
       return t->GetArena();
     }
     static void* GetMaybeArenaPointer(Type* t) {
       return t->GetMaybeArenaPointer();
     }
     static inline Type* NewFromPrototype(
-        const Type* prototype, ::proto2::Arena* arena = NULL) {
+        const Type* prototype, goog::Arena* arena = NULL) {
       return prototype->New(arena);
     }
     static void Delete(Type* t, goog::Arena* arena = NULL) {
@@ -1277,6 +1277,11 @@ case goog::FieldDescriptor::cpptype:                                           \
       return lazy_field_.SetAllocated(static_cast<proto2::Message*>(message));
     }
 
+    virtual void UnsafeArenaSetAllocatedMessage(proto2::MessageLite* message) {
+      return lazy_field_.UnsafeArenaSetAllocated(
+          static_cast<proto2::Message*>(message));
+    }
+
     virtual proto2::MessageLite* ReleaseMessage(
         const proto2::MessageLite& prototype) {
       return lazy_field_.ReleaseByPrototype(
diff --git a/upb/bindings/lua/upb.c b/upb/bindings/lua/upb.c
index 17fc0a8..5ad0235 100644
--- a/upb/bindings/lua/upb.c
+++ b/upb/bindings/lua/upb.c
@@ -139,7 +139,8 @@ bool lupb_openlib(lua_State *L, void *ptr, const char *name,
 
 // Pushes a new userdata with the given metatable and ensures that it has a
 // uservalue.
-static void *newudata_with_userval(lua_State *L, size_t size, const char *type) {
+static void *newudata_with_userval(lua_State *L, size_t size,
+                                   const char *type) {
   void *ret = lua_newuserdata(L, size);
 
   // Set metatable.
@@ -952,17 +953,17 @@ static int lupb_msgdef_field(lua_State *L) {
 }
 
 static int lupb_msgiter_next(lua_State *L) {
-  upb_msg_iter *i = lua_touserdata(L, lua_upvalueindex(1));
-  if (upb_msg_done(i)) return 0;
+  upb_msg_field_iter *i = lua_touserdata(L, lua_upvalueindex(1));
+  if (upb_msg_field_done(i)) return 0;
   lupb_def_pushwrapper(L, UPB_UPCAST(upb_msg_iter_field(i)), NULL);
-  upb_msg_next(i);
+  upb_msg_field_next(i);
   return 1;
 }
 
 static int lupb_msgdef_fields(lua_State *L) {
   const upb_msgdef *m = lupb_msgdef_check(L, 1);
-  upb_msg_iter *i = lua_newuserdata(L, sizeof(upb_msg_iter));
-  upb_msg_begin(i, m);
+  upb_msg_field_iter *i = lua_newuserdata(L, sizeof(upb_msg_field_iter));
+  upb_msg_field_begin(i, m);
   // Need to guarantee that the msgdef outlives the iter.
   lua_pushvalue(L, 1);
   lua_pushcclosure(L, &lupb_msgiter_next, 2);
@@ -1416,8 +1417,10 @@ static lupb_msgdef *lupb_msg_assignoffsets(lua_State *L, int narg) {
   size_t userval_idx = 1;
 
   // Assign offsets.
-  upb_msg_iter i;
-  for (upb_msg_begin(&i, lmd->md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  upb_msg_field_iter i;
+  for (upb_msg_field_begin(&i, lmd->md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     upb_fielddef *f = upb_msg_iter_field(&i);
     if (in_userval(f)) {
       offsets[upb_fielddef_index(f)] = userval_idx++;
@@ -1442,7 +1445,9 @@ static lupb_msgdef *lupb_msg_assignoffsets(lua_State *L, int narg) {
   lua_newtable(L);  // This will be our userval.
 
   int idx = 1;
-  for (upb_msg_begin(&i, lmd->md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  for (upb_msg_field_begin(&i, lmd->md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     upb_fielddef *f = upb_msg_iter_field(&i);
     if (upb_fielddef_type(f) == UPB_TYPE_MESSAGE) {
       bool created = lupb_def_pushwrapper(L, upb_fielddef_subdef(f), NULL);
@@ -1660,9 +1665,9 @@ void callback(const void *closure, upb_handlers *h) {
   lua_State *L = (lua_State*)closure;
   lupb_def_pushwrapper(L, UPB_UPCAST(upb_handlers_msgdef(h)), NULL);
   lupb_msgdef *lmd = lupb_msg_assignoffsets(L, -1);
-  upb_msg_iter i;
-  upb_msg_begin(&i, upb_handlers_msgdef(h));
-  for (; !upb_msg_done(&i); upb_msg_next(&i)) {
+  upb_msg_field_iter i;
+  upb_msg_field_begin(&i, upb_handlers_msgdef(h));
+  for (; !upb_msg_field_done(&i); upb_msg_field_next(&i)) {
     upb_fielddef *f = upb_msg_iter_field(&i);
     int hasbit = upb_fielddef_index(f);
     uint16_t ofs = lmd->field_offsets[upb_fielddef_index(f)];
diff --git a/upb/bindings/lua/upb/pb.c b/upb/bindings/lua/upb/pb.c
index c9f1f47..ea3d755 100644
--- a/upb/bindings/lua/upb/pb.c
+++ b/upb/bindings/lua/upb/pb.c
@@ -41,6 +41,13 @@ static int lupb_pbdecodermethod_new(lua_State *L) {
   return 1;  // The DecoderMethod wrapper.
 }
 
+// We implement upb's allocation function by allocating a Lua userdata.
+// This is a raw hunk of memory that will be GC'd by Lua.
+static void *lua_alloc(void *ud, size_t size) {
+  lua_State *L = ud;
+  return lua_newuserdata(L, size);
+}
+
 // Unlike most of our exposed Lua functions, this does not correspond to an
 // actual method on the underlying DecoderMethod.  But it's convenient, and
 // important to implement in C because we can do stack allocation and
@@ -61,19 +68,22 @@ static int lupb_pbdecodermethod_parse(lua_State *L) {
   // Handlers need this.
   lua_getuservalue(L, -1);
 
-  upb_pbdecoder decoder;
   upb_status status = UPB_STATUS_INIT;
-  upb_pbdecoder_init(&decoder, method, &status);
+  upb_env env;
+  upb_env_init(&env);
+  upb_env_reporterrorsto(&env, &status);
   upb_sink sink;
   upb_sink_reset(&sink, handlers, msg);
-  upb_pbdecoder_resetoutput(&decoder, &sink);
-  upb_bufsrc_putbuf(pb, len, upb_pbdecoder_input(&decoder));
-  // TODO: Our need to call uninit isn't longjmp-safe; what if the decode
-  // triggers a Lua error?  uninit is only needed if the decoder
-  // dynamically-allocated a growing stack -- ditch this feature and live with
-  // the compile-time limit?  Or have a custom allocation function that
-  // allocates Lua GC-rooted memory?
-  upb_pbdecoder_uninit(&decoder);
+  upb_pbdecoder *decoder = upb_pbdecoder_create(&env, method, &sink);
+  upb_bufsrc_putbuf(pb, len, upb_pbdecoder_input(decoder));
+
+  // This won't get called in the error case, which longjmp's across us.  But
+  // since we made our alloc function allocate only GC-able memory, that
+  // shouldn't matter.  It *would* matter if the environment had references to
+  // any non-memory resources (ie.  filehandles).  As an alternative to this we
+  // could make the environment itself a userdata.
+  upb_env_uninit(&env);
+
   lupb_checkstatus(L, &status);
 
   lua_pop(L, 1);  // Uservalue.
diff --git a/upb/bindings/python/upb.c b/upb/bindings/python/upb.c
index 497074b..6cfc8e9 100644
--- a/upb/bindings/python/upb.c
+++ b/upb/bindings/python/upb.c
@@ -298,7 +298,8 @@ static PyObject *PyUpb_MessageDef_new(PyTypeObject *subtype,
 
 static PyObject *PyUpb_MessageDef_add_fields(PyObject *o, PyObject *args);
 
-static int PyUpb_MessageDef_init(PyObject *self, PyObject *args, PyObject *kwds) {
+static int PyUpb_MessageDef_init(
+    PyObject *self, PyObject *args, PyObject *kwds) {
   if (!kwds) return 0;
   PyObject *key, *value;
   Py_ssize_t pos = 0;
@@ -323,7 +324,8 @@ static PyObject *PyUpb_MessageDef_getattro(PyObject *obj, PyObject *attr_name) {
   return PyObject_GenericGetAttr(obj, attr_name);
 }
 
-static int PyUpb_MessageDef_setattro(PyObject *o, PyObject *key, PyObject *val) {
+static int PyUpb_MessageDef_setattro(
+    PyObject *o, PyObject *key, PyObject *val) {
   upb_msgdef *m = Check_MessageDef(o, -1);
   if (!upb_def_ismutable(UPB_UPCAST(m))) {
     PyErr_SetString(PyExc_TypeError, "MessageDef is not mutable.");
@@ -343,9 +345,11 @@ static int PyUpb_MessageDef_setattro(PyObject *o, PyObject *key, PyObject *val)
 static PyObject *PyUpb_MessageDef_fields(PyObject *obj, PyObject *args) {
   upb_msgdef *m = Check_MessageDef(obj, NULL);
   PyObject *ret = PyList_New(0);
-  upb_msg_iter i;
-  for(i = upb_msg_begin(m); !upb_msg_done(i); i = upb_msg_next(m, i)) {
-    upb_fielddef *f = upb_msg_iter_field(i);
+  upb_msg_field_iter i;
+  for(upb_msg_field_begin(&i, m);
+      !upb_msg_field_done(&i);
+      upb_msg_field_next(&ii)) {
+    upb_fielddef *f = upb_msg_iter_field(&i);
     PyList_Append(ret, PyUpb_FieldDef_GetOrCreate(f));
   }
   return ret;
@@ -374,9 +378,12 @@ static PyObject *PyUpb_MessageDef_add_field(PyObject *o, PyObject *field) {
 }
 
 static PyMethodDef PyUpb_MessageDef_methods[] = {
-  {"add_field", &PyUpb_MessageDef_add_field, METH_O, "Adds a list of fields."},
-  {"add_fields", &PyUpb_MessageDef_add_fields, METH_O, "Adds a list of fields."},
-  {"fields", &PyUpb_MessageDef_fields, METH_NOARGS, "Returns list of fields."},
+  {"add_field", &PyUpb_MessageDef_add_field, METH_O,
+    "Adds a list of fields."},
+  {"add_fields", &PyUpb_MessageDef_add_fields, METH_O,
+    "Adds a list of fields."},
+  {"fields", &PyUpb_MessageDef_fields, METH_NOARGS,
+    "Returns list of fields."},
   {NULL, NULL}
 };
 
@@ -448,7 +455,8 @@ static PyObject *PyUpb_SymbolTable_new(PyTypeObject *subtype,
   return PyUpb_ObjCacheGet(upb_symtab_new(), subtype);
 }
 
-static int PyUpb_SymbolTable_init(PyObject *self, PyObject *args, PyObject *kwds) {
+static int PyUpb_SymbolTable_init(
+    PyObject *self, PyObject *args, PyObject *kwds) {
   return 0;
 }
 
@@ -475,8 +483,10 @@ static PyObject *PyUpb_SymbolTable_add_defs(PyObject *o, PyObject *defs) {
     cdefs[i++] = def;
     upb_msgdef *md = upb_dyncast_msgdef(def);
     if (!md) continue;
-    upb_msg_iter j;
-    for(j = upb_msg_begin(md); !upb_msg_done(j); j = upb_msg_next(md, j)) {
+    upb_msg_field_iter j;
+    for(upb_msg_field_begin(&j, md);
+        !upb_msg_field_done(&j);
+        upb_msg_field_next(&j)) {
       upb_fielddef *f = upb_msg_iter_field(j);
       upb_fielddef_setaccessor(f, PyUpb_AccessorForField(f));
     }
@@ -601,7 +611,8 @@ static upb_sflow_t PyUpb_Message_StartSubmessage(void *m, upb_value fval) {
   return UPB_CONTINUE_WITH(*submsg);
 }
 
-static upb_sflow_t PyUpb_Message_StartRepeatedSubmessage(void *a, upb_value fval) {
+static upb_sflow_t PyUpb_Message_StartRepeatedSubmessage(
+    void *a, upb_value fval) {
   (void)fval;
   PyObject **elem = upb_stdarray_append(a, sizeof(void*));
   PyTypeObject *type = ((PyUpb_MessageType*)Py_TYPE(a))->alt_type;
@@ -609,7 +620,8 @@ static upb_sflow_t PyUpb_Message_StartRepeatedSubmessage(void *a, upb_value fval
   return UPB_CONTINUE_WITH(*elem);
 }
 
-static upb_flow_t PyUpb_Message_StringValue(void *m, upb_value fval, upb_value val) {
+static upb_flow_t PyUpb_Message_StringValue(
+    void *m, upb_value fval, upb_value val) {
   PyObject **str = PyUpb_Accessor_GetPtr(m, fval);
   if (*str) { Py_DECREF(*str); }
   *str = PyString_FromStringAndSize(NULL, upb_value_getstrref(val)->len);
@@ -618,7 +630,8 @@ static upb_flow_t PyUpb_Message_StringValue(void *m, upb_value fval, upb_value v
   return UPB_CONTINUE;
 }
 
-static upb_flow_t PyUpb_Message_AppendStringValue(void *a, upb_value fval, upb_value val) {
+static upb_flow_t PyUpb_Message_AppendStringValue(
+    void *a, upb_value fval, upb_value val) {
   (void)fval;
   PyObject **elem = upb_stdarray_append(a, sizeof(void*));
   *elem = PyString_FromStringAndSize(NULL, upb_value_getstrref(val)->len);
diff --git a/upb/bindings/ruby/upb.c b/upb/bindings/ruby/upb.c
index 2817a15..9618366 100644
--- a/upb/bindings/ruby/upb.c
+++ b/upb/bindings/ruby/upb.c
@@ -273,9 +273,11 @@ static size_t rupb_sizeof(const upb_fielddef *f) {
 static void assign_offsets(rb_msglayout *layout, const upb_msgdef *md) {
   layout->field_offsets = ALLOC_N(uint32_t, upb_msgdef_numfields(md));
   size_t ofs = 0;
-  upb_msg_iter i;
+  upb_msg_field_iter i;
 
-  for (upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  for (upb_msg_field_begin(&i, md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     const upb_fielddef *f = upb_msg_iter_field(&i);
     size_t field_size = rupb_sizeof(f);
 
@@ -301,8 +303,10 @@ static void make_prototype(rb_msglayout *layout, const upb_msgdef *md) {
   // more specific initialization.
   memset(prototype, 0, layout->size);
 
-  upb_msg_iter i;
-  for (upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  upb_msg_field_iter i;
+  for (upb_msg_field_begin(&i, md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     const upb_fielddef *f = upb_msg_iter_field(&i);
     if (is_ruby_value(f)) {
       size_t ofs = layout->field_offsets[upb_fielddef_index(f)];
@@ -373,8 +377,10 @@ static void msgdef_mark(void *_rmd) {
   rb_gc_mark(rmd->klass);
 
   // Mark all submessage types.
-  upb_msg_iter i;
-  for (upb_msg_begin(&i, rmd->md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  upb_msg_field_iter i;
+  for (upb_msg_field_begin(&i, rmd->md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     upb_fielddef *f = upb_msg_iter_field(&i);
     if (upb_fielddef_issubmsg(f)) {
       // If we were trying to be more aggressively lazy, the submessage might
@@ -495,8 +501,10 @@ static void msg_mark(void *p) {
 
   // We need to mark all references to other Ruby values: strings, arrays, and
   // submessages that we point to.
-  upb_msg_iter i;
-  for (upb_msg_begin(&i, rmd->md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  upb_msg_field_iter i;
+  for (upb_msg_field_begin(&i, rmd->md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     upb_fielddef *f = upb_msg_iter_field(&i);
     if (is_ruby_value(f)) {
       size_t ofs = rmd->layout.field_offsets[upb_fielddef_index(f)];
@@ -903,7 +911,8 @@ static void *submsg_handler(void *closure, const void *hd) {
   const submsg_handlerdata_t *submsgdata = hd;
 
   if (DEREF(msg, submsgdata->ofs, VALUE) == Qnil) {
-    DEREF(msg, submsgdata->ofs, VALUE) = msg_new(msgdef_getwrapper(submsgdata->md));
+    DEREF(msg, submsgdata->ofs, VALUE) =
+        msg_new(msgdef_getwrapper(submsgdata->md));
   }
 
   VALUE submsg = DEREF(msg, submsgdata->ofs, VALUE);
@@ -912,9 +921,11 @@ static void *submsg_handler(void *closure, const void *hd) {
 
 static void add_handlers_for_message(const void *closure, upb_handlers *h) {
   const rupb_MessageDef *rmd = get_rbmsgdef(upb_handlers_msgdef(h));
-  upb_msg_iter i;
+  upb_msg_field_iter i;
 
-  for (upb_msg_begin(&i, rmd->md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  for (upb_msg_field_begin(&i, rmd->md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     const upb_fielddef *f = upb_msg_iter_field(&i);
     size_t ofs = rmd->layout.field_offsets[upb_fielddef_index(f)];
 
@@ -1085,8 +1096,10 @@ static void putmsg(rupb_Message *msg, const rupb_MessageDef *rmd,
                    upb_sink *sink) {
   upb_sink_startmsg(sink);
 
-  upb_msg_iter i;
-  for (upb_msg_begin(&i, rmd->md); !upb_msg_done(&i); upb_msg_next(&i)) {
+  upb_msg_field_iter i;
+  for (upb_msg_field_begin(&i, rmd->md);
+       !upb_msg_field_done(&i);
+       upb_msg_field_next(&i)) {
     upb_fielddef *f = upb_msg_iter_field(&i);
     uint32_t ofs = rmd->layout.field_offsets[upb_fielddef_index(f)];