Make all handlers objects refcounted.

I'm realizing that basically all upb objects
will need to be refcounted to be sharable
across languages, but *not* messages which
are on their way out so we can get out of
the business of data representations.

Things which must be refcounted:
- encoders, decoders
- handlers objects
- defs

13 files changed, 179 insertions, 196 deletions

diff --git a/src/upb.h b/src/upb.h
index 3a9dd61..d3e7b34 100644
--- a/src/upb.h
+++ b/src/upb.h
@@ -162,7 +162,7 @@ typedef struct {
     upb_bytesrc *bytesrc;
     upb_msg *msg;
     upb_array *arr;
-    upb_atomic_refcount_t *refcount;
+    upb_atomic_t *refcount;
     upb_fielddef *fielddef;
     void *_void;
   } val;
@@ -204,7 +204,7 @@ UPB_VALUE_ACCESSORS(fielddef, fielddef, upb_fielddef*, UPB_VALUETYPE_FIELDDEF);
 
 extern upb_value UPB_NO_VALUE;
 
-INLINE upb_atomic_refcount_t *upb_value_getrefcount(upb_value val) {
+INLINE upb_atomic_t *upb_value_getrefcount(upb_value val) {
   assert(val.type == UPB_TYPE(MESSAGE) ||
          val.type == UPB_TYPE(STRING) ||
          val.type == UPB_VALUETYPE_ARRAY);
diff --git a/src/upb_atomic.h b/src/upb_atomic.h
index b1ba60d..53501b5 100644
--- a/src/upb_atomic.h
+++ b/src/upb_atomic.h
@@ -39,35 +39,19 @@ extern "C" {
 
 typedef struct {
   int v;
-} upb_atomic_refcount_t;
+} upb_atomic_t;
 
-INLINE void upb_atomic_refcount_init(upb_atomic_refcount_t *a, int val) {
-  a->v = val;
-}
-
-INLINE bool upb_atomic_ref(upb_atomic_refcount_t *a) {
-  return a->v++ == 0;
-}
-
-INLINE bool upb_atomic_unref(upb_atomic_refcount_t *a) {
-  return --a->v == 0;
-}
+#define UPB_ATOMIC_INIT(x) {x}
 
-INLINE int upb_atomic_read(upb_atomic_refcount_t *a) {
-  return a->v;
-}
-
-INLINE bool upb_atomic_add(upb_atomic_refcount_t *a, int val) {
+INLINE void upb_atomic_init(upb_atomic_t *a, int val) { a->v = val; }
+INLINE bool upb_atomic_ref(upb_atomic_t *a) { return a->v++ == 0; }
+INLINE bool upb_atomic_unref(upb_atomic_t *a) { return --a->v == 0; }
+INLINE int upb_atomic_read(upb_atomic_t *a) { return a->v; }
+INLINE bool upb_atomic_add(upb_atomic_t *a, int val) {
   a->v += val;
   return a->v == 0;
 }
 
-INLINE int upb_atomic_fetch_and_add(upb_atomic_refcount_t *a, int val) {
-  int ret = a->v;
-  a->v += val;
-  return ret;
-}
-
 #endif
 
 /* Atomic refcount ************************************************************/
@@ -82,26 +66,26 @@ INLINE int upb_atomic_fetch_and_add(upb_atomic_refcount_t *a, int val) {
 
 typedef struct {
   volatile int v;
-} upb_atomic_refcount_t;
+} upb_atomic_t;
 
-INLINE void upb_atomic_refcount_init(upb_atomic_refcount_t *a, int val) {
+INLINE void upb_atomic_init(upb_atomic_t *a, int val) {
   a->v = val;
   __sync_synchronize();   /* Ensure the initialized value is visible. */
 }
 
-INLINE bool upb_atomic_ref(upb_atomic_refcount_t *a) {
+INLINE bool upb_atomic_ref(upb_atomic_t *a) {
   return __sync_fetch_and_add(&a->v, 1) == 0;
 }
 
-INLINE bool upb_atomic_add(upb_atomic_refcount_t *a, int n) {
+INLINE bool upb_atomic_add(upb_atomic_t *a, int n) {
   return __sync_add_and_fetch(&a->v, n) == 0;
 }
 
-INLINE bool upb_atomic_unref(upb_atomic_refcount_t *a) {
+INLINE bool upb_atomic_unref(upb_atomic_t *a) {
   return __sync_sub_and_fetch(&a->v, 1) == 0;
 }
 
-INLINE bool upb_atomic_read(upb_atomic_refcount_t *a) {
+INLINE bool upb_atomic_read(upb_atomic_t *a) {
   return __sync_fetch_and_add(&a->v, 0);
 }
 
@@ -112,17 +96,17 @@ INLINE bool upb_atomic_read(upb_atomic_refcount_t *a) {
 
 typedef struct {
   volatile LONG val;
-} upb_atomic_refcount_t;
+} upb_atomic_t;
 
-INLINE void upb_atomic_refcount_init(upb_atomic_refcount_t *a, int val) {
+INLINE void upb_atomic_init(upb_atomic_t *a, int val) {
   InterlockedExchange(&a->val, val);
 }
 
-INLINE bool upb_atomic_ref(upb_atomic_refcount_t *a) {
+INLINE bool upb_atomic_ref(upb_atomic_t *a) {
   return InterlockedIncrement(&a->val) == 1;
 }
 
-INLINE bool upb_atomic_unref(upb_atomic_refcount_t *a) {
+INLINE bool upb_atomic_unref(upb_atomic_t *a) {
   return InterlockedDecrement(&a->val) == 0;
 }
 
@@ -131,7 +115,7 @@ INLINE bool upb_atomic_unref(upb_atomic_refcount_t *a) {
        Implement them or compile with UPB_THREAD_UNSAFE.
 #endif
 
-INLINE bool upb_atomic_only(upb_atomic_refcount_t *a) {
+INLINE bool upb_atomic_only(upb_atomic_t *a) {
   return upb_atomic_read(a) == 1;
 }
 
diff --git a/src/upb_decoder.c b/src/upb_decoder.c
index 68fb7a5..34cd811 100644
--- a/src/upb_decoder.c
+++ b/src/upb_decoder.c
@@ -399,10 +399,10 @@ void upb_decoder_reset(upb_decoder *d, upb_bytesrc *bytesrc, void *closure) {
 }
 
 void upb_decoder_uninit(upb_decoder *d) {
-  upb_dispatcher_uninit(&d->dispatcher);
   upb_string_unref(d->bufstr);
   upb_string_unref(d->tmp);
 #ifdef UPB_USE_JIT_X64
   if (d->dispatcher.handlers->should_jit) upb_decoder_freejit(d);
 #endif
+  upb_dispatcher_uninit(&d->dispatcher);
 }
diff --git a/src/upb_decoder_x86.dasc b/src/upb_decoder_x86.dasc
index 1780c98..fec0ffe 100644
--- a/src/upb_decoder_x86.dasc
+++ b/src/upb_decoder_x86.dasc
@@ -286,7 +286,7 @@ static void upb_decoder_jit_field(upb_decoder *d, uint32_t tag, uint32_t next_ta
   |  jne  ->exit_jit     // In the future: could be an unknown field or packed.
   |=>f->jit_pclabel_notypecheck:
   if (f->repeated) {
-    if (f->startseq != upb_startfield_nop) {
+    if (f->startseq) {
       |  mov   ARG1_64, CLOSURE
       |  loadfval f
       |  callp f->startseq
@@ -394,7 +394,7 @@ static void upb_decoder_jit_field(upb_decoder *d, uint32_t tag, uint32_t next_ta
   // Call callbacks.
   if (upb_issubmsgtype(f->type)) {
     // Call startsubmsg handler (if any).
-    if (f->startsubmsg != upb_startfield_nop) {
+    if (f->startsubmsg) {
       // upb_sflow_t startsubmsg(void *closure, upb_value fval)
       |  mov   r12d, ARG3_32
       |  callp f->startsubmsg
@@ -423,7 +423,7 @@ static void upb_decoder_jit_field(upb_decoder *d, uint32_t tag, uint32_t next_ta
     |   popframe
 
     // Call endsubmsg handler (if any).
-    if (f->endsubmsg != upb_endfield_nop) {
+    if (f->endsubmsg) {
       // upb_flow_t endsubmsg(void *closure, upb_value fval);
       |  mov   ARG1_64, CLOSURE
       |  loadfval  f
@@ -441,7 +441,7 @@ static void upb_decoder_jit_field(upb_decoder *d, uint32_t tag, uint32_t next_ta
     |  checktag  tag
     |  je  <1
     |  popframe
-    if (f->endseq != upb_endfield_nop) {
+    if (f->endseq) {
       |  mov   ARG1_64, CLOSURE
       |  loadfval f
       |  callp f->endseq
@@ -466,7 +466,7 @@ static int upb_compare_uint32(const void *a, const void *b) {
 static void upb_decoder_jit_msg(upb_decoder *d, upb_mhandlers *m) {
   |=>m->jit_startmsg_pclabel:
   // Call startmsg handler (if any):
-  if (m->startmsg != upb_startmsg_nop) {
+  if (m->startmsg) {
     // upb_flow_t startmsg(void *closure);
     |  mov   ARG1_64, FRAME->closure
     |  callp m->startmsg
@@ -520,7 +520,7 @@ static void upb_decoder_jit_msg(upb_decoder *d, upb_mhandlers *m) {
 
   |=>m->jit_endofmsg_pclabel:
   // We are at end-of-submsg: call endmsg handler (if any):
-  if (m->endmsg != upb_endmsg_nop) {
+  if (m->endmsg) {
     // void endmsg(void *closure, upb_status *status) {
     |  mov   ARG1_64, FRAME->closure
     |  lea   ARG2_64, DECODER->dispatcher.status
diff --git a/src/upb_def.c b/src/upb_def.c
index a6fe041..791b885 100644
--- a/src/upb_def.c
+++ b/src/upb_def.c
@@ -172,7 +172,7 @@ static void upb_def_init(upb_def *def, upb_deftype type) {
   def->is_cyclic = 0;  // We detect this later, after resolving refs.
   def->search_depth = 0;
   def->fqname = NULL;
-  upb_atomic_refcount_init(&def->refcount, 1);
+  upb_atomic_init(&def->refcount, 1);
 }
 
 static void upb_def_uninit(upb_def *def) {
@@ -868,7 +868,7 @@ static upb_flow_t upb_msgdef_startmsg(void *_b) {
   upb_defbuilder *b = _b;
   upb_msgdef *m = malloc(sizeof(*m));
   upb_def_init(&m->base, UPB_DEF_MSG);
-  upb_atomic_refcount_init(&m->cycle_refcount, 0);
+  upb_atomic_init(&m->cycle_refcount, 0);
   upb_inttable_init(&m->itof, 4, sizeof(upb_itof_ent));
   upb_strtable_init(&m->ntof, 4, sizeof(upb_ntof_ent));
   m->default_message = NULL;
@@ -898,7 +898,7 @@ static void upb_msgdef_endmsg(void *_b, upb_status *status) {
 
   // Assign offsets in the msg.
   m->set_flags_bytes = upb_div_round_up(n, 8);
-  m->size = sizeof(upb_atomic_refcount_t) + m->set_flags_bytes;
+  m->size = sizeof(upb_atomic_t) + m->set_flags_bytes;
 
   size_t max_align = 0;
   for (int i = 0; i < n; i++) {
@@ -926,7 +926,7 @@ static void upb_msgdef_endmsg(void *_b, upb_status *status) {
     // whole must be a multiple of the greatest alignment of any member.
     size_t offset = upb_align_up(m->size, align);
     // Offsets are relative to the end of the refcount.
-    f->byte_offset = offset - sizeof(upb_atomic_refcount_t);
+    f->byte_offset = offset - sizeof(upb_atomic_t);
     m->size = offset + size;
     max_align = UPB_MAX(max_align, align);
   }
@@ -1242,7 +1242,7 @@ err:
 upb_symtab *upb_symtab_new()
 {
   upb_symtab *s = malloc(sizeof(*s));
-  upb_atomic_refcount_init(&s->refcount, 1);
+  upb_atomic_init(&s->refcount, 1);
   upb_rwlock_init(&s->lock);
   upb_strtable_init(&s->symtab, 16, sizeof(upb_symtab_ent));
   s->fds_msgdef = NULL;
diff --git a/src/upb_def.h b/src/upb_def.h
index c2509ec..776231a 100644
--- a/src/upb_def.h
+++ b/src/upb_def.h
@@ -56,7 +56,7 @@ typedef int8_t upb_deftype_t;
 
 typedef struct {
   upb_string *fqname;  // Fully qualified.
-  upb_atomic_refcount_t refcount;
+  upb_atomic_t refcount;
   upb_deftype_t type;
 
   // The is_cyclic flag could go in upb_msgdef instead of here, because only
@@ -110,7 +110,7 @@ struct _upb_fielddef {
 
   // For the case of an enum or a submessage, points to the def for that type.
   upb_def *def;
-  upb_atomic_refcount_t refcount;
+  upb_atomic_t refcount;
 };
 
 // A variety of tests about the type of a field.
@@ -159,7 +159,7 @@ INLINE bool upb_elem_ismm(upb_fielddef *f) {
 // Structure that describes a single .proto message type.
 typedef struct _upb_msgdef {
   upb_def base;
-  upb_atomic_refcount_t cycle_refcount;
+  upb_atomic_t cycle_refcount;
   uint32_t size;
   uint32_t set_flags_bytes;
 
@@ -283,7 +283,7 @@ INLINE int32_t upb_enum_iter_number(upb_enum_iter iter) {
 // Clients add definitions to the symtab by supplying descriptors (as defined
 // in descriptor.proto) via the upb_stream interface.
 struct _upb_symtab {
-  upb_atomic_refcount_t refcount;
+  upb_atomic_t refcount;
   upb_rwlock_t lock;       // Protects all members except the refcount.
   upb_strtable symtab;     // The symbol table.
   upb_msgdef *fds_msgdef;  // Msgdef for google.protobuf.FileDescriptorSet.
diff --git a/src/upb_glue.c b/src/upb_glue.c
index d54b446..1422463 100644
--- a/src/upb_glue.c
+++ b/src/upb_glue.c
@@ -17,13 +17,13 @@ void upb_strtomsg(upb_string *str, upb_msg *msg, upb_msgdef *md,
   upb_stringsrc_init(&strsrc);
   upb_stringsrc_reset(&strsrc, str);
 
-  upb_handlers h;
-  upb_handlers_init(&h);
-  upb_msg_reghandlers(&h, md);
+  upb_handlers *h = upb_handlers_new();
+  upb_msg_reghandlers(h, md);
 
   upb_decoder d;
-  upb_decoder_init(&d, &h);
+  upb_decoder_init(&d, h);
   upb_decoder_reset(&d, upb_stringsrc_bytesrc(&strsrc), msg);
+  upb_handlers_unref(h);
 
   upb_decoder_decode(&d, status);
 
@@ -38,13 +38,12 @@ void upb_msgtotext(upb_string *str, upb_msg *msg, upb_msgdef *md,
   upb_stringsink_reset(&strsink, str);
 
   upb_textprinter *p = upb_textprinter_new();
-  upb_handlers h;
-  upb_handlers_init(&h);
-  upb_textprinter_reghandlers(&h, md);
+  upb_handlers *h = upb_handlers_new();
+  upb_textprinter_reghandlers(h, md);
   upb_textprinter_reset(p, upb_stringsink_bytesink(&strsink), single_line);
 
   upb_status status = UPB_STATUS_INIT;
-  upb_msg_runhandlers(msg, md, &h, p, &status);
+  upb_msg_runhandlers(msg, md, h, p, &status);
   // None of {upb_msg_runhandlers, upb_textprinter, upb_stringsink} should be
   // capable of returning an error.
   assert(upb_ok(&status));
@@ -52,6 +51,7 @@ void upb_msgtotext(upb_string *str, upb_msg *msg, upb_msgdef *md,
 
   upb_stringsink_uninit(&strsink);
   upb_textprinter_free(p);
+  upb_handlers_unref(h);
 }
 
 void upb_parsedesc(upb_symtab *symtab, upb_string *str, upb_status *status) {
@@ -59,12 +59,12 @@ void upb_parsedesc(upb_symtab *symtab, upb_string *str, upb_status *status) {
   upb_stringsrc_init(&strsrc);
   upb_stringsrc_reset(&strsrc, str);
 
-  upb_handlers h;
-  upb_handlers_init(&h);
-  upb_defbuilder_reghandlers(&h);
+  upb_handlers *h = upb_handlers_new();
+  upb_defbuilder_reghandlers(h);
 
   upb_decoder d;
-  upb_decoder_init(&d, &h);
+  upb_decoder_init(&d, h);
+  upb_handlers_unref(h);
   upb_defbuilder *b = upb_defbuilder_new(symtab);
   upb_decoder_reset(&d, upb_stringsrc_bytesrc(&strsrc), b);
 
diff --git a/src/upb_handlers.c b/src/upb_handlers.c
index f4664a0..e630975 100644
--- a/src/upb_handlers.c
+++ b/src/upb_handlers.c
@@ -9,42 +9,13 @@
 #include "upb_handlers.h"
 
 
-upb_flow_t upb_startmsg_nop(void *closure) {
-  (void)closure;
-  return UPB_CONTINUE;
-}
-
-void upb_endmsg_nop(void *closure, upb_status *status) {
-  (void)closure;
-  (void)status;
-}
-
-upb_flow_t upb_value_nop(void *closure, upb_value fval, upb_value val) {
-  (void)closure;
-  (void)fval;
-  (void)val;
-  return UPB_CONTINUE;
-}
-
-upb_sflow_t upb_startfield_nop(void *closure, upb_value fval) {
-  (void)fval;
-  return UPB_CONTINUE_WITH(closure);
-}
-
-upb_flow_t upb_endfield_nop(void *closure, upb_value fval) {
-  (void)closure;
-  (void)fval;
-  return UPB_CONTINUE;
-}
-
-
 /* upb_mhandlers **************************************************************/
 
 static upb_mhandlers *upb_mhandlers_new() {
   upb_mhandlers *m = malloc(sizeof(*m));
   upb_inttable_init(&m->fieldtab, 8, sizeof(upb_fhandlers));
-  m->startmsg = &upb_startmsg_nop;
-  m->endmsg = &upb_endmsg_nop;
+  m->startmsg = NULL;
+  m->endmsg = NULL;
   m->tablearray = NULL;
   m->is_group = false;
   return m;
@@ -57,11 +28,8 @@ static upb_fhandlers *_upb_mhandlers_newfhandlers(upb_mhandlers *m, uint32_t n,
   upb_fhandlers *f = upb_inttable_lookup(&m->fieldtab, tag);
   if (f) abort();
   upb_fhandlers new_f = {false, type, repeated,
-      repeated && upb_isprimitivetype(type), n, m, NULL, UPB_NO_VALUE,
-      &upb_value_nop,
-      &upb_startfield_nop, &upb_endfield_nop,
-      &upb_startfield_nop, &upb_endfield_nop,
-      0, 0, 0, NULL};
+      repeated && upb_isprimitivetype(type), UPB_ATOMIC_INIT(0),
+      n, m, NULL, UPB_NO_VALUE, NULL, NULL, NULL, NULL, NULL, 0, 0, 0, NULL};
   upb_inttable_insert(&m->fieldtab, tag, &new_f);
   f = upb_inttable_lookup(&m->fieldtab, tag);
   assert(f);
@@ -92,21 +60,29 @@ upb_fhandlers *upb_mhandlers_newfhandlers_subm(upb_mhandlers *m, uint32_t n,
 
 /* upb_handlers ***************************************************************/
 
-void upb_handlers_init(upb_handlers *h) {
+upb_handlers *upb_handlers_new() {
+  upb_handlers *h = malloc(sizeof(*h));
+  upb_atomic_init(&h->refcount, 1);
   h->msgs_len = 0;
   h->msgs_size = 4;
   h->msgs = malloc(h->msgs_size * sizeof(*h->msgs));
   h->should_jit = true;
+  return h;
 }
 
-void upb_handlers_uninit(upb_handlers *h) {
-  for (int i = 0; i < h->msgs_len; i++) {
-    upb_mhandlers *mh = h->msgs[i];
-    upb_inttable_free(&mh->fieldtab);
-    free(mh->tablearray);
-    free(mh);
+void upb_handlers_ref(upb_handlers *h) { upb_atomic_ref(&h->refcount); }
+
+void upb_handlers_unref(upb_handlers *h) {
+  if (upb_atomic_unref(&h->refcount)) {
+    for (int i = 0; i < h->msgs_len; i++) {
+      upb_mhandlers *mh = h->msgs[i];
+      upb_inttable_free(&mh->fieldtab);
+      free(mh->tablearray);
+      free(mh);
+    }
+    free(h->msgs);
+    free(h);
   }
-  free(h->msgs);
 }
 
 upb_mhandlers *upb_handlers_newmhandlers(upb_handlers *h) {
@@ -172,7 +148,7 @@ upb_mhandlers *upb_handlers_regmsgdef(upb_handlers *h, upb_msgdef *m,
 /* upb_dispatcher *************************************************************/
 
 static upb_fhandlers toplevel_f = {
-  false, UPB_TYPE(GROUP), false, false, 0,
+  false, UPB_TYPE(GROUP), false, false, UPB_ATOMIC_INIT(0), 0,
   NULL, NULL, // submsg
 #ifdef NDEBUG
   {{0}},
@@ -185,6 +161,7 @@ void upb_dispatcher_init(upb_dispatcher *d, upb_handlers *h,
                          upb_skip_handler *skip, upb_exit_handler *exit,
                          void *srcclosure) {
   d->handlers = h;
+  upb_handlers_ref(h);
   for (int i = 0; i < h->msgs_len; i++) {
     upb_mhandlers *m = h->msgs[i];
     upb_inttable_compact(&m->fieldtab);
@@ -207,18 +184,19 @@ upb_dispatcher_frame *upb_dispatcher_reset(upb_dispatcher *d, void *closure) {
 }
 
 void upb_dispatcher_uninit(upb_dispatcher *d) {
-  upb_handlers_uninit(d->handlers);
+  upb_handlers_unref(d->handlers);
   upb_status_uninit(&d->status);
 }
 
 void upb_dispatch_startmsg(upb_dispatcher *d) {
-  upb_flow_t flow = d->msgent->startmsg(d->top->closure);
+  upb_flow_t flow = UPB_CONTINUE;
+  if (d->msgent->startmsg) d->msgent->startmsg(d->top->closure);
   if (flow != UPB_CONTINUE) _upb_dispatcher_unwind(d, flow);
 }
 
 void upb_dispatch_endmsg(upb_dispatcher *d, upb_status *status) {
   assert(d->top == d->stack);
-  d->msgent->endmsg(d->top->closure, &d->status);
+  if (d->msgent->endmsg) d->msgent->endmsg(d->top->closure, &d->status);
   // TODO: should we avoid this copy by passing client's status obj to cbs?
   upb_copyerr(status, &d->status);
 }
@@ -241,7 +219,8 @@ upb_dispatcher_frame *upb_dispatch_startseq(upb_dispatcher *d,
     return d->top;  // Dummy.
   }
 
-  upb_sflow_t sflow = f->startseq(d->top->closure, f->fval);
+  upb_sflow_t sflow = UPB_CONTINUE_WITH(d->top->closure);
+  if (f->startseq) sflow = f->startseq(d->top->closure, f->fval);
   if (sflow.flow != UPB_CONTINUE) {
     _upb_dispatcher_unwind(d, sflow.flow);
     return d->top;  // Dummy.
@@ -261,7 +240,8 @@ upb_dispatcher_frame *upb_dispatch_endseq(upb_dispatcher *d) {
   assert(d->top->is_sequence);
   upb_fhandlers *f = d->top->f;
   --d->top;
-  upb_flow_t flow = f->endseq(d->top->closure, f->fval);
+  upb_flow_t flow = UPB_CONTINUE;
+  if (f->endseq) flow = f->endseq(d->top->closure, f->fval);
   if (flow != UPB_CONTINUE) {
     printf("YO, UNWINDING!\n");
     _upb_dispatcher_unwind(d, flow);
@@ -282,7 +262,8 @@ upb_dispatcher_frame *upb_dispatch_startsubmsg(upb_dispatcher *d,
     return d->top;  // Dummy.
   }
 
-  upb_sflow_t sflow = f->startsubmsg(d->top->closure, f->fval);
+  upb_sflow_t sflow = UPB_CONTINUE_WITH(d->top->closure);
+  if (f->startsubmsg) sflow = f->startsubmsg(d->top->closure, f->fval);
   if (sflow.flow != UPB_CONTINUE) {
     _upb_dispatcher_unwind(d, sflow.flow);
     return d->top;  // Dummy.
@@ -304,11 +285,12 @@ upb_dispatcher_frame *upb_dispatch_endsubmsg(upb_dispatcher *d) {
   assert(d->top > d->stack);
   assert(!d->top->is_sequence);
   upb_fhandlers *f = d->top->f;
-  d->msgent->endmsg(d->top->closure, &d->status);
+  if (d->msgent->endmsg) d->msgent->endmsg(d->top->closure, &d->status);
   d->msgent = d->top->f->msg;
   d->dispatch_table = &d->msgent->fieldtab;
   --d->top;
-  upb_flow_t flow = f->endsubmsg(d->top->closure, f->fval);
+  upb_flow_t flow = UPB_CONTINUE;
+  if (f->endsubmsg) f->endsubmsg(d->top->closure, f->fval);
   if (flow != UPB_CONTINUE) _upb_dispatcher_unwind(d, flow);
   return d->top;
 }
diff --git a/src/upb_handlers.h b/src/upb_handlers.h
index 6479f7a..caf0645 100644
--- a/src/upb_handlers.h
+++ b/src/upb_handlers.h
@@ -22,7 +22,7 @@
 extern "C" {
 #endif
 
-/* upb_handlers ***************************************************************/
+/* Handlers protocol definition ***********************************************/
 
 // A upb_handlers object represents a graph of handlers.  Each message can have
 // a set of handlers as well as a set of fields which themselves have handlers.
@@ -106,49 +106,42 @@ typedef enum {
   // TODO: Add UPB_SUSPEND, for resumable producers/consumers.
 } upb_flow_t;
 
+// The startsubmsg handler needs to also pass a closure to the submsg.
+typedef struct {
+  upb_flow_t flow;
+  void *closure;
+} upb_sflow_t;
+
+INLINE upb_sflow_t UPB_SFLOW(upb_flow_t flow, void *closure) {
+  upb_sflow_t ret = {flow, closure};
+  return ret;
+}
+#define UPB_CONTINUE_WITH(c) UPB_SFLOW(UPB_CONTINUE, c)
+#define UPB_SBREAK UPB_SFLOW(UPB_BREAK, NULL)
+
 // Typedefs for all of the handler functions defined above.
-typedef struct _upb_sflow upb_sflow_t;
 typedef upb_flow_t (upb_startmsg_handler)(void *c);
 typedef void (upb_endmsg_handler)(void *c, upb_status *status);
 typedef upb_flow_t (upb_value_handler)(void *c, upb_value fval, upb_value val);
 typedef upb_sflow_t (upb_startfield_handler)(void *closure, upb_value fval);
 typedef upb_flow_t (upb_endfield_handler)(void *closure, upb_value fval);
 
-// No-op implementations of all of the above handlers.  Use these instead of
-// rolling your own -- the JIT can recognize these and optimize away the call.
-upb_flow_t upb_startmsg_nop(void *closure);
-void upb_endmsg_nop(void *closure, upb_status *status);
-upb_flow_t upb_value_nop(void *closure, upb_value fval, upb_value val);
-upb_sflow_t upb_startfield_nop(void *closure, upb_value fval);
-upb_flow_t upb_endfield_nop(void *closure, upb_value fval);
 
-// Structure definitions.  Do not access any fields directly!  Accessors are
-// provided for the fields that may be get/set.
-typedef struct _upb_mhandlers {
-  upb_startmsg_handler *startmsg;
-  upb_endmsg_handler *endmsg;
-  upb_inttable fieldtab;  // Maps field number -> upb_fhandlers.
-  uint32_t jit_startmsg_pclabel;
-  uint32_t jit_endofbuf_pclabel;
-  uint32_t jit_endofmsg_pclabel;
-  uint32_t jit_unknownfield_pclabel;
-  bool is_group;
-  int32_t jit_parent_field_done_pclabel;
-  uint32_t max_field_number;
-  // Currently keyed on field number.  Could also try keying it
-  // on encoded or decoded tag, or on encoded field number.
-  void **tablearray;
-} upb_mhandlers;
+/* upb_fhandlers **************************************************************/
 
+// A upb_fhandlers object represents the set of handlers associated with one
+// specific message field.
 struct _upb_decoder;
+struct _upb_mhandlers;
 typedef struct _upb_fieldent {
   bool junk;
   upb_fieldtype_t type;
   bool repeated;
   bool is_repeated_primitive;
+  upb_atomic_t refcount;
   uint32_t number;
-  upb_mhandlers *msg;
-  upb_mhandlers *submsg;  // Must be set iff upb_issubmsgtype(type) == true.
+  struct _upb_mhandlers *msg;
+  struct _upb_mhandlers *submsg;  // Set iff upb_issubmsgtype(type) == true.
   upb_value fval;
   upb_value_handler *value;
   upb_startfield_handler *startsubmsg;
@@ -161,34 +154,50 @@ typedef struct _upb_fieldent {
   void (*decode)(struct _upb_decoder *d, struct _upb_fieldent *f);
 } upb_fhandlers;
 
-struct _upb_handlers {
-  // Array of msgdefs, [0]=toplevel.
-  upb_mhandlers **msgs;
-  int msgs_len, msgs_size;
-  bool should_jit;
-};
-typedef struct _upb_handlers upb_handlers;
+// fhandlers are created as part of a upb_handlers instance, but can be ref'd
+// and unref'd to prolong the life of the handlers.
+void upb_fhandlers_ref(upb_fhandlers *m);
+void upb_fhandlers_unref(upb_fhandlers *m);
 
-void upb_handlers_init(upb_handlers *h);
-void upb_handlers_uninit(upb_handlers *h);
+// upb_fhandlers accessors
+#define UPB_FHANDLERS_ACCESSORS(name, type) \
+  INLINE void upb_fhandlers_set ## name(upb_fhandlers *f, type v){f->name = v;} \
+  INLINE type upb_fhandlers_get ## name(upb_fhandlers *f) { return f->name; }
+UPB_FHANDLERS_ACCESSORS(fval, upb_value)
+UPB_FHANDLERS_ACCESSORS(value, upb_value_handler*)
+UPB_FHANDLERS_ACCESSORS(startsubmsg, upb_startfield_handler*)
+UPB_FHANDLERS_ACCESSORS(endsubmsg, upb_endfield_handler*)
+UPB_FHANDLERS_ACCESSORS(startseq, upb_startfield_handler*)
+UPB_FHANDLERS_ACCESSORS(endseq, upb_endfield_handler*)
+UPB_FHANDLERS_ACCESSORS(submsg, struct _upb_mhandlers*)
 
-// The startsubmsg handler needs to also pass a closure to the submsg.
-struct _upb_sflow {
-  upb_flow_t flow;
-  void *closure;
-};
-INLINE upb_sflow_t UPB_SFLOW(upb_flow_t flow, void *closure) {
-  upb_sflow_t ret = {flow, closure};
-  return ret;
-}
-#define UPB_CONTINUE_WITH(c) UPB_SFLOW(UPB_CONTINUE, c)
-#define UPB_SBREAK UPB_SFLOW(UPB_BREAK, NULL)
 
-// Appends a new message to the graph of handlers and returns it.  This message
-// can be obtained later at index upb_handlers_msgcount()-1.  All handlers will
-// be initialized to no-op handlers.
-upb_mhandlers *upb_handlers_newmhandlers(upb_handlers *h);
-upb_mhandlers *upb_handlers_getmhandlers(upb_handlers *h, int index);
+/* upb_mhandlers **************************************************************/
+
+// A upb_mhandlers object represents the set of handlers associated with a
+// message in the graph of messages.
+
+typedef struct _upb_mhandlers {
+  upb_atomic_t refcount;
+  upb_startmsg_handler *startmsg;
+  upb_endmsg_handler *endmsg;
+  upb_inttable fieldtab;  // Maps field number -> upb_fhandlers.
+  uint32_t jit_startmsg_pclabel;
+  uint32_t jit_endofbuf_pclabel;
+  uint32_t jit_endofmsg_pclabel;
+  uint32_t jit_unknownfield_pclabel;
+  bool is_group;
+  int32_t jit_parent_field_done_pclabel;
+  uint32_t max_field_number;
+  // Currently keyed on field number.  Could also try keying it
+  // on encoded or decoded tag, or on encoded field number.
+  void **tablearray;
+} upb_mhandlers;
+
+// mhandlers are created as part of a upb_handlers instance, but can be ref'd
+// and unref'd to prolong the life of the handlers.
+void upb_mhandlers_ref(upb_mhandlers *m);
+void upb_mhandlers_unref(upb_mhandlers *m);
 
 // Creates a new field with the given name and number.  There must not be an
 // existing field with either this name or number or abort() will be called.
@@ -209,17 +218,26 @@ upb_fhandlers *upb_mhandlers_newfhandlers_subm(upb_mhandlers *m, uint32_t n,
 UPB_MHANDLERS_ACCESSORS(startmsg, upb_startmsg_handler*);
 UPB_MHANDLERS_ACCESSORS(endmsg, upb_endmsg_handler*);
 
-// upb_fhandlers accessors
-#define UPB_FHANDLERS_ACCESSORS(name, type) \
-  INLINE void upb_fhandlers_set ## name(upb_fhandlers *f, type v){f->name = v;} \
-  INLINE type upb_fhandlers_get ## name(upb_fhandlers *f) { return f->name; }
-UPB_FHANDLERS_ACCESSORS(fval, upb_value)
-UPB_FHANDLERS_ACCESSORS(value, upb_value_handler*)
-UPB_FHANDLERS_ACCESSORS(startsubmsg, upb_startfield_handler*)
-UPB_FHANDLERS_ACCESSORS(endsubmsg, upb_endfield_handler*)
-UPB_FHANDLERS_ACCESSORS(startseq, upb_startfield_handler*)
-UPB_FHANDLERS_ACCESSORS(endseq, upb_endfield_handler*)
-UPB_FHANDLERS_ACCESSORS(submsg, upb_mhandlers*)
+
+/* upb_handlers ***************************************************************/
+
+struct _upb_handlers {
+  upb_atomic_t refcount;
+  upb_mhandlers **msgs;  // Array of msgdefs, [0]=toplevel.
+  int msgs_len, msgs_size;
+  bool should_jit;
+};
+typedef struct _upb_handlers upb_handlers;
+
+upb_handlers *upb_handlers_new();
+void upb_handlers_ref(upb_handlers *h);
+void upb_handlers_unref(upb_handlers *h);
+
+// Appends a new message to the graph of handlers and returns it.  This message
+// can be obtained later at index upb_handlers_msgcount()-1.  All handlers will
+// be initialized to no-op handlers.
+upb_mhandlers *upb_handlers_newmhandlers(upb_handlers *h);
+upb_mhandlers *upb_handlers_getmhandlers(upb_handlers *h, int index);
 
 // Convenience function for registering handlers for all messages and
 // fields in a msgdef and all its children.  For every registered message
@@ -338,16 +356,15 @@ void _upb_dispatcher_unwind(upb_dispatcher *d, upb_flow_t flow);
 // Dispatch functions -- call the user handler and handle errors.
 INLINE void upb_dispatch_value(upb_dispatcher *d, upb_fhandlers *f,
                                upb_value val) {
-  upb_flow_t flow = f->value(d->top->closure, f->fval, val);
+  upb_flow_t flow = UPB_CONTINUE;
+  if (f->value) flow = f->value(d->top->closure, f->fval, val);
   if (flow != UPB_CONTINUE) _upb_dispatcher_unwind(d, flow);
 }
 void upb_dispatch_startmsg(upb_dispatcher *d);
 void upb_dispatch_endmsg(upb_dispatcher *d, upb_status *status);
-upb_dispatcher_frame *upb_dispatch_startsubmsg(upb_dispatcher *d,
-                                               upb_fhandlers *f);
+upb_dispatcher_frame *upb_dispatch_startsubmsg(upb_dispatcher *d, upb_fhandlers *f);
 upb_dispatcher_frame *upb_dispatch_endsubmsg(upb_dispatcher *d);
-upb_dispatcher_frame *upb_dispatch_startseq(upb_dispatcher *d,
-                                               upb_fhandlers *f);
+upb_dispatcher_frame *upb_dispatch_startseq(upb_dispatcher *d, upb_fhandlers *f);
 upb_dispatcher_frame *upb_dispatch_endseq(upb_dispatcher *d);
 
 #ifdef __cplusplus
diff --git a/src/upb_msg.c b/src/upb_msg.c
index 9f1f00d..91f1454 100644
--- a/src/upb_msg.c
+++ b/src/upb_msg.c
@@ -38,7 +38,7 @@ static void upb_elem_free(upb_value v, upb_fielddef *f) {
 
 static void upb_elem_unref(upb_value v, upb_fielddef *f) {
   assert(upb_elem_ismm(f));
-  upb_atomic_refcount_t *refcount = upb_value_getrefcount(v);
+  upb_atomic_t *refcount = upb_value_getrefcount(v);
   if (refcount && upb_atomic_unref(refcount))
     upb_elem_free(v, f);
 }
@@ -53,7 +53,7 @@ static void upb_field_free(upb_value v, upb_fielddef *f) {
 
 static void upb_field_unref(upb_value v, upb_fielddef *f) {
   assert(upb_field_ismm(f));
-  upb_atomic_refcount_t *refcount = upb_value_getrefcount(v);
+  upb_atomic_t *refcount = upb_value_getrefcount(v);
   if (refcount && upb_atomic_unref(refcount))
     upb_field_free(v, f);
 }
@@ -63,7 +63,7 @@ static void upb_field_unref(upb_value v, upb_fielddef *f) {
 
 upb_array *upb_array_new(void) {
   upb_array *arr = malloc(sizeof(*arr));
-  upb_atomic_refcount_init(&arr->refcount, 1);
+  upb_atomic_init(&arr->refcount, 1);
   arr->size = 0;
   arr->len = 0;
   arr->ptr = NULL;
@@ -134,7 +134,7 @@ upb_msg *upb_msg_new(upb_msgdef *md) {
   upb_msg *msg = malloc(md->size);
   // Clear all set bits and cached pointers.
   memset(msg, 0, md->size);
-  upb_atomic_refcount_init(&msg->refcount, 1);
+  upb_atomic_init(&msg->refcount, 1);
   return msg;
 }
 
@@ -178,7 +178,7 @@ void upb_msg_set(upb_msg *msg, upb_fielddef *f, upb_value val) {
     upb_field_unref(oldval, f);
 
     // Ref the new value.
-    upb_atomic_refcount_t *refcount = upb_value_getrefcount(val);
+    upb_atomic_t *refcount = upb_value_getrefcount(val);
     if (refcount) upb_atomic_ref(refcount);
   }
   upb_msg_sethas(msg, f);
@@ -194,7 +194,7 @@ upb_value upb_msg_get(upb_msg *msg, upb_fielddef *f) {
       upb_msg *m = upb_msg_new(md);
       // Copy all set bits and values, except the refcount.
       memcpy(m , upb_value_getmsg(val), md->size);
-      upb_atomic_refcount_init(&m->refcount, 0); // The msg will take a ref.
+      upb_atomic_init(&m->refcount, 0); // The msg will take a ref.
       upb_value_setmsg(&val, m);
     }
     upb_msg_set(msg, f, val);
diff --git a/src/upb_msg.h b/src/upb_msg.h
index ac113a8..4e1b4d5 100644
--- a/src/upb_msg.h
+++ b/src/upb_msg.h
@@ -142,7 +142,7 @@ INLINE void upb_value_write(upb_valueptr ptr, upb_value val,
 
 typedef uint32_t upb_arraylen_t;
 struct _upb_array {
-  upb_atomic_refcount_t refcount;
+  upb_atomic_t refcount;
   // "len" and "size" are measured in elements, not bytes.
   int32_t len;
   int32_t size;
@@ -192,7 +192,7 @@ INLINE upb_value upb_array_get(upb_array *arr, upb_fielddef *f,
 // 2. you would want the msg to own a ref on its msgdef, but this would require
 //    an atomic operation for every message create or destroy!
 struct _upb_msg {
-  upb_atomic_refcount_t refcount;
+  upb_atomic_t refcount;
   uint8_t data[4];  // We allocate the appropriate amount per message.
 };
 
diff --git a/src/upb_string.c b/src/upb_string.c
index 8625f76..122eec4 100644
--- a/src/upb_string.c
+++ b/src/upb_string.c
@@ -35,7 +35,7 @@ upb_string *upb_string_new() {
   str->size = 0;
 #endif
   str->src = NULL;
-  upb_atomic_refcount_init(&str->refcount, 1);
+  upb_atomic_init(&str->refcount, 1);
   return str;
 }
 
diff --git a/src/upb_string.h b/src/upb_string.h
index e017268..1463bbf 100644
--- a/src/upb_string.h
+++ b/src/upb_string.h
@@ -59,7 +59,7 @@ extern "C" {
 // the associated functions.
 struct _upb_string {
   // The string's refcount.
-  upb_atomic_refcount_t refcount;
+  upb_atomic_t refcount;
 
   // The pointer to our currently active data.  This may be memory we own
   // or a pointer into memory we don't own.