Sync from Google-internal development.

Most notably, a JSON parser is now provided.

1 files changed, 373 insertions, 486 deletions

diff --git a/upb/pb/compile_decoder_x64.h b/upb/pb/compile_decoder_x64.h
index 9787a57..095d2ef 100644
--- a/upb/pb/compile_decoder_x64.h
+++ b/upb/pb/compile_decoder_x64.h
@@ -21,19 +21,22 @@
 //|
 //|.arch x64
 //|.actionlist upb_jit_actionlist
-static const unsigned char upb_jit_actionlist[1914] = {
-  249,255,248,10,248,1,85,65,87,65,86,65,85,65,84,83,72,131,252,236,8,72,137,
-  252,243,73,137,252,255,255,72,184,237,237,252,255,208,255,133,192,15,137,
-  244,247,73,137,167,233,72,137,216,77,139,183,233,73,139,159,233,77,139,167,
-  233,77,139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,73,139,151,233,
-  72,133,210,15,133,244,248,252,255,208,73,139,135,233,73,199,135,233,0,0,0,
-  0,248,1,72,131,196,8,91,65,92,65,93,65,94,65,95,93,195,248,2,73,141,183,233,
-  72,41,212,72,137,231,255,195,255,248,11,73,141,191,233,72,137,230,73,139,
-  151,233,72,41,226,73,137,151,233,137,195,255,137,216,73,139,167,233,72,131,
-  196,8,91,65,92,65,93,65,94,65,95,93,195,255,248,12,73,57,159,233,15,132,244,
-  247,73,137,159,233,248,1,77,137,183,233,73,137,159,233,77,137,167,233,73,
-  137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,76,137,
-  252,255,255,252,233,244,11,255,248,13,248,1,77,137,174,233,73,137,159,233,
+static const unsigned char upb_jit_actionlist[2158] = {
+  249,255,248,10,248,1,85,65,87,65,86,65,85,65,84,83,72,137,252,243,73,137,
+  252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,
+  228,65,92,133,192,15,137,244,247,73,137,167,233,72,137,216,77,139,183,233,
+  73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,
+  73,3,175,233,73,139,151,233,72,133,210,15,133,244,248,252,255,208,73,139,
+  135,233,73,199,135,233,0,0,0,0,248,1,255,91,65,92,65,93,65,94,65,95,93,195,
+  248,2,73,141,183,233,72,41,212,72,137,231,72,184,237,237,65,84,73,137,228,
+  72,129,228,239,252,255,208,76,137,228,65,92,195,255,248,11,73,141,191,233,
+  72,137,230,73,139,151,233,72,41,226,73,137,151,233,137,195,72,184,237,237,
+  65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,137,216,73,139,
+  167,233,91,65,92,65,93,65,94,65,95,93,195,255,248,12,73,57,159,233,15,132,
+  244,247,73,137,159,233,248,1,77,137,183,233,73,137,159,233,77,137,167,233,
+  73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,76,
+  137,252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,
+  137,228,65,92,252,233,244,11,255,248,13,248,1,77,137,174,233,73,137,159,233,
   255,76,57,227,15,132,244,253,255,76,137,225,72,41,217,72,131,252,249,1,15,
   130,244,253,255,15,182,19,132,210,15,137,244,254,248,7,232,244,14,248,8,72,
   131,195,1,72,137,252,233,72,41,217,72,41,209,15,130,244,15,73,137,142,233,
@@ -41,51 +44,58 @@ static const unsigned char upb_jit_actionlist[1914] = {
   233,0,0,0,0,72,133,201,15,132,244,248,77,139,167,233,72,57,252,235,15,135,
   244,248,76,57,229,15,135,244,248,255,73,137,252,236,248,2,195,248,3,73,139,
   159,233,76,137,252,255,255,72,190,237,237,255,190,237,255,49,252,246,255,
-  232,244,12,252,233,244,1,255,248,16,72,131,252,236,16,72,137,188,253,36,233,
-  248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137,
-  183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,
-  233,73,137,174,233,77,137,174,233,252,255,148,253,36,233,77,139,183,233,73,
-  139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,73,
-  3,175,233,255,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72,131,
-  196,16,195,248,2,232,244,11,252,233,244,1,255,248,17,76,137,252,255,137,214,
-  15,182,209,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,
-  43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,255,77,139,183,233,
-  73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,
-  73,3,175,233,129,252,248,239,15,133,244,247,195,248,1,129,252,248,239,15,
-  132,244,247,232,244,11,248,1,49,192,195,255,248,18,248,19,72,191,237,237,
-  232,244,16,72,131,252,235,4,73,137,159,233,195,255,248,20,248,21,72,191,237,
-  237,232,244,16,72,131,252,235,8,73,137,159,233,195,255,248,22,248,23,255,
-  76,57,227,15,132,244,247,255,76,137,225,72,41,217,72,131,252,249,16,15,130,
-  244,247,255,252,243,15,111,3,102,15,215,192,252,247,208,15,188,192,60,10,
-  15,131,244,24,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73,15,
-  71,204,248,2,72,131,192,1,72,57,200,15,132,244,24,252,246,0,128,15,133,244,
-  2,248,3,72,137,195,195,255,248,25,72,131,252,236,16,248,1,72,57,252,235,15,
-  133,244,248,72,131,196,16,49,192,195,248,2,76,137,252,255,72,137,230,77,137,
-  183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,
-  233,73,137,174,233,77,137,174,233,255,77,139,183,233,73,139,159,233,77,139,
-  167,233,77,139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,131,252,
-  248,0,15,141,244,249,139,20,36,72,131,196,16,195,248,3,232,244,11,252,233,
-  244,1,255,248,14,248,26,255,76,57,227,15,132,244,24,255,76,137,225,72,41,
-  217,72,131,252,249,10,15,130,244,24,255,72,137,223,255,72,133,192,15,132,
+  72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,
+  92,232,244,12,252,233,244,1,255,248,16,72,131,252,236,16,72,137,188,253,36,
+  233,248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,
+  137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,
+  3,175,233,73,137,174,233,77,137,174,233,252,255,148,253,36,233,77,139,183,
+  233,73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,
+  233,73,3,175,233,255,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4,
+  36,72,131,196,16,195,248,2,232,244,11,252,233,244,1,255,248,17,76,137,252,
+  255,137,214,15,182,209,77,137,183,233,73,137,159,233,77,137,167,233,73,137,
+  175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,
+  237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,
+  233,73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,
+  233,73,3,175,233,129,252,248,239,255,15,133,244,247,195,248,1,129,252,248,
+  239,15,132,244,247,232,244,11,248,1,49,192,195,255,248,18,248,19,72,191,237,
+  237,232,244,16,72,131,252,235,4,73,137,159,233,195,255,248,20,248,21,72,191,
+  237,237,232,244,16,72,131,252,235,8,73,137,159,233,195,255,248,22,248,23,
+  255,76,57,227,15,132,244,247,255,76,137,225,72,41,217,72,131,252,249,16,15,
+  130,244,247,255,252,243,15,111,3,102,15,215,192,252,247,208,15,188,192,60,
+  10,15,131,244,24,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73,
+  15,71,204,248,2,72,131,192,1,72,57,200,15,132,244,24,252,246,0,128,15,133,
+  244,2,248,3,72,137,195,195,255,248,25,72,131,252,236,16,248,1,72,57,252,235,
+  15,133,244,248,72,131,196,16,49,192,195,248,2,76,137,252,255,72,137,230,77,
+  137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,
+  3,175,233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228,72,
+  129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,
+  139,167,233,77,139,174,233,255,73,139,174,233,73,43,175,233,73,3,175,233,
+  131,252,248,0,15,141,244,249,139,20,36,72,131,196,16,195,248,3,232,244,11,
+  252,233,244,1,255,248,14,248,26,255,76,57,227,15,132,244,24,255,76,137,225,
+  72,41,217,72,131,252,249,10,15,130,244,24,255,72,137,223,72,184,237,237,65,
+  84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,72,133,192,15,132,
   244,24,72,137,195,72,131,252,235,1,73,137,159,233,195,255,248,24,72,191,237,
   237,232,244,16,72,131,252,235,1,73,137,159,233,195,255,248,27,72,131,252,
   236,8,72,137,52,36,248,1,76,137,252,255,77,137,183,233,73,137,159,233,77,
   137,167,233,73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,
-  174,233,73,137,159,233,255,77,139,183,233,73,139,159,233,77,139,167,233,77,
-  139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,131,252,248,0,15,141,
-  244,248,72,131,196,8,195,248,2,232,244,11,72,139,52,36,72,57,252,235,15,133,
-  244,1,184,237,72,131,196,8,195,255,248,28,81,82,72,131,252,236,16,72,137,
-  252,247,72,137,214,72,137,226,255,72,131,196,16,90,89,132,192,15,132,244,
-  248,72,139,68,36,224,195,248,2,72,49,192,72,252,247,208,195,255,248,1,255,
-  76,57,227,15,133,244,249,255,76,137,225,72,41,217,72,129,252,249,239,15,131,
-  244,249,255,248,2,255,232,244,14,255,232,244,26,255,232,244,19,255,232,244,
-  21,255,252,233,244,250,255,248,3,255,139,19,255,72,139,19,255,252,243,15,
-  16,3,255,252,242,15,16,3,255,15,182,19,132,210,15,136,244,2,255,248,4,255,
-  137,208,209,252,234,131,224,1,252,247,216,49,194,255,72,137,208,72,209,252,
-  234,72,131,224,1,72,252,247,216,72,49,194,255,72,133,210,15,149,210,255,73,
-  137,149,233,255,65,137,149,233,255,252,242,65,15,17,133,233,255,252,243,65,
-  15,17,133,233,255,65,136,149,233,255,65,128,141,233,235,255,76,137,252,239,
-  255,132,192,15,133,244,251,232,244,12,252,233,244,1,248,5,255,72,129,195,
+  174,233,73,137,159,233,72,184,237,237,65,84,73,137,228,72,129,228,239,252,
+  255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139,167,233,77,
+  139,174,233,73,139,174,233,73,43,175,233,255,73,3,175,233,131,252,248,0,15,
+  141,244,248,72,131,196,8,195,248,2,232,244,11,72,139,52,36,72,57,252,235,
+  15,133,244,1,184,237,72,131,196,8,195,255,248,28,81,82,72,131,252,236,16,
+  72,137,252,247,72,137,214,72,137,226,72,184,237,237,65,84,73,137,228,72,129,
+  228,239,252,255,208,76,137,228,65,92,72,131,196,16,90,89,132,192,15,132,244,
+  248,72,139,68,36,224,195,248,2,72,49,192,72,252,247,208,195,255,76,57,227,
+  15,133,244,249,255,76,137,225,72,41,217,72,129,252,249,239,15,131,244,249,
+  255,248,2,255,232,244,14,255,232,244,26,255,232,244,19,255,232,244,21,255,
+  252,233,244,250,255,248,3,255,139,19,255,72,139,19,255,252,243,15,16,3,255,
+  252,242,15,16,3,255,15,182,19,132,210,15,136,244,2,255,248,4,255,137,208,
+  209,252,234,131,224,1,252,247,216,49,194,255,72,137,208,72,209,252,234,72,
+  131,224,1,72,252,247,216,72,49,194,255,72,133,210,15,149,210,255,73,137,149,
+  233,255,65,137,149,233,255,252,242,65,15,17,133,233,255,252,243,65,15,17,
+  133,233,255,65,136,149,233,255,65,128,141,233,235,255,76,137,252,239,255,
+  72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,
+  92,255,132,192,15,133,244,251,232,244,12,252,233,244,1,248,5,255,72,129,195,
   239,255,232,244,22,255,232,244,23,255,232,244,18,255,232,244,20,255,252,246,
   3,128,15,133,244,2,255,249,248,1,255,76,57,227,15,132,244,252,255,76,137,
   225,72,41,217,72,131,252,249,2,15,130,244,252,255,15,182,19,132,210,15,137,
@@ -104,11 +114,13 @@ static const unsigned char upb_jit_actionlist[1914] = {
   59,239,15,133,244,249,128,187,233,235,255,15,132,244,250,248,3,255,232,245,
   72,133,192,15,132,245,252,255,224,255,252,233,245,255,248,4,72,129,195,239,
   248,5,255,248,1,76,137,252,239,255,132,192,15,133,244,248,232,244,12,252,
-  233,244,1,248,2,255,144,255,248,9,255,73,139,151,233,255,249,249,72,131,252,
-  236,8,255,72,137,252,234,72,41,218,255,72,133,192,15,133,244,248,232,244,
-  12,252,233,244,1,248,2,255,73,137,197,255,72,57,252,235,15,132,244,250,248,
-  1,76,57,227,15,133,244,248,232,244,12,252,233,244,1,248,2,255,72,137,218,
-  76,137,225,72,41,217,77,139,135,233,255,72,1,195,255,76,57,227,15,132,244,
+  233,244,1,248,2,255,144,255,248,9,255,73,139,151,233,72,184,237,237,65,84,
+  73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,255,249,249,72,131,
+  252,236,8,255,72,137,252,234,72,41,218,255,72,133,192,15,133,244,248,232,
+  244,12,252,233,244,1,248,2,255,73,137,197,255,72,57,252,235,15,132,244,250,
+  248,1,76,57,227,15,133,244,248,232,244,12,252,233,244,1,248,2,255,72,137,
+  218,76,137,225,72,41,217,77,139,135,233,72,184,237,237,65,84,73,137,228,72,
+  129,228,239,252,255,208,76,137,228,65,92,72,1,195,255,76,57,227,15,132,244,
   249,232,244,29,248,3,255,76,137,227,255,72,57,252,235,15,133,244,1,248,4,
   255,77,137,174,233,73,199,134,233,0,0,0,0,73,129,198,239,77,59,183,233,15,
   132,244,15,65,199,134,233,237,255,232,244,13,255,73,129,252,238,239,77,139,
@@ -222,17 +234,21 @@ static const char *const upb_jit_globalnames[] = {
 //|  add  DELIMEND, DECODER->buf
 //|.endmacro
 //|
-//| // OPT: use "call rel32" where possible.
+//| // Calls an external C function at address "addr".
 //|.macro callp, addr
-//|| {
-//|| //int64_t ofs = (int64_t)addr - (int64_t)upb_status_init;
-//|| //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
 //|    mov64  rax, (uintptr_t)addr
+//|
+//|    // Stack must be 16-byte aligned (x86-64 ABI requires this).
+//|    //
+//|    // OPT: possibly remove this by statically ensuring correct alignment.
+//|    //
+//|    // OPT: use "call rel32" where possible.
+//|    push   r12
+//|    mov    r12, rsp
+//|    and    rsp, 0xfffffffffffffff0UL   // Align stack.
 //|    call   rax
-//|| //} else {
-//|  //  call   &addr
-//|| //}
-//|| }
+//|    mov    rsp, r12
+//|    pop    r12
 //|.endmacro
 //|
 //|.macro ld64, val
@@ -331,7 +347,7 @@ static void asmlabel(jitcompiler *jc, const char *fmt, ...) {
   // we do it here instead.
   //|=>pclabel:
   dasm_put(Dst, 0, pclabel);
-# 172 "upb/pb/compile_decoder_x64.dasc"
+# 176 "upb/pb/compile_decoder_x64.dasc"
   upb_inttable_insert(&jc->asmlabels, pclabel, upb_value_ptr(str));
 #endif
 }
@@ -371,25 +387,10 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  push  r12
   //|  push  rbx
   //|
-  //| // Align stack.
-  //| // Since the JIT can call other functions (the JIT'ted code is not a leaf
-  //| // function) we must respect alignment rules.  All x86-64 systems require
-  //| // 16-byte stack alignment.
-  //|  sub   rsp, 8
-  //|
   //|  mov   rbx, ARG2_64  // Preserve JIT method.
   //|
   //|  mov   DECODER, rdi
   //|  callp upb_pbdecoder_resume  // Same args as us; reuse regs.
-  dasm_put(Dst, 2);
-   {
-   //int64_t ofs = (int64_t)upb_pbdecoder_resume - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_resume), (unsigned int)(((uintptr_t)upb_pbdecoder_resume)>>32));
-   //} else {
-   //}
-   }
-# 221 "upb/pb/compile_decoder_x64.dasc"
   //|  test  eax, eax
   //|  jns   >1
   //|  mov   DECODER->saved_rsp, rsp
@@ -406,8 +407,9 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  mov   rax, DECODER->size_param
   //|  mov   qword DECODER->call_len, 0
   //|1:
-  //|  add   rsp, 8  // Counter previous alignment.
   //|  pop   rbx
+  dasm_put(Dst, 2, (unsigned int)((uintptr_t)upb_pbdecoder_resume), (unsigned int)(((uintptr_t)upb_pbdecoder_resume)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->call_len), Dt2(->size_param), Dt2(->call_len));
+# 236 "upb/pb/compile_decoder_x64.dasc"
   //|  pop   r12
   //|  pop   r13
   //|  pop   r14
@@ -421,23 +423,14 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  sub   rsp, ARG3_64
   //|  mov   ARG1_64, rsp
   //|  callp memcpy  // Restore stack.
-  dasm_put(Dst, 38, Dt2(->saved_rsp), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->call_len), Dt2(->size_param), Dt2(->call_len), Dt2(->callstack));
-   {
-   //int64_t ofs = (int64_t)memcpy - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32));
-   //} else {
-   //}
-   }
-# 252 "upb/pb/compile_decoder_x64.dasc"
   //|  ret  // Return to resumed function (not ->enterjit caller).
   //|
   //| // Other code can call this to suspend the JIT.
   //| // To the calling code, it will appear that the function returns when
   //| // the JIT resumes, and more buffer space will be available.
   //| // Args: eax=the value that decode() should return.
-  dasm_put(Dst, 135);
-# 258 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 115, Dt2(->callstack), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL);
+# 255 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "exitjit");
   //|->exitjit:
   //|  // Save the stack into DECODER->callstack.
@@ -448,21 +441,11 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  mov   DECODER->call_len, ARG3_64  // Preserve len for next resume.
   //|  mov   ebx, eax  // Preserve return value across memcpy.
   //|  callp memcpy    // Copy stack into decoder.
-  dasm_put(Dst, 137, Dt2(->callstack), Dt2(->saved_rsp), Dt2(->call_len));
-   {
-   //int64_t ofs = (int64_t)memcpy - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32));
-   //} else {
-   //}
-   }
-# 268 "upb/pb/compile_decoder_x64.dasc"
   //|  mov   eax, ebx  // This will be our return value.
   //|
   //|  // Must NOT do this before the memcpy(), otherwise memcpy() will
   //|  // clobber the stack we are trying to save!
   //|  mov   rsp, DECODER->saved_rsp
-  //|  add   rsp, 8  // Counter previous alignment.
   //|  pop   rbx
   //|  pop   r12
   //|  pop   r13
@@ -474,8 +457,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //| // Like suspend() in the C decoder, except that the function appears
   //| // (from the caller's perspective) not to return until the decoder is
   //| // resumed.
-  dasm_put(Dst, 160, Dt2(->saved_rsp));
-# 285 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 161, Dt2(->callstack), Dt2(->saved_rsp), Dt2(->call_len), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp));
+# 281 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "suspend");
   //|->suspend:
   //|  cmp   DECODER->ptr, PTR
@@ -485,32 +468,23 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  commit_regs
   //|  mov   rdi, DECODER
   //|  callp upb_pbdecoder_suspend
-  dasm_put(Dst, 182, Dt2(->ptr), Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure));
-   {
-   //int64_t ofs = (int64_t)upb_pbdecoder_suspend - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_suspend), (unsigned int)(((uintptr_t)upb_pbdecoder_suspend)>>32));
-   //} else {
-   //}
-   }
-# 294 "upb/pb/compile_decoder_x64.dasc"
   //|  jmp   ->exitjit
   //|
-  dasm_put(Dst, 235);
-# 296 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 222, Dt2(->ptr), Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_suspend), (unsigned int)(((uintptr_t)upb_pbdecoder_suspend)>>32), 0xfffffffffffffff0UL);
+# 292 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "pushlendelim");
   //|->pushlendelim:
   //|1:
   //|  mov   FRAME->sink.closure, CLOSURE
   //|  mov   DECODER->checkpoint, PTR
   //|  dv32
-  dasm_put(Dst, 240, Dt1(->sink.closure), Dt2(->checkpoint));
+  dasm_put(Dst, 300, Dt1(->sink.closure), Dt2(->checkpoint));
    if (1 == 1) {
-  dasm_put(Dst, 253);
+  dasm_put(Dst, 313);
    } else {
-  dasm_put(Dst, 261);
+  dasm_put(Dst, 321);
    }
-# 302 "upb/pb/compile_decoder_x64.dasc"
+# 298 "upb/pb/compile_decoder_x64.dasc"
   //|  mov   rcx, DELIMEND
   //|  sub   rcx, PTR
   //|  sub   rcx, rdx
@@ -530,8 +504,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  cmp   DELIMEND, DATAEND
   //|  ja    >2
   //|  mov   DATAEND, DELIMEND  // If DELIMEND >= PTR && DELIMEND < DATAEND
-  dasm_put(Dst, 277, Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), Dt2(->end));
-# 321 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 337, Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), Dt2(->end));
+# 317 "upb/pb/compile_decoder_x64.dasc"
   //|2:
   //|  ret
   //|3:
@@ -540,34 +514,26 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  // Prepare seterr args.
   //|  mov   ARG1_64, DECODER
   //|  ld64  kPbDecoderStackOverflow
-  dasm_put(Dst, 368, Dt2(->checkpoint));
+  dasm_put(Dst, 428, Dt2(->checkpoint));
    {
    uintptr_t v = (uintptr_t)kPbDecoderStackOverflow;
    if (v > 0xffffffff) {
-  dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+  dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
    } else if (v) {
-  dasm_put(Dst, 391, v);
+  dasm_put(Dst, 451, v);
    } else {
-  dasm_put(Dst, 394);
+  dasm_put(Dst, 454);
    }
    }
-# 329 "upb/pb/compile_decoder_x64.dasc"
+# 325 "upb/pb/compile_decoder_x64.dasc"
   //|  callp upb_pbdecoder_seterr
-   {
-   //int64_t ofs = (int64_t)upb_pbdecoder_seterr - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32));
-   //} else {
-   //}
-   }
-# 330 "upb/pb/compile_decoder_x64.dasc"
   //|  call  ->suspend
   //|  jmp   <1
   //|
   //| // For getting a value that spans a buffer seam.  Falls back to C.
   //| // Args: rdi=C decoding function (prototype: int f(upb_pbdecoder*, void*))
-  dasm_put(Dst, 398);
-# 335 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 458, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32), 0xfffffffffffffff0UL);
+# 331 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "getvalue_slow");
   //|->getvalue_slow:
   //|  sub   rsp, 16         // Stack is [8-byte value, 8-byte func pointer]
@@ -581,8 +547,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  call  aword [rsp + 8]
   //|  load_regs
   //|  test  eax, eax
-  dasm_put(Dst, 406, 8, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), 8, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf));
-# 348 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 487, 8, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), 8, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf));
+# 344 "upb/pb/compile_decoder_x64.dasc"
   //|  jns   >2
   //|  // Success; return parsed data (in rdx AND xmm0).
   //|  mov   rdx, [rsp]
@@ -593,8 +559,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  call  ->exitjit   // Return eax from decode function.
   //|  jmp   <1
   //|
-  dasm_put(Dst, 507);
-# 358 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 588);
+# 354 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "parse_unknown");
   //| // Args: edx=fieldnum, cl=wire type
   //|->parse_unknown:
@@ -605,18 +571,11 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  movzx   ARG3_32, cl
   //|  commit_regs
   //|  callp   upb_pbdecoder_skipunknown
-  dasm_put(Dst, 538, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure));
-   {
-   //int64_t ofs = (int64_t)upb_pbdecoder_skipunknown - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32));
-   //} else {
-   //}
-   }
-# 368 "upb/pb/compile_decoder_x64.dasc"
   //|  load_regs
   //|  cmp     eax, DECODE_ENDGROUP
   //|  jne     >1
+  dasm_put(Dst, 619, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP);
+# 367 "upb/pb/compile_decoder_x64.dasc"
   //|  ret     // Return eax=DECODE_ENDGROUP, not zero
   //|1:
   //|  cmp     eax, DECODE_OK
@@ -635,8 +594,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //| // re-join the fast path which will add fast_path_bytes after the callback
   //| // completes.  We also set DECODER->ptr to this value which is a signal to
   //| // ->suspend that DECODER->checkpoint is up to date.
-  dasm_put(Dst, 582, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP, DECODE_OK);
-# 389 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 716, DECODE_OK);
+# 385 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "skip_decode_f32_fallback");
   //|->skipf32_fallback:
   //|->decodef32_fallback:
@@ -646,8 +605,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  mov      DECODER->ptr, PTR
   //|  ret
   //|
-  dasm_put(Dst, 638, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), Dt2(->ptr));
-# 398 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 740, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), Dt2(->ptr));
+# 394 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "skip_decode_f64_fallback");
   //|->skipf64_fallback:
   //|->decodef64_fallback:
@@ -658,19 +617,19 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  ret
   //|
   //| // Called for varint >= 1 byte.
-  dasm_put(Dst, 660, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), Dt2(->ptr));
-# 408 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 762, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), Dt2(->ptr));
+# 404 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "skip_decode_v32_fallback");
   //|->skipv32_fallback:
   //|->skipv64_fallback:
   //|  chkeob   16, >1
-  dasm_put(Dst, 682);
+  dasm_put(Dst, 784);
    if (16 == 1) {
-  dasm_put(Dst, 687);
+  dasm_put(Dst, 789);
    } else {
-  dasm_put(Dst, 695);
+  dasm_put(Dst, 797);
    }
-# 412 "upb/pb/compile_decoder_x64.dasc"
+# 408 "upb/pb/compile_decoder_x64.dasc"
   //|  // With at least 16 bytes left, we can do a branch-less SSE version.
   //|  movdqu   xmm0, [PTR]
   //|  pmovmskb eax, xmm0   // bits 0-15 are continuation bits, 16-31 are 0.
@@ -698,8 +657,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  ret
   //|
   //| // Returns tag in edx
-  dasm_put(Dst, 711, 10);
-# 439 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 813, 10);
+# 435 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "decode_unknown_tag_fallback");
   //|->decode_unknown_tag_fallback:
   //|  sub   rsp, 16
@@ -715,16 +674,9 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  mov   ARG2_64, rsp
   //|  commit_regs
   //|  callp upb_pbdecoder_decode_varint_slow
-  dasm_put(Dst, 780, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure));
-   {
-   //int64_t ofs = (int64_t)upb_pbdecoder_decode_varint_slow - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32));
-   //} else {
-   //}
-   }
-# 454 "upb/pb/compile_decoder_x64.dasc"
   //|  load_regs
+  dasm_put(Dst, 882, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure));
+# 451 "upb/pb/compile_decoder_x64.dasc"
   //|  cmp   eax, 0
   //|  jge   >3
   //|  mov   edx, [rsp]   // Success; return parsed data.
@@ -735,31 +687,22 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  jmp   <1
   //|
   //| // Called for varint >= 1 byte.
-  dasm_put(Dst, 846, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf));
-# 465 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 985, Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf));
+# 461 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "decode_v32_v64_fallback");
   //|->decodev32_fallback:
   //|->decodev64_fallback:
   //|  chkeob   10, ->decode_varint_slow
-  dasm_put(Dst, 900);
+  dasm_put(Dst, 1023);
    if (10 == 1) {
-  dasm_put(Dst, 905);
+  dasm_put(Dst, 1028);
    } else {
-  dasm_put(Dst, 913);
+  dasm_put(Dst, 1036);
    }
-# 469 "upb/pb/compile_decoder_x64.dasc"
+# 465 "upb/pb/compile_decoder_x64.dasc"
   //|  // OPT: do something faster than just calling the C version.
   //|  mov      rdi, PTR
   //|  callp    upb_vdecode_fast
-  dasm_put(Dst, 929);
-   {
-   //int64_t ofs = (int64_t)upb_vdecode_fast - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32));
-   //} else {
-   //}
-   }
-# 472 "upb/pb/compile_decoder_x64.dasc"
   //|  test     rax, rax
   //|  je       ->decode_varint_slow  // Unterminated varint.
   //|  mov      PTR, rax
@@ -767,8 +710,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  mov      DECODER->ptr, PTR
   //|  ret
   //|
-  dasm_put(Dst, 933, Dt2(->ptr));
-# 479 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1052, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32), 0xfffffffffffffff0UL, Dt2(->ptr));
+# 475 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "decode_varint_slow");
   //|->decode_varint_slow:
   //|  // Slow path: end of buffer or error (varint length >= 10).
@@ -779,8 +722,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  ret
   //|
   //| // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH})
-  dasm_put(Dst, 954, (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), Dt2(->ptr));
-# 489 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1097, (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), Dt2(->ptr));
+# 485 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "checktag_fallback");
   //|->checktag_fallback:
   //|  sub      rsp, 8
@@ -790,16 +733,9 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  commit_regs
   //|  mov      DECODER->checkpoint, PTR
   //|  callp    upb_pbdecoder_checktag_slow
-  dasm_put(Dst, 974, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint));
-   {
-   //int64_t ofs = (int64_t)upb_pbdecoder_checktag_slow - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32));
-   //} else {
-   //}
-   }
-# 498 "upb/pb/compile_decoder_x64.dasc"
   //|  load_regs
+  dasm_put(Dst, 1117, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint), (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs));
+# 495 "upb/pb/compile_decoder_x64.dasc"
   //|  cmp      eax, 0
   //|  jge      >2
   //|  add      rsp, 8
@@ -816,8 +752,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //| // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found).
   //| // Preserves: rcx, rdx
   //| // OPT: Could write this in assembly if it's a hotspot.
-  dasm_put(Dst, 1028, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_EOF);
-# 515 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1216, Dt2(->buf), DECODE_EOF);
+# 511 "upb/pb/compile_decoder_x64.dasc"
   asmlabel(jc, "hashlookup");
   //|->hashlookup:
   //|  push   rcx
@@ -827,15 +763,6 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  mov    rsi, rdx
   //|  mov    rdx, rsp
   //|  callp  upb_inttable_lookup
-  dasm_put(Dst, 1094);
-   {
-   //int64_t ofs = (int64_t)upb_inttable_lookup - (int64_t)upb_status_init;
-   //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-  dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32));
-   //} else {
-   //}
-   }
-# 524 "upb/pb/compile_decoder_x64.dasc"
   //|  add    rsp, 16
   //|  pop    rdx
   //|  pop    rcx
@@ -847,8 +774,8 @@ static void emit_static_asm(jitcompiler *jc) {
   //|  xor    rax, rax
   //|  not    rax
   //|  ret
-  dasm_put(Dst, 1114);
-# 535 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1258, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32), 0xfffffffffffffff0UL);
+# 531 "upb/pb/compile_decoder_x64.dasc"
 }
 
 static void jitprimitive(jitcompiler *jc, opcode op,
@@ -865,85 +792,85 @@ static void jitprimitive(jitcompiler *jc, opcode op,
   if (handler) {
     //|1:
     //|  chkneob  fastbytes, >3
-    dasm_put(Dst, 1143);
+    dasm_put(Dst, 112);
      if (fastbytes == 1) {
-    dasm_put(Dst, 1146);
+    dasm_put(Dst, 1327);
      } else {
-    dasm_put(Dst, 1154, fastbytes);
+    dasm_put(Dst, 1335, fastbytes);
      }
-# 551 "upb/pb/compile_decoder_x64.dasc"
+# 547 "upb/pb/compile_decoder_x64.dasc"
     //|2:
-    dasm_put(Dst, 1170);
-# 552 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1351);
+# 548 "upb/pb/compile_decoder_x64.dasc"
     switch (type) {
     case V32:
       //|  call   ->decodev32_fallback
-      dasm_put(Dst, 1173);
-# 555 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1354);
+# 551 "upb/pb/compile_decoder_x64.dasc"
       break;
     case V64:
       //|  call   ->decodev64_fallback
-      dasm_put(Dst, 1177);
-# 558 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1358);
+# 554 "upb/pb/compile_decoder_x64.dasc"
       break;
     case F32:
       //|  call   ->decodef32_fallback
-      dasm_put(Dst, 1181);
-# 561 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1362);
+# 557 "upb/pb/compile_decoder_x64.dasc"
       break;
     case F64:
       //|  call   ->decodef64_fallback
-      dasm_put(Dst, 1185);
-# 564 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1366);
+# 560 "upb/pb/compile_decoder_x64.dasc"
       break;
     case X: break;
     }
     //|  jmp    >4
-    dasm_put(Dst, 1189);
-# 568 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1370);
+# 564 "upb/pb/compile_decoder_x64.dasc"
 
     // Fast path decode; for when check_bytes bytes are available.
     //|3:
-    dasm_put(Dst, 1194);
-# 571 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1375);
+# 567 "upb/pb/compile_decoder_x64.dasc"
     switch (op) {
     case OP_PARSE_SFIXED32:
     case OP_PARSE_FIXED32:
       //|  mov    edx, dword [PTR]
-      dasm_put(Dst, 1197);
-# 575 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1378);
+# 571 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_PARSE_SFIXED64:
     case OP_PARSE_FIXED64:
       //|  mov    rdx, qword [PTR]
-      dasm_put(Dst, 1200);
-# 579 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1381);
+# 575 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_PARSE_FLOAT:
       //|  movss  xmm0, dword [PTR]
-      dasm_put(Dst, 1204);
-# 582 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1385);
+# 578 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_PARSE_DOUBLE:
       //|  movsd  xmm0, qword [PTR]
-      dasm_put(Dst, 1210);
-# 585 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1391);
+# 581 "upb/pb/compile_decoder_x64.dasc"
       break;
     default:
       // Inline one byte of varint decoding.
       //|  movzx  edx, byte [PTR]
       //|  test   dl, dl
       //|  js     <2   // Fallback to slow path for >1 byte varint.
-      dasm_put(Dst, 1216);
-# 591 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1397);
+# 587 "upb/pb/compile_decoder_x64.dasc"
       break;
     }
 
     // Second-stage decode; used for both fast and slow paths
     // (only needed for a few types).
     //|4:
-    dasm_put(Dst, 1226);
-# 597 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1407);
+# 593 "upb/pb/compile_decoder_x64.dasc"
     switch (op) {
     case OP_PARSE_SINT32:
       // 32-bit zig-zag decode.
@@ -952,8 +879,8 @@ static void jitprimitive(jitcompiler *jc, opcode op,
       //|  and    eax, 1
       //|  neg    eax
       //|  xor    edx, eax
-      dasm_put(Dst, 1229);
-# 605 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1410);
+# 601 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_PARSE_SINT64:
       // 64-bit zig-zag decode.
@@ -962,14 +889,14 @@ static void jitprimitive(jitcompiler *jc, opcode op,
       //|  and    rax, 1
       //|  neg    rax
       //|  xor    rdx, rax
-      dasm_put(Dst, 1243);
-# 613 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1424);
+# 609 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_PARSE_BOOL:
       //|  test   rdx, rdx
       //|  setne  dl
-      dasm_put(Dst, 1262);
-# 617 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1443);
+# 613 "upb/pb/compile_decoder_x64.dasc"
       break;
     default: break;
     }
@@ -982,30 +909,30 @@ static void jitprimitive(jitcompiler *jc, opcode op,
         case UPB_TYPE_INT64:
         case UPB_TYPE_UINT64:
           //|  mov   [CLOSURE + data->offset], rdx
-          dasm_put(Dst, 1269, data->offset);
-# 629 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1450, data->offset);
+# 625 "upb/pb/compile_decoder_x64.dasc"
           break;
         case UPB_TYPE_INT32:
         case UPB_TYPE_UINT32:
         case UPB_TYPE_ENUM:
           //|  mov   [CLOSURE + data->offset], edx
-          dasm_put(Dst, 1274, data->offset);
-# 634 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1455, data->offset);
+# 630 "upb/pb/compile_decoder_x64.dasc"
           break;
         case UPB_TYPE_DOUBLE:
           //|  movsd  qword [CLOSURE + data->offset], XMMARG1
-          dasm_put(Dst, 1279, data->offset);
-# 637 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1460, data->offset);
+# 633 "upb/pb/compile_decoder_x64.dasc"
           break;
         case UPB_TYPE_FLOAT:
           //|  movss  dword [CLOSURE + data->offset], XMMARG1
-          dasm_put(Dst, 1287, data->offset);
-# 640 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1468, data->offset);
+# 636 "upb/pb/compile_decoder_x64.dasc"
           break;
         case UPB_TYPE_BOOL:
           //|  mov   [CLOSURE + data->offset], dl
-          dasm_put(Dst, 1295, data->offset);
-# 643 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1476, data->offset);
+# 639 "upb/pb/compile_decoder_x64.dasc"
           break;
         case UPB_TYPE_STRING:
         case UPB_TYPE_BYTES:
@@ -1014,98 +941,92 @@ static void jitprimitive(jitcompiler *jc, opcode op,
       }
       //|  sethas CLOSURE, data->hasbit
        if (data->hasbit >= 0) {
-      dasm_put(Dst, 1300, ((uint32_t)data->hasbit / 8), (1 << ((uint32_t)data->hasbit % 8)));
+      dasm_put(Dst, 1481, ((uint32_t)data->hasbit / 8), (1 << ((uint32_t)data->hasbit % 8)));
        }
-# 650 "upb/pb/compile_decoder_x64.dasc"
+# 646 "upb/pb/compile_decoder_x64.dasc"
     } else if (handler) {
       //|  mov    ARG1_64, CLOSURE
       //|  load_handler_data h, sel
-      dasm_put(Dst, 1306);
+      dasm_put(Dst, 1487);
        {
        uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, sel);
        if (v > 0xffffffff) {
-      dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+      dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
        } else if (v) {
-      dasm_put(Dst, 391, v);
+      dasm_put(Dst, 451, v);
        } else {
-      dasm_put(Dst, 394);
+      dasm_put(Dst, 454);
        }
        }
-# 653 "upb/pb/compile_decoder_x64.dasc"
+# 649 "upb/pb/compile_decoder_x64.dasc"
       //|  callp  handler
-       {
-       //int64_t ofs = (int64_t)handler - (int64_t)upb_status_init;
-       //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-      dasm_put(Dst, 30, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32));
-       //} else {
-       //}
-       }
-# 654 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1492, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32), 0xfffffffffffffff0UL);
+# 650 "upb/pb/compile_decoder_x64.dasc"
       if (!alwaysok(h, sel)) {
         //|  test   al, al
         //|  jnz    >5
         //|  call   ->suspend
         //|  jmp    <1
         //|5:
-        dasm_put(Dst, 1311);
-# 660 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1514);
+# 656 "upb/pb/compile_decoder_x64.dasc"
       }
     }
 
     // We do this last so that the checkpoint is not advanced past the user's
     // data until the callback has returned success.
     //|  add    PTR, fastbytes
-    dasm_put(Dst, 1327, fastbytes);
-# 666 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1530, fastbytes);
+# 662 "upb/pb/compile_decoder_x64.dasc"
   } else {
     // No handler registered for this value, just skip it.
     //|  chkneob  fastbytes, >3
      if (fastbytes == 1) {
-    dasm_put(Dst, 1146);
+    dasm_put(Dst, 1327);
      } else {
-    dasm_put(Dst, 1154, fastbytes);
+    dasm_put(Dst, 1335, fastbytes);
      }
-# 669 "upb/pb/compile_decoder_x64.dasc"
+# 665 "upb/pb/compile_decoder_x64.dasc"
     //|2:
-    dasm_put(Dst, 1170);
-# 670 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1351);
+# 666 "upb/pb/compile_decoder_x64.dasc"
     switch (type) {
     case V32:
       //|  call   ->skipv32_fallback
-      dasm_put(Dst, 1332);
-# 673 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1535);
+# 669 "upb/pb/compile_decoder_x64.dasc"
       break;
     case V64:
       //|  call   ->skipv64_fallback
-      dasm_put(Dst, 1336);
-# 676 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1539);
+# 672 "upb/pb/compile_decoder_x64.dasc"
       break;
     case F32:
       //|  call   ->skipf32_fallback
-      dasm_put(Dst, 1340);
-# 679 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1543);
+# 675 "upb/pb/compile_decoder_x64.dasc"
       break;
     case F64:
       //|  call   ->skipf64_fallback
-      dasm_put(Dst, 1344);
-# 682 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1547);
+# 678 "upb/pb/compile_decoder_x64.dasc"
       break;
     case X: break;
     }
 
     // Fast-path skip.
     //|3:
-    dasm_put(Dst, 1194);
-# 688 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1375);
+# 684 "upb/pb/compile_decoder_x64.dasc"
     if (type == V32 || type == V64) {
       //|  test   byte [PTR], 0x80
       //|  jnz    <2
-      dasm_put(Dst, 1348);
-# 691 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1551);
+# 687 "upb/pb/compile_decoder_x64.dasc"
     }
     //|  add    PTR, fastbytes
-    dasm_put(Dst, 1327, fastbytes);
-# 693 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1530, fastbytes);
+# 689 "upb/pb/compile_decoder_x64.dasc"
   }
 }
 
@@ -1125,18 +1046,18 @@ static void jitdispatch(jitcompiler *jc,
 
   //|=>define_jmptarget(jc, &method->dispatch):
   //|1:
-  dasm_put(Dst, 1357, define_jmptarget(jc, &method->dispatch));
-# 712 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1560, define_jmptarget(jc, &method->dispatch));
+# 708 "upb/pb/compile_decoder_x64.dasc"
   // Decode the field tag.
   //|  mov     aword DECODER->checkpoint, PTR
   //|  chkeob  2, >6
-  dasm_put(Dst, 248, Dt2(->checkpoint));
+  dasm_put(Dst, 308, Dt2(->checkpoint));
    if (2 == 1) {
-  dasm_put(Dst, 1361);
+  dasm_put(Dst, 1564);
    } else {
-  dasm_put(Dst, 1369);
+  dasm_put(Dst, 1572);
    }
-# 715 "upb/pb/compile_decoder_x64.dasc"
+# 711 "upb/pb/compile_decoder_x64.dasc"
   //|  movzx   edx, byte [PTR]
   //|  test    dl, dl
   //|  jns     >7    // Jump if first byte has no continuation bit.
@@ -1160,49 +1081,49 @@ static void jitdispatch(jitcompiler *jc,
   //|  mov     ecx, edx
   //|  shr     edx, 3
   //|  and     cl, 7
-  dasm_put(Dst, 1385, 1);
-# 738 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1588, 1);
+# 734 "upb/pb/compile_decoder_x64.dasc"
 
   // See comment attached to upb_pbdecodermethod.dispatch for layout of the
   // dispatch table.
   //|2:
   //|  cmp     edx, dispatch->array_size
-  dasm_put(Dst, 1450, dispatch->array_size);
-# 743 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1653, dispatch->array_size);
+# 739 "upb/pb/compile_decoder_x64.dasc"
   if (has_hash_entries) {
     //|  jae     >7
-    dasm_put(Dst, 1457);
-# 745 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1660);
+# 741 "upb/pb/compile_decoder_x64.dasc"
   } else {
     //|  jae     >5
-    dasm_put(Dst, 1462);
-# 747 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1665);
+# 743 "upb/pb/compile_decoder_x64.dasc"
   }
   //|  // OPT: Compact the lookup arr into 32-bit entries.
   if ((uintptr_t)dispatch->array > 0x7fffffff) {
     //|  mov64 rax, (uintptr_t)dispatch->array
     //|  mov   rax, qword [rax + rdx * 8]
-    dasm_put(Dst, 1467, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32));
-# 752 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1670, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32));
+# 748 "upb/pb/compile_decoder_x64.dasc"
   } else {
     //|  mov   rax, qword [rdx * 8 + dispatch->array]
-    dasm_put(Dst, 1476, dispatch->array);
-# 754 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1679, dispatch->array);
+# 750 "upb/pb/compile_decoder_x64.dasc"
   }
   //|3:
   //|  // We take advantage of the fact that non-present entries are stored
   //|  // as -1, which will result in wire types that will never match.
   //|  cmp  al, cl
-  dasm_put(Dst, 1482);
-# 759 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1685);
+# 755 "upb/pb/compile_decoder_x64.dasc"
   if (has_multi_wiretype) {
     //|  jne  >6
-    dasm_put(Dst, 1487);
-# 761 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1690);
+# 757 "upb/pb/compile_decoder_x64.dasc"
   } else {
     //|  jne  >5
-    dasm_put(Dst, 1492);
-# 763 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1695);
+# 759 "upb/pb/compile_decoder_x64.dasc"
   }
   //|  shr  rax, 16
   //|
@@ -1227,8 +1148,8 @@ static void jitdispatch(jitcompiler *jc,
   //|  jz   <1
   //|  lea  rax, [>9]  // ENDGROUP; Load address of OP_ENDMSG.
   //|  ret
-  dasm_put(Dst, 1497, define_jmptarget(jc, dispatch->array));
-# 787 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1700, define_jmptarget(jc, dispatch->array));
+# 783 "upb/pb/compile_decoder_x64.dasc"
 
   if (has_multi_wiretype) {
     //|6:
@@ -1238,46 +1159,46 @@ static void jitdispatch(jitcompiler *jc,
     //|  // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER.
     //|  add   rdx, UPB_MAX_FIELDNUMBER
     //|  // This key will never be in the array part, so do a hash lookup.
-    dasm_put(Dst, 1531, UPB_MAX_FIELDNUMBER);
-# 796 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1734, UPB_MAX_FIELDNUMBER);
+# 792 "upb/pb/compile_decoder_x64.dasc"
     assert(has_hash_entries);
     //|  ld64  dispatch
      {
      uintptr_t v = (uintptr_t)dispatch;
      if (v > 0xffffffff) {
-    dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+    dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
      } else if (v) {
-    dasm_put(Dst, 391, v);
+    dasm_put(Dst, 451, v);
      } else {
-    dasm_put(Dst, 394);
+    dasm_put(Dst, 454);
      }
      }
-# 798 "upb/pb/compile_decoder_x64.dasc"
+# 794 "upb/pb/compile_decoder_x64.dasc"
     //|  jmp   ->hashlookup  // Tail call.
-    dasm_put(Dst, 1544);
-# 799 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1747);
+# 795 "upb/pb/compile_decoder_x64.dasc"
   }
 
   if (has_hash_entries) {
     //|7:
     //|  // Hash table lookup.
     //|  ld64   dispatch
-    dasm_put(Dst, 1549);
+    dasm_put(Dst, 1752);
      {
      uintptr_t v = (uintptr_t)dispatch;
      if (v > 0xffffffff) {
-    dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+    dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
      } else if (v) {
-    dasm_put(Dst, 391, v);
+    dasm_put(Dst, 451, v);
      } else {
-    dasm_put(Dst, 394);
+    dasm_put(Dst, 454);
      }
      }
-# 805 "upb/pb/compile_decoder_x64.dasc"
+# 801 "upb/pb/compile_decoder_x64.dasc"
     //|  call   ->hashlookup
     //|  jmp    <3
-    dasm_put(Dst, 1552);
-# 807 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1755);
+# 803 "upb/pb/compile_decoder_x64.dasc"
   }
 }
 
@@ -1299,11 +1220,11 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs,
 
   //|  chkneob n, >1
    if (n == 1) {
-  dasm_put(Dst, 1560);
+  dasm_put(Dst, 1763);
    } else {
-  dasm_put(Dst, 1568, n);
+  dasm_put(Dst, 1771, n);
    }
-# 827 "upb/pb/compile_decoder_x64.dasc"
+# 823 "upb/pb/compile_decoder_x64.dasc"
 
   //|  // OPT: this is way too much fallback code to put here.
   //|  // Reduce and/or move to a separate section to make better icache usage.
@@ -1311,36 +1232,36 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs,
    {
    uintptr_t v = (uintptr_t)tag;
    if (v > 0xffffffff) {
-  dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+  dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
    } else if (v) {
-  dasm_put(Dst, 391, v);
+  dasm_put(Dst, 451, v);
    } else {
-  dasm_put(Dst, 394);
+  dasm_put(Dst, 454);
    }
    }
-# 831 "upb/pb/compile_decoder_x64.dasc"
+# 827 "upb/pb/compile_decoder_x64.dasc"
   //|  call  ->checktag_fallback
   //|  cmp   eax, DECODE_MISMATCH
   //|  je    >3
   //|  cmp   eax, DECODE_EOF
   //|  je     =>jmptarget(jc, delimend)
   //|  jmp   >5
-  dasm_put(Dst, 1584, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend));
-# 837 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1787, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend));
+# 833 "upb/pb/compile_decoder_x64.dasc"
 
   //|1:
-  dasm_put(Dst, 1143);
-# 839 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 112);
+# 835 "upb/pb/compile_decoder_x64.dasc"
   switch (n) {
   case 1:
     //|  cmp  byte [PTR], tag
-    dasm_put(Dst, 1607, tag);
-# 842 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1810, tag);
+# 838 "upb/pb/compile_decoder_x64.dasc"
     break;
   case 2:
     //|  cmp  word [PTR], tag
-    dasm_put(Dst, 1611, tag);
-# 845 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1814, tag);
+# 841 "upb/pb/compile_decoder_x64.dasc"
     break;
   case 3:
     //|   // OPT: Slightly more efficient code, but depends on an extra byte.
@@ -1351,42 +1272,42 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs,
     //|   jne  >2
     //|   cmp  byte [PTR + 2], (tag >> 16)
     //|2:
-    dasm_put(Dst, 1616, (tag & 0xffff), 2, (tag >> 16));
-# 855 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1819, (tag & 0xffff), 2, (tag >> 16));
+# 851 "upb/pb/compile_decoder_x64.dasc"
     break;
   case 4:
     //|   cmp  dword [PTR], tag
-    dasm_put(Dst, 1631, tag);
-# 858 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1834, tag);
+# 854 "upb/pb/compile_decoder_x64.dasc"
     break;
   case 5:
     //|   cmp  dword [PTR], (tag & 0xffffffff)
     //|   jne  >3
     //|   cmp  byte  [PTR + 4], (tag >> 32)
-    dasm_put(Dst, 1635, (tag & 0xffffffff), 4, (tag >> 32));
-# 863 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1838, (tag & 0xffffffff), 4, (tag >> 32));
+# 859 "upb/pb/compile_decoder_x64.dasc"
   }
   //|  je    >4
   //|3:
-  dasm_put(Dst, 1647);
-# 866 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1850);
+# 862 "upb/pb/compile_decoder_x64.dasc"
   if (ofs == 0) {
     //|  call   =>jmptarget(jc, &method->dispatch)
     //|  test   rax, rax
     //|  jz     =>jmptarget(jc, delimend)
     //|  jmp    rax
-    dasm_put(Dst, 1654, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend));
-# 871 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1857, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend));
+# 867 "upb/pb/compile_decoder_x64.dasc"
   } else {
     //|  jmp    =>jmptarget(jc, jc->pc + ofs)
-    dasm_put(Dst, 1666, jmptarget(jc, jc->pc + ofs));
-# 873 "upb/pb/compile_decoder_x64.dasc"
+    dasm_put(Dst, 1869, jmptarget(jc, jc->pc + ofs));
+# 869 "upb/pb/compile_decoder_x64.dasc"
   }
   //|4:
   //|  add    PTR, n
   //|5:
-  dasm_put(Dst, 1670, n);
-# 877 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1873, n);
+# 873 "upb/pb/compile_decoder_x64.dasc"
 }
 
 // Compile the bytecode to x64.
@@ -1409,7 +1330,7 @@ static void jitbytecode(jitcompiler *jc) {
       // TODO: optimize this to only define pclabels that are actually used.
       //|=>define_jmptarget(jc, jc->pc):
       dasm_put(Dst, 0, define_jmptarget(jc, jc->pc));
-# 898 "upb/pb/compile_decoder_x64.dasc"
+# 894 "upb/pb/compile_decoder_x64.dasc"
     }
 
     jc->pc++;
@@ -1422,75 +1343,62 @@ static void jitbytecode(jitcompiler *jc) {
         //|1:
         //|  mov   ARG1_64, CLOSURE
         //|  load_handler_data h, UPB_STARTMSG_SELECTOR
-        dasm_put(Dst, 1679);
+        dasm_put(Dst, 1882);
          {
          uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_STARTMSG_SELECTOR);
          if (v > 0xffffffff) {
-        dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+        dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
          } else if (v) {
-        dasm_put(Dst, 391, v);
+        dasm_put(Dst, 451, v);
          } else {
-        dasm_put(Dst, 394);
+        dasm_put(Dst, 454);
          }
          }
-# 910 "upb/pb/compile_decoder_x64.dasc"
+# 906 "upb/pb/compile_decoder_x64.dasc"
         //|  callp startmsg
-         {
-         //int64_t ofs = (int64_t)startmsg - (int64_t)upb_status_init;
-         //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-        dasm_put(Dst, 30, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32));
-         //} else {
-         //}
-         }
-# 911 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1492, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32), 0xfffffffffffffff0UL);
+# 907 "upb/pb/compile_decoder_x64.dasc"
         if (!alwaysok(h, UPB_STARTMSG_SELECTOR)) {
           //|  test  al, al
           //|  jnz   >2
           //|  call  ->suspend
           //|  jmp   <1
           //|2:
-          dasm_put(Dst, 1686);
-# 917 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1889);
+# 913 "upb/pb/compile_decoder_x64.dasc"
         }
       } else {
         //| nop
-        dasm_put(Dst, 1702);
-# 920 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1905);
+# 916 "upb/pb/compile_decoder_x64.dasc"
       }
       break;
     }
     case OP_ENDMSG: {
       upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR);
       //|9:
-      dasm_put(Dst, 1704);
-# 926 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1907);
+# 922 "upb/pb/compile_decoder_x64.dasc"
       if (endmsg) {
         // bool endmsg(void *closure, const void *hd, upb_status *status)
         //|  mov   ARG1_64, CLOSURE
         //|  load_handler_data h, UPB_ENDMSG_SELECTOR
-        dasm_put(Dst, 1306);
+        dasm_put(Dst, 1487);
          {
          uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_ENDMSG_SELECTOR);
          if (v > 0xffffffff) {
-        dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+        dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
          } else if (v) {
-        dasm_put(Dst, 391, v);
+        dasm_put(Dst, 451, v);
          } else {
-        dasm_put(Dst, 394);
+        dasm_put(Dst, 454);
          }
          }
-# 930 "upb/pb/compile_decoder_x64.dasc"
+# 926 "upb/pb/compile_decoder_x64.dasc"
         //|  mov   ARG3_64, DECODER->status
         //|  callp endmsg
-        dasm_put(Dst, 1707, Dt2(->status));
-         {
-         //int64_t ofs = (int64_t)endmsg - (int64_t)upb_status_init;
-         //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-        dasm_put(Dst, 30, (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32));
-         //} else {
-         //}
-         }
-# 932 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1910, Dt2(->status), (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32), 0xfffffffffffffff0UL);
+# 928 "upb/pb/compile_decoder_x64.dasc"
       }
       break;
     }
@@ -1521,8 +1429,8 @@ static void jitbytecode(jitcompiler *jc) {
       //|=>define_jmptarget(jc, op_pc):
       //|=>define_jmptarget(jc, method):
       //|  sub   rsp, 8
-      dasm_put(Dst, 1712, define_jmptarget(jc, op_pc), define_jmptarget(jc, method));
-# 962 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1936, define_jmptarget(jc, op_pc), define_jmptarget(jc, method));
+# 958 "upb/pb/compile_decoder_x64.dasc"
 
       break;
     }
@@ -1552,50 +1460,44 @@ static void jitbytecode(jitcompiler *jc) {
         //|1:
         //|  mov   ARG1_64, CLOSURE
         //|  load_handler_data h, arg
-        dasm_put(Dst, 1679);
+        dasm_put(Dst, 1882);
          {
          uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg);
          if (v > 0xffffffff) {
-        dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+        dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
          } else if (v) {
-        dasm_put(Dst, 391, v);
+        dasm_put(Dst, 451, v);
          } else {
-        dasm_put(Dst, 394);
+        dasm_put(Dst, 454);
          }
          }
-# 991 "upb/pb/compile_decoder_x64.dasc"
+# 987 "upb/pb/compile_decoder_x64.dasc"
         if (op == OP_STARTSTR) {
           //|  mov    ARG3_64, DELIMEND
           //|  sub    ARG3_64, PTR
-          dasm_put(Dst, 1720);
-# 994 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1944);
+# 990 "upb/pb/compile_decoder_x64.dasc"
         }
         //|  callp start
-         {
-         //int64_t ofs = (int64_t)start - (int64_t)upb_status_init;
-         //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-        dasm_put(Dst, 30, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32));
-         //} else {
-         //}
-         }
-# 996 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1492, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32), 0xfffffffffffffff0UL);
+# 992 "upb/pb/compile_decoder_x64.dasc"
         if (!alwaysok(h, arg)) {
           //|  test  rax, rax
           //|  jnz   >2
           //|  call  ->suspend
           //|  jmp   <1
           //|2:
-          dasm_put(Dst, 1728);
-# 1002 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1952);
+# 998 "upb/pb/compile_decoder_x64.dasc"
         }
         //|  mov   CLOSURE, rax
-        dasm_put(Dst, 1745);
-# 1004 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1969);
+# 1000 "upb/pb/compile_decoder_x64.dasc"
       } else {
         // TODO: nop is only required because of asmlabel().
         //|  nop
-        dasm_put(Dst, 1702);
-# 1007 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1905);
+# 1003 "upb/pb/compile_decoder_x64.dasc"
       }
       break;
     }
@@ -1610,41 +1512,35 @@ static void jitbytecode(jitcompiler *jc) {
         //|1:
         //|  mov   ARG1_64, CLOSURE
         //|  load_handler_data h, arg
-        dasm_put(Dst, 1679);
+        dasm_put(Dst, 1882);
          {
          uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg);
          if (v > 0xffffffff) {
-        dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+        dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
          } else if (v) {
-        dasm_put(Dst, 391, v);
+        dasm_put(Dst, 451, v);
          } else {
-        dasm_put(Dst, 394);
+        dasm_put(Dst, 454);
          }
          }
-# 1021 "upb/pb/compile_decoder_x64.dasc"
+# 1017 "upb/pb/compile_decoder_x64.dasc"
         //|  callp end
-         {
-         //int64_t ofs = (int64_t)end - (int64_t)upb_status_init;
-         //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-        dasm_put(Dst, 30, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32));
-         //} else {
-         //}
-         }
-# 1022 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1492, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32), 0xfffffffffffffff0UL);
+# 1018 "upb/pb/compile_decoder_x64.dasc"
         if (!alwaysok(h, arg)) {
           //|  test  al, al
           //|  jnz   >2
           //|  call  ->suspend
           //|  jmp   <1
           //|2:
-          dasm_put(Dst, 1686);
-# 1028 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 1889);
+# 1024 "upb/pb/compile_decoder_x64.dasc"
         }
       } else {
         // TODO: nop is only required because of asmlabel().
         //|  nop
-        dasm_put(Dst, 1702);
-# 1032 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 1905);
+# 1028 "upb/pb/compile_decoder_x64.dasc"
       }
       break;
     }
@@ -1658,59 +1554,50 @@ static void jitbytecode(jitcompiler *jc) {
       //|  call  ->suspend
       //|  jmp   <1
       //|2:
-      dasm_put(Dst, 1749);
-# 1045 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1973);
+# 1041 "upb/pb/compile_decoder_x64.dasc"
       if (str) {
         // size_t str(void *closure, const void *hd, const char *str, size_t n)
         //|  mov   ARG1_64, CLOSURE
         //|  load_handler_data h, arg
-        dasm_put(Dst, 1306);
+        dasm_put(Dst, 1487);
          {
          uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg);
          if (v > 0xffffffff) {
-        dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32));
+        dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
          } else if (v) {
-        dasm_put(Dst, 391, v);
+        dasm_put(Dst, 451, v);
          } else {
-        dasm_put(Dst, 394);
+        dasm_put(Dst, 454);
          }
          }
-# 1049 "upb/pb/compile_decoder_x64.dasc"
+# 1045 "upb/pb/compile_decoder_x64.dasc"
         //|  mov   ARG3_64, PTR
         //|  mov   ARG4_64, DATAEND
         //|  sub   ARG4_64, PTR
         //|  mov   ARG5_64, qword DECODER->handle
         //|  callp str
-        dasm_put(Dst, 1776, Dt2(->handle));
-         {
-         //int64_t ofs = (int64_t)str - (int64_t)upb_status_init;
-         //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
-        dasm_put(Dst, 30, (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32));
-         //} else {
-         //}
-         }
-# 1054 "upb/pb/compile_decoder_x64.dasc"
         //|  add   PTR, rax
-        dasm_put(Dst, 1790);
-# 1055 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 2000, Dt2(->handle), (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32), 0xfffffffffffffff0UL);
+# 1051 "upb/pb/compile_decoder_x64.dasc"
         if (!alwaysok(h, arg)) {
           //|  cmp   PTR, DATAEND
           //|  je    >3
           //|  call  ->strret_fallback
           //|3:
-          dasm_put(Dst, 1794);
-# 1060 "upb/pb/compile_decoder_x64.dasc"
+          dasm_put(Dst, 2038);
+# 1056 "upb/pb/compile_decoder_x64.dasc"
         }
       } else {
         //|  mov   PTR, DATAEND
-        dasm_put(Dst, 1807);
-# 1063 "upb/pb/compile_decoder_x64.dasc"
+        dasm_put(Dst, 2051);
+# 1059 "upb/pb/compile_decoder_x64.dasc"
       }
       //|  cmp   PTR, DELIMEND
       //|  jne   <1
       //|4:
-      dasm_put(Dst, 1811);
-# 1067 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2055);
+# 1063 "upb/pb/compile_decoder_x64.dasc"
       break;
     }
     case OP_PUSHTAGDELIM:
@@ -1725,19 +1612,19 @@ static void jitbytecode(jitcompiler *jc) {
       //|  cmp   FRAME, DECODER->limit
       //|  je    ->err
       //|  mov   dword FRAME->groupnum, arg
-      dasm_put(Dst, 1822, Dt1(->sink.closure), Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), arg);
-# 1081 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2066, Dt1(->sink.closure), Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), arg);
+# 1077 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_PUSHLENDELIM:
       //|  call  ->pushlendelim
-      dasm_put(Dst, 1852);
-# 1084 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2096);
+# 1080 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_POP:
       //|  sub   FRAME, sizeof(upb_pbdecoder_frame)
       //|  mov   CLOSURE, FRAME->sink.closure
-      dasm_put(Dst, 1856, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure));
-# 1088 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2100, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure));
+# 1084 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_SETDELIM:
       // OPT: experiment with testing vs old offset to optimize away.
@@ -1749,36 +1636,36 @@ static void jitbytecode(jitcompiler *jc) {
       //|  ja    >1   // OPT: try cmov.
       //|  mov   DATAEND, DELIMEND
       //|1:
-      dasm_put(Dst, 1866, Dt2(->end), Dt1(->end_ofs), Dt2(->buf));
-# 1099 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2110, Dt2(->end), Dt1(->end_ofs), Dt2(->buf));
+# 1095 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_SETBIGGROUPNUM:
       //|  mov   dword FRAME->groupnum, *jc->pc++
-      dasm_put(Dst, 1846, Dt1(->groupnum), *jc->pc++);
-# 1102 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2090, Dt1(->groupnum), *jc->pc++);
+# 1098 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_CHECKDELIM:
       //|  cmp  DELIMEND, PTR
       //|  je   =>jmptarget(jc, jc->pc + longofs)
-      dasm_put(Dst, 1896, jmptarget(jc, jc->pc + longofs));
-# 1106 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2140, jmptarget(jc, jc->pc + longofs));
+# 1102 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_CALL:
       //|  call =>jmptarget(jc, jc->pc + longofs)
-      dasm_put(Dst, 1903, jmptarget(jc, jc->pc + longofs));
-# 1109 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2147, jmptarget(jc, jc->pc + longofs));
+# 1105 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_BRANCH:
       //|  jmp  =>jmptarget(jc, jc->pc + longofs);
-      dasm_put(Dst, 1666, jmptarget(jc, jc->pc + longofs));
-# 1112 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 1869, jmptarget(jc, jc->pc + longofs));
+# 1108 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_RET:
       //|9:
       //|  add  rsp, 8
       //|  ret
-      dasm_put(Dst, 1906);
-# 1117 "upb/pb/compile_decoder_x64.dasc"
+      dasm_put(Dst, 2150);
+# 1113 "upb/pb/compile_decoder_x64.dasc"
       break;
     case OP_TAG1:
       jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method);
@@ -1799,6 +1686,6 @@ static void jitbytecode(jitcompiler *jc) {
 
   asmlabel(jc, "eof");
   //|  nop
-  dasm_put(Dst, 1702);
-# 1137 "upb/pb/compile_decoder_x64.dasc"
+  dasm_put(Dst, 1905);
+# 1133 "upb/pb/compile_decoder_x64.dasc"
 }