Sync from Google development.

1 files changed, 42 insertions, 36 deletions

diff --git a/upb/pb/compile_decoder_x64.dasc b/upb/pb/compile_decoder_x64.dasc
index 571aa9b..97fb5ce 100644
--- a/upb/pb/compile_decoder_x64.dasc
+++ b/upb/pb/compile_decoder_x64.dasc
@@ -42,6 +42,9 @@
 |  mov  DECODER->top, FRAME
 |  mov  DECODER->ptr, PTR
 |  mov  DECODER->data_end, DATAEND
+|  // We don't guarantee that delim_end is NULL when out of range like the
+|  // interpreter does.
+|  mov  DECODER->delim_end, DELIMEND
 |  sub  DELIMEND, DECODER->buf
 |  add  DELIMEND, DECODER->bufstart_ofs
 |  mov  FRAME->end_ofs, DELIMEND
@@ -205,6 +208,8 @@ static void emit_static_asm(jitcompiler *jc) {
   |
   |  mov   DECODER, rdi
   |  callp upb_pbdecoder_resume  // Same args as us; reuse regs.
+  |  test  eax, eax
+  |  jns   >1
   |  mov   DECODER->saved_rsp, rsp
   |  mov   rax, rbx
   |  load_regs
@@ -212,12 +217,13 @@ static void emit_static_asm(jitcompiler *jc) {
   |  // Test whether we have a saved stack to resume.
   |  mov   ARG3_64, DECODER->call_len
   |  test  ARG3_64, ARG3_64
-  |  jnz   >1
+  |  jnz   >2
   |
   |  call  rax
   |
   |  mov   rax, DECODER->size_param
   |  mov   qword DECODER->call_len, 0
+  |1:
   |  add   rsp, 8  // Counter previous alignment.
   |  pop   rbx
   |  pop   r12
@@ -227,7 +233,7 @@ static void emit_static_asm(jitcompiler *jc) {
   |  pop   rbp
   |  ret
   |
-  |1:
+  |2:
   |  // Resume decoder.
   |  lea   ARG2_64, DECODER->callstack
   |  sub   rsp, ARG3_64
@@ -293,6 +299,7 @@ static void emit_static_asm(jitcompiler *jc) {
   |  add   DELIMEND, rdx
   |  cmp   FRAME, DECODER->limit
   |  je    >3   // Stack overflow
+  |  mov   dword FRAME->groupnum, 0
   |  test  rcx, rcx
   |  jz    >2
   |  mov   DATAEND, DECODER->end
@@ -850,24 +857,7 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs,
   |5:
 }
 
-// Emit message-specific assembly.  Overall code layout is:
-// +---------------------------------------------------------------------------+
-// | Message A                                                                 |
-// | 1. function prologue (startmsg), jmps to OP_CHECKDELIM_RET before first   |
-// |    OP_TAG* in 4.                                                          |
-// | 2. function epilogue (endmsg), returns from function.                     |
-// | 3. dispatch function (returns fptr to 4)                                  |
-// |  - loops internally to skip unknown fields                                |
-// |  - after each unknown field does OP_CHECKDELIM_RET (returns 2)            |
-// |  - also returns 2 for END_GROUP.
-// | 4. code for each op:                                                      |
-// |  - OP_TAG* on mismatch calls 3 to get addr, then jumps to 4 (or 2 on EOM).|
-// |  - OP_CHECKDELIM_RET jumps to 2                                           |
-// +---------------------------------------------------------------------------+
-// | Message B                                                                 |
-// | 1. ...                                                                    |
-// | ...                                                                       |
-// +---------------------------------------------------------------------------+
+// Compile the bytecode to x64.
 static void jitbytecode(jitcompiler *jc) {
   upb_pbdecodermethod *method = NULL;
   const upb_handlers *h = NULL;
@@ -877,20 +867,21 @@ static void jitbytecode(jitcompiler *jc) {
     uint32_t arg = instr >> 8;
     int32_t longofs = arg;
 
-    if (op != OP_STARTMSG && op != OP_SETDISPATCH) {
+    if (op != OP_SETDISPATCH) {
+      // Skipped for SETDISPATCH because it defines its own asmlabel for the
+      // dispatch code it emits.
       asmlabel(jc, "0x%lx.%s", pcofs(jc), upb_pbdecoder_getopname(op));
+
+      // Skipped for SETDISPATCH because it should point at the function
+      // prologue, not the dispatch function that is emitted first.
+      // TODO: optimize this to only define pclabels that are actually used.
+      |=>define_pclabel(jc, jc->pc):
     }
-    // TODO: optimize this to only define pclabels that are actually used.
-    |=>define_pclabel(jc, jc->pc):
+
     jc->pc++;
 
     switch (op) {
     case OP_STARTMSG: {
-      // This opcode serves as a function prolouge also.
-      const char *msgname = upb_msgdef_fullname(method->schema_);
-      asmlabel(jc, "0x%lx.parse.%s", pcofs(jc), msgname);
-      |=>define_pclabel(jc, method):
-      |  sub   rsp, 8
       upb_func *startmsg = gethandler(h, UPB_STARTMSG_SELECTOR);
       if (startmsg) {
         // bool startmsg(void *closure, const void *hd)
@@ -905,11 +896,12 @@ static void jitbytecode(jitcompiler *jc) {
           |  jmp   <1
           |2:
         }
+      } else {
+        | nop
       }
       break;
     }
     case OP_ENDMSG: {
-      // This opcode serves as a function epiloue also.
       upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR);
       |9:
       if (endmsg) {
@@ -919,11 +911,12 @@ static void jitbytecode(jitcompiler *jc) {
         |  mov   ARG3_64, DECODER->status
         |  callp endmsg
       }
-      |  add  rsp, 8
-      |  ret
       break;
     }
     case OP_SETDISPATCH: {
+      uint32_t *op_pc = jc->pc - 1;
+
+      // Load info for new method.
       upb_inttable *dispatch;
       memcpy(&dispatch, jc->pc, sizeof(void*));
       jc->pc += sizeof(void*) / sizeof(uint32_t);
@@ -936,9 +929,18 @@ static void jitbytecode(jitcompiler *jc) {
       // case instead of parsing it field by field.  We should also do the skip
       // in the containing message's code.
       h = method->dest_handlers_;
-      const char *msgname = upb_msgdef_fullname(method->schema_);
+      const char *msgname = upb_msgdef_fullname(upb_handlers_msgdef(h));
+
+      // Emit dispatch code for new method.
       asmlabel(jc, "0x%lx.dispatch.%s", pcofs(jc), msgname);
       jitdispatch(jc, method);
+
+      // Emit function prologue for new method.
+      asmlabel(jc, "0x%lx.parse.%s", pcofs(jc), msgname);
+      |=>define_pclabel(jc, op_pc):
+      |=>define_pclabel(jc, method):
+      |  sub   rsp, 8
+
       break;
     }
     case OP_PARSE_DOUBLE:
@@ -1056,6 +1058,7 @@ static void jitbytecode(jitcompiler *jc) {
       |  add   FRAME, sizeof(upb_pbdecoder_frame)
       |  cmp   FRAME, DECODER->limit
       |  je    ->err
+      |  mov   dword FRAME->groupnum, arg
       break;
     case OP_PUSHLENDELIM:
       |  call  ->pushlendelim
@@ -1075,9 +1078,6 @@ static void jitbytecode(jitcompiler *jc) {
       |  mov   DATAEND, DELIMEND
       |1:
       break;
-    case OP_SETGROUPNUM:
-      |  mov   dword FRAME->groupnum, arg
-      break;
     case OP_SETBIGGROUPNUM:
       |  mov   dword FRAME->groupnum, *jc->pc++
       break;
@@ -1086,11 +1086,16 @@ static void jitbytecode(jitcompiler *jc) {
       |  je   =>pclabel(jc, jc->pc + longofs)
       break;
     case OP_CALL:
-      |  call =>pclabel(jc, jc->pc + longofs + 3)
+      |  call =>pclabel(jc, jc->pc + longofs)
       break;
     case OP_BRANCH:
       |  jmp  =>pclabel(jc, jc->pc + longofs);
       break;
+    case OP_RET:
+      |9:
+      |  add  rsp, 8
+      |  ret
+      break;
     case OP_TAG1:
       jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method);
       break;
@@ -1107,6 +1112,7 @@ static void jitbytecode(jitcompiler *jc) {
       assert(false);
     }
   }
+
   asmlabel(jc, "eof");
   |  nop
 }