diff options
Diffstat (limited to 'upb/pb/compile_decoder_x64.h')
| -rw-r--r-- | upb/pb/compile_decoder_x64.h | 2188 |
1 files changed, 1097 insertions, 1091 deletions
diff --git a/upb/pb/compile_decoder_x64.h b/upb/pb/compile_decoder_x64.h index 9527361..525f143 100644 --- a/upb/pb/compile_decoder_x64.h +++ b/upb/pb/compile_decoder_x64.h @@ -10,18 +10,18 @@ #endif # 1 "upb/pb/compile_decoder_x64.dasc" -//|// -//|// upb - a minimalist implementation of protocol buffers. -//|// -//|// Copyright (c) 2011-2013 Google Inc. See LICENSE for details. -//|// Author: Josh Haberman <jhaberman@gmail.com> -//|// -//|// JIT compiler for upb_pbdecoder on x86-64. Generates machine code from the -//|// bytecode generated in compile_decoder.c. -//| -//|.arch x64 -//|.actionlist upb_jit_actionlist -static const unsigned char upb_jit_actionlist[2162] = { +/*|// */ +/*|// upb - a minimalist implementation of protocol buffers. */ +/*|// */ +/*|// Copyright (c) 2011-2013 Google Inc. See LICENSE for details. */ +/*|// Author: Josh Haberman <jhaberman@gmail.com> */ +/*|// */ +/*|// JIT compiler for upb_pbdecoder on x86-64. Generates machine code from the */ +/*|// bytecode generated in compile_decoder.c. */ +/*| */ +/*|.arch x64 */ +/*|.actionlist upb_jit_actionlist */ +static const unsigned char upb_jit_actionlist[2420] = { 249,255,248,10,248,1,85,65,87,65,86,65,85,65,84,83,72,137,252,243,73,137, 252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137, 228,65,92,133,192,15,137,244,247,73,137,167,233,72,137,216,77,139,183,233, @@ -45,50 +45,62 @@ static const unsigned char upb_jit_actionlist[2162] = { 244,248,76,57,229,15,135,244,248,255,73,137,252,236,248,2,195,248,3,73,139, 159,233,76,137,252,255,255,72,190,237,237,255,190,237,255,49,252,246,255, 72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65, - 92,232,244,12,252,233,244,1,255,248,16,72,131,252,236,16,72,137,188,253,36, - 233,248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77, - 137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73, - 3,175,233,73,137,174,233,77,137,174,233,252,255,148,253,36,233,77,139,183, - 233,73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175, - 233,73,3,175,233,255,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4, - 36,72,131,196,16,195,248,2,232,244,11,252,233,244,1,255,248,17,76,137,252, - 255,137,214,15,182,209,77,137,183,233,73,137,159,233,77,137,167,233,73,137, - 175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237, - 237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183, - 233,73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175, - 233,73,3,175,233,129,252,248,239,255,15,133,244,247,195,248,1,129,252,248, - 239,15,132,244,247,232,244,11,248,1,49,192,195,255,248,18,248,19,72,191,237, - 237,232,244,16,72,131,252,235,4,73,137,159,233,195,255,248,20,248,21,72,191, - 237,237,232,244,16,72,131,252,235,8,73,137,159,233,195,255,248,22,248,23, + 92,232,244,12,252,233,244,1,255,248,16,76,137,252,255,137,214,15,182,209, + 77,137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233, + 73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228, + 72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233, + 77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,129, + 252,248,239,255,15,133,244,247,195,248,1,129,252,248,239,15,132,244,247,232, + 244,11,248,1,49,192,195,255,248,17,248,18,72,131,252,236,8,248,1,72,199,4, + 36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137,183,233,73,137, + 159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,233,73,137,174, + 233,77,137,174,233,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255, + 208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139,167,233,77,139, + 174,233,73,139,174,233,255,73,43,175,233,73,3,175,233,133,192,15,137,244, + 248,72,139,20,36,252,242,15,16,4,36,72,131,196,8,72,131,252,235,4,73,137, + 159,233,195,248,2,232,244,11,252,233,244,1,255,248,19,248,20,72,131,252,236, + 8,248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137, + 183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175, + 233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228,72,129, + 228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139, + 167,233,77,139,174,233,73,139,174,233,255,73,43,175,233,73,3,175,233,133, + 192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72,131,196,8,72,131,252, + 235,8,73,137,159,233,195,248,2,232,244,11,252,233,244,1,255,248,21,248,22, 255,76,57,227,15,132,244,247,255,76,137,225,72,41,217,72,131,252,249,16,15, 130,244,247,255,252,243,15,111,3,102,15,215,192,252,247,208,15,188,192,60, - 10,15,131,244,24,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73, - 15,71,204,248,2,72,57,200,15,132,244,24,252,246,0,128,15,132,244,249,72,131, - 192,1,252,233,244,2,248,3,72,137,195,195,255,248,25,72,131,252,236,16,248, + 10,15,131,244,23,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73, + 15,71,204,248,2,72,57,200,15,132,244,23,252,246,0,128,15,132,244,249,72,131, + 192,1,252,233,244,2,248,3,72,137,195,195,255,248,24,72,131,252,236,16,248, 1,72,57,252,235,15,133,244,248,72,131,196,16,49,192,195,248,2,76,137,252, 255,72,137,230,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175,233, 73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,237,65, 84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233, 73,139,159,233,77,139,167,233,77,139,174,233,255,73,139,174,233,73,43,175, 233,73,3,175,233,131,252,248,0,15,141,244,249,139,20,36,72,131,196,16,195, - 248,3,232,244,11,252,233,244,1,255,248,14,248,26,255,76,57,227,15,132,244, - 24,255,76,137,225,72,41,217,72,131,252,249,10,15,130,244,24,255,72,137,223, + 248,3,232,244,11,252,233,244,1,255,248,14,248,25,255,76,57,227,15,132,244, + 23,255,76,137,225,72,41,217,72,131,252,249,10,15,130,244,23,255,72,137,223, 72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65, - 92,72,133,192,15,132,244,24,72,137,195,72,131,252,235,1,73,137,159,233,195, - 255,248,24,72,191,237,237,232,244,16,72,131,252,235,1,73,137,159,233,195, - 255,248,27,72,131,252,236,8,72,137,52,36,248,1,76,137,252,255,77,137,183, + 92,72,133,192,15,132,244,23,72,137,195,72,131,252,235,1,73,137,159,233,195, + 255,248,23,72,131,252,236,8,248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137, + 230,73,137,159,233,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175, + 233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,237, + 65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233, + 73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233, + 255,73,3,175,233,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72, + 131,196,8,72,131,252,235,1,73,137,159,233,195,248,2,232,244,11,252,233,244, + 1,255,248,26,72,131,252,236,8,72,137,52,36,248,1,76,137,252,255,77,137,183, 233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,233, 73,137,174,233,77,137,174,233,73,137,159,233,72,184,237,237,65,84,73,137, 228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159, 233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,255,73,3,175, 233,131,252,248,0,15,141,244,248,72,131,196,8,195,248,2,232,244,11,72,139, - 52,36,72,57,252,235,15,133,244,1,184,237,72,131,196,8,195,255,248,28,81,82, + 52,36,72,57,252,235,15,133,244,1,184,237,72,131,196,8,195,255,248,27,81,82, 72,131,252,236,16,72,137,252,247,72,137,214,72,137,226,72,184,237,237,65, 84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,72,131,196,16,90, 89,132,192,15,132,244,248,72,139,68,36,224,195,248,2,72,49,192,72,252,247, 208,195,255,76,57,227,15,133,244,249,255,76,137,225,72,41,217,72,129,252, - 249,239,15,131,244,249,255,248,2,255,232,244,14,255,232,244,26,255,232,244, - 19,255,232,244,21,255,252,233,244,250,255,248,3,255,139,19,255,72,139,19, + 249,239,15,131,244,249,255,248,2,255,232,244,14,255,232,244,25,255,232,244, + 18,255,232,244,20,255,252,233,244,250,255,248,3,255,139,19,255,72,139,19, 255,252,243,15,16,3,255,252,242,15,16,3,255,15,182,19,132,210,15,136,244, 2,255,248,4,255,137,208,209,252,234,131,224,1,252,247,216,49,194,255,72,137, 208,72,209,252,234,72,131,224,1,72,252,247,216,72,49,194,255,72,133,210,15, @@ -96,19 +108,19 @@ static const unsigned char upb_jit_actionlist[2162] = { 255,252,243,65,15,17,133,233,255,65,136,149,233,255,65,128,141,233,235,255, 76,137,252,239,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255, 208,76,137,228,65,92,255,132,192,15,133,244,251,232,244,12,252,233,244,1, - 248,5,255,72,129,195,239,255,232,244,22,255,232,244,23,255,232,244,18,255, - 232,244,20,255,252,246,3,128,15,133,244,2,255,249,248,1,255,76,57,227,15, + 248,5,255,72,129,195,239,255,232,244,21,255,232,244,22,255,232,244,17,255, + 232,244,19,255,252,246,3,128,15,133,244,2,255,249,248,1,255,76,57,227,15, 132,244,252,255,76,137,225,72,41,217,72,131,252,249,2,15,130,244,252,255, 15,182,19,132,210,15,137,244,253,15,182,139,233,132,201,15,136,244,252,193, - 225,7,131,226,127,9,202,72,131,195,2,252,233,244,254,248,6,232,244,25,133, + 225,7,131,226,127,9,202,72,131,195,2,252,233,244,254,248,6,232,244,24,133, 192,15,133,244,254,195,248,7,72,131,195,1,248,8,137,209,193,252,234,3,128, 225,7,255,248,2,129,252,250,239,255,15,131,244,253,255,15,131,244,251,255, 72,184,237,237,72,139,4,208,255,72,139,4,213,237,255,248,3,56,200,255,15, 133,244,252,255,15,133,244,251,255,72,193,232,16,72,141,21,244,250,249,248, - 4,72,1,208,195,248,5,232,244,17,133,192,15,132,244,1,72,141,5,244,255,195, - 255,248,6,56,204,15,133,244,5,72,129,194,239,255,252,233,244,28,255,248,7, - 255,232,244,28,252,233,244,3,255,76,57,227,15,133,244,247,255,76,137,225, - 72,41,217,72,129,252,249,239,15,131,244,247,255,232,244,27,129,252,248,239, + 4,72,1,208,195,248,5,232,244,16,133,192,15,132,244,1,72,141,5,244,255,195, + 255,248,6,56,204,15,133,244,5,72,129,194,239,255,252,233,244,27,255,248,7, + 255,232,244,27,252,233,244,3,255,76,57,227,15,133,244,247,255,76,137,225, + 72,41,217,72,129,252,249,239,15,131,244,247,255,232,244,26,129,252,248,239, 15,132,244,249,129,252,248,239,15,132,245,252,233,244,251,255,128,59,235, 255,102,129,59,238,255,102,129,59,238,15,133,244,248,128,187,233,235,248, 2,255,129,59,239,255,129,59,239,15,133,244,249,128,187,233,235,255,15,132, @@ -121,7 +133,7 @@ static const unsigned char upb_jit_actionlist[2162] = { 235,15,132,244,250,248,1,76,57,227,15,133,244,248,232,244,12,252,233,244, 1,248,2,255,72,137,218,76,137,225,72,41,217,77,139,135,233,72,184,237,237, 65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,72,1,195,255, - 76,57,227,15,132,244,249,232,244,29,248,3,255,76,137,227,255,72,57,252,235, + 76,57,227,15,132,244,249,232,244,28,248,3,255,76,137,227,255,72,57,252,235, 15,133,244,1,248,4,255,77,137,174,233,73,199,134,233,0,0,0,0,77,59,183,233, 15,132,244,15,73,129,198,239,65,199,134,233,237,255,232,244,13,255,73,129, 252,238,239,77,139,174,233,255,77,139,167,233,73,3,174,233,73,59,175,233, @@ -130,7 +142,7 @@ static const unsigned char upb_jit_actionlist[2162] = { }; # 12 "upb/pb/compile_decoder_x64.dasc" -//|.globals UPB_JIT_GLOBAL_ +/*|.globals UPB_JIT_GLOBAL_ */ enum { UPB_JIT_GLOBAL_enterjit, UPB_JIT_GLOBAL_exitjit, @@ -138,7 +150,6 @@ enum { UPB_JIT_GLOBAL_pushlendelim, UPB_JIT_GLOBAL_decodev32_fallback, UPB_JIT_GLOBAL_err, - UPB_JIT_GLOBAL_getvalue_slow, UPB_JIT_GLOBAL_parse_unknown, UPB_JIT_GLOBAL_skipf32_fallback, UPB_JIT_GLOBAL_decodef32_fallback, @@ -155,7 +166,7 @@ enum { UPB_JIT_GLOBAL__MAX }; # 13 "upb/pb/compile_decoder_x64.dasc" -//|.globalnames upb_jit_globalnames +/*|.globalnames upb_jit_globalnames */ static const char *const upb_jit_globalnames[] = { "enterjit", "exitjit", @@ -163,7 +174,6 @@ static const char *const upb_jit_globalnames[] = { "pushlendelim", "decodev32_fallback", "err", - "getvalue_slow", "parse_unknown", "skipf32_fallback", "decodef32_fallback", @@ -180,135 +190,135 @@ static const char *const upb_jit_globalnames[] = { (const char *)0 }; # 14 "upb/pb/compile_decoder_x64.dasc" -//| -//|// Calling conventions. Note -- this will need to be changed for -//|// Windows, which uses a different calling convention! -//|.define ARG1_64, rdi -//|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi. -//|.define ARG2_32, esi -//|.define ARG2_64, rsi -//|.define ARG3_8, dl -//|.define ARG3_32, edx -//|.define ARG3_64, rdx -//|.define ARG4_64, rcx -//|.define ARG5_64, r8 -//|.define XMMARG1, xmm0 -//| -//|// Register allocation / type map. -//|// ALL of the code in this file uses these register allocations. -//|// When we "call" within this file, we do not use regular calling -//|// conventions, but of course when calling to user callbacks we must. -//|.define PTR, rbx // DECODER->ptr (unsynced) -//|.define DATAEND, r12 // DECODER->data_end (unsynced) -//|.define CLOSURE, r13 // FRAME->closure (unsynced) -//|.type FRAME, upb_pbdecoder_frame, r14 // DECODER->top (unsynced) +/*| */ +/*|// Calling conventions. Note -- this will need to be changed for */ +/*|// Windows, which uses a different calling convention! */ +/*|.define ARG1_64, rdi */ +/*|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi. */ +/*|.define ARG2_32, esi */ +/*|.define ARG2_64, rsi */ +/*|.define ARG3_8, dl */ +/*|.define ARG3_32, edx */ +/*|.define ARG3_64, rdx */ +/*|.define ARG4_64, rcx */ +/*|.define ARG5_64, r8 */ +/*|.define XMMARG1, xmm0 */ +/*| */ +/*|// Register allocation / type map. */ +/*|// ALL of the code in this file uses these register allocations. */ +/*|// When we "call" within this file, we do not use regular calling */ +/*|// conventions, but of course when calling to user callbacks we must. */ +/*|.define PTR, rbx // DECODER->ptr (unsynced) */ +/*|.define DATAEND, r12 // DECODER->data_end (unsynced) */ +/*|.define CLOSURE, r13 // FRAME->closure (unsynced) */ +/*|.type FRAME, upb_pbdecoder_frame, r14 // DECODER->top (unsynced) */ #define Dt1(_V) (int)(ptrdiff_t)&(((upb_pbdecoder_frame *)0)_V) # 36 "upb/pb/compile_decoder_x64.dasc" -//|.type DECODER, upb_pbdecoder, r15 // DECODER (immutable) +/*|.type DECODER, upb_pbdecoder, r15 // DECODER (immutable) */ #define Dt2(_V) (int)(ptrdiff_t)&(((upb_pbdecoder *)0)_V) # 37 "upb/pb/compile_decoder_x64.dasc" -//|.define DELIMEND, rbp -//| -//| // Spills unsynced registers back to memory. -//|.macro commit_regs -//| mov DECODER->top, FRAME -//| mov DECODER->ptr, PTR -//| mov DECODER->data_end, DATAEND -//| // We don't guarantee that delim_end is NULL when out of range like the -//| // interpreter does. -//| mov DECODER->delim_end, DELIMEND -//| sub DELIMEND, DECODER->buf -//| add DELIMEND, DECODER->bufstart_ofs -//| mov FRAME->end_ofs, DELIMEND -//| mov FRAME->sink.closure, CLOSURE -//|.endmacro -//| -//| // Loads unsynced registers from memory back into registers. -//|.macro load_regs -//| mov FRAME, DECODER->top -//| mov PTR, DECODER->ptr -//| mov DATAEND, DECODER->data_end -//| mov CLOSURE, FRAME->sink.closure -//| mov DELIMEND, FRAME->end_ofs -//| sub DELIMEND, DECODER->bufstart_ofs -//| add DELIMEND, DECODER->buf -//|.endmacro -//| -//| // Calls an external C function at address "addr". -//|.macro callp, addr -//| mov64 rax, (uintptr_t)addr -//| -//| // Stack must be 16-byte aligned (x86-64 ABI requires this). -//| // -//| // OPT: possibly remove this by statically ensuring correct alignment. -//| // -//| // OPT: use "call rel32" where possible. -//| push r12 -//| mov r12, rsp -//| and rsp, 0xfffffffffffffff0UL // Align stack. -//| call rax -//| mov rsp, r12 -//| pop r12 -//|.endmacro -//| -//|.macro ld64, val -//|| { -//|| uintptr_t v = (uintptr_t)val; -//|| if (v > 0xffffffff) { -//| mov64 ARG2_64, v -//|| } else if (v) { -//| mov ARG2_32, v -//|| } else { -//| xor ARG2_32, ARG2_32 -//|| } -//|| } -//|.endmacro -//| -//|.macro load_handler_data, h, arg -//| ld64 upb_handlers_gethandlerdata(h, arg) -//|.endmacro -//| -//|.macro chkeob, bytes, target -//|| if (bytes == 1) { -//| cmp PTR, DATAEND -//| je target -//|| } else { -//| mov rcx, DATAEND -//| sub rcx, PTR -//| cmp rcx, bytes -//| jb target -//|| } -//|.endmacro -//| -//|.macro chkneob, bytes, target -//|| if (bytes == 1) { -//| cmp PTR, DATAEND -//| jne target -//|| } else { -//| mov rcx, DATAEND -//| sub rcx, PTR -//| cmp rcx, bytes -//| jae target -//|| } -//|.endmacro +/*|.define DELIMEND, rbp */ +/*| */ +/*| // Spills unsynced registers back to memory. */ +/*|.macro commit_regs */ +/*| mov DECODER->top, FRAME */ +/*| mov DECODER->ptr, PTR */ +/*| mov DECODER->data_end, DATAEND */ +/*| // We don't guarantee that delim_end is NULL when out of range like the */ +/*| // interpreter does. */ +/*| mov DECODER->delim_end, DELIMEND */ +/*| sub DELIMEND, DECODER->buf */ +/*| add DELIMEND, DECODER->bufstart_ofs */ +/*| mov FRAME->end_ofs, DELIMEND */ +/*| mov FRAME->sink.closure, CLOSURE */ +/*|.endmacro */ +/*| */ +/*| // Loads unsynced registers from memory back into registers. */ +/*|.macro load_regs */ +/*| mov FRAME, DECODER->top */ +/*| mov PTR, DECODER->ptr */ +/*| mov DATAEND, DECODER->data_end */ +/*| mov CLOSURE, FRAME->sink.closure */ +/*| mov DELIMEND, FRAME->end_ofs */ +/*| sub DELIMEND, DECODER->bufstart_ofs */ +/*| add DELIMEND, DECODER->buf */ +/*|.endmacro */ +/*| */ +/*| // Calls an external C function at address "addr". */ +/*|.macro callp, addr */ +/*| mov64 rax, (uintptr_t)addr */ +/*| */ +/*| // Stack must be 16-byte aligned (x86-64 ABI requires this). */ +/*| // */ +/*| // OPT: possibly remove this by statically ensuring correct alignment. */ +/*| // */ +/*| // OPT: use "call rel32" where possible. */ +/*| push r12 */ +/*| mov r12, rsp */ +/*| and rsp, 0xfffffffffffffff0UL // Align stack. */ +/*| call rax */ +/*| mov rsp, r12 */ +/*| pop r12 */ +/*|.endmacro */ +/*| */ +/*|.macro ld64, val */ +/*|| { */ +/*|| uintptr_t v = (uintptr_t)val; */ +/*|| if (v > 0xffffffff) { */ +/*| mov64 ARG2_64, v */ +/*|| } else if (v) { */ +/*| mov ARG2_32, v */ +/*|| } else { */ +/*| xor ARG2_32, ARG2_32 */ +/*|| } */ +/*|| } */ +/*|.endmacro */ +/*| */ +/*|.macro load_handler_data, h, arg */ +/*| ld64 upb_handlers_gethandlerdata(h, arg) */ +/*|.endmacro */ +/*| */ +/*|.macro chkeob, bytes, target */ +/*|| if (bytes == 1) { */ +/*| cmp PTR, DATAEND */ +/*| je target */ +/*|| } else { */ +/*| mov rcx, DATAEND */ +/*| sub rcx, PTR */ +/*| cmp rcx, bytes */ +/*| jb target */ +/*|| } */ +/*|.endmacro */ +/*| */ +/*|.macro chkneob, bytes, target */ +/*|| if (bytes == 1) { */ +/*| cmp PTR, DATAEND */ +/*| jne target */ +/*|| } else { */ +/*| mov rcx, DATAEND */ +/*| sub rcx, PTR */ +/*| cmp rcx, bytes */ +/*| jae target */ +/*|| } */ +/*|.endmacro */ -//|.macro sethas, reg, hasbit -//|| if (hasbit >= 0) { -//| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) -//|| } -//|.endmacro -//| -//| // Decodes 32-bit varint into rdx, inlining 1 byte. -//|.macro dv32 -//| chkeob 1, >7 -//| movzx edx, byte [PTR] -//| test dl, dl -//| jns >8 -//|7: -//| call ->decodev32_fallback -//|8: -//| add PTR, 1 -//|.endmacro +/*|.macro sethas, reg, hasbit */ +/*|| if (hasbit >= 0) { */ +/*| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) */ +/*|| } */ +/*|.endmacro */ +/*| */ +/*| // Decodes 32-bit varint into rdx, inlining 1 byte. */ +/*|.macro dv32 */ +/*| chkeob 1, >7 */ +/*| movzx edx, byte [PTR] */ +/*| test dl, dl */ +/*| jns >8 */ +/*|7: */ +/*| call ->decodev32_fallback */ +/*|8: */ +/*| add PTR, 1 */ +/*|.endmacro */ #define DECODE_EOF -3 @@ -316,13 +326,13 @@ static upb_func *gethandler(const upb_handlers *h, upb_selector_t sel) { return h ? upb_handlers_gethandler(h, sel) : NULL; } -// Defines an "assembly label" for the current code generation offset. -// This label exists *purely* for debugging purposes: it is emitted into -// the .so, and printed as part of JIT debugging output when UPB_JIT_LOAD_SO is -// defined. -// -// We would define this in the .c file except that it conditionally defines a -// pclabel. +/* Defines an "assembly label" for the current code generation offset. + * This label exists *purely* for debugging purposes: it is emitted into + * the .so, and printed as part of JIT debugging output when UPB_JIT_LOAD_SO is + * defined. + * + * We would define this in the .c file except that it conditionally defines a + * pclabel. */ static void asmlabel(jitcompiler *jc, const char *fmt, ...) { #ifndef NDEBUG int ofs = jc->dynasm->section->ofs; @@ -340,180 +350,182 @@ static void asmlabel(jitcompiler *jc, const char *fmt, ...) { va_end(args); int pclabel = alloc_pclabel(jc); - // Normally we would prefer to allocate this inline with the codegen, - // ie. - // |=>asmlabel(...) - // But since we do this conditionally, only when UPB_JIT_LOAD_SO is defined, - // we do it here instead. - //|=>pclabel: + /* Normally we would prefer to allocate this inline with the codegen, + * ie. + * |=>asmlabel(...) + * But since we do this conditionally, only when UPB_JIT_LOAD_SO is defined, + * we do it here instead. */ + /*|=>pclabel: */ dasm_put(Dst, 0, pclabel); # 176 "upb/pb/compile_decoder_x64.dasc" upb_inttable_insert(&jc->asmlabels, pclabel, upb_value_ptr(str)); #endif } -// Should only be called when the associated handler is known to exist. +/* Should only be called when the associated handler is known to exist. */ static bool alwaysok(const upb_handlers *h, upb_selector_t sel) { upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; bool ok = upb_handlers_getattr(h, sel, &attr); + bool ret; + UPB_ASSERT_VAR(ok, ok); - bool ret = upb_handlerattr_alwaysok(&attr); + ret = upb_handlerattr_alwaysok(&attr); upb_handlerattr_uninit(&attr); return ret; } -// Emit static assembly routines; code that does not vary based on the message -// schema. Since it's not input-dependent, we only need one single copy of it. -// For the moment we generate a single copy per generated handlers. Eventually -// we should generate this code at compile time and link it into the binary so -// we have one copy total. To do that we'll want to be sure that it is within -// 2GB of our JIT code, so that branches between the two are near (rel32). -// -// We'd put this assembly in a .s file directly, but DynASM's ability to -// calculate structure offsets automatically is too useful to pass up (it's way -// more convenient to write DECODER->sink than [rbx + 0x96], especially since -// the latter would have to be changed whenever the structure is updated). +/* Emit static assembly routines; code that does not vary based on the message + * schema. Since it's not input-dependent, we only need one single copy of it. + * For the moment we generate a single copy per generated handlers. Eventually + * we should generate this code at compile time and link it into the binary so + * we have one copy total. To do that we'll want to be sure that it is within + * 2GB of our JIT code, so that branches between the two are near (rel32). + * + * We'd put this assembly in a .s file directly, but DynASM's ability to + * calculate structure offsets automatically is too useful to pass up (it's way + * more convenient to write DECODER->sink than [rbx + 0x96], especially since + * the latter would have to be changed whenever the structure is updated). */ static void emit_static_asm(jitcompiler *jc) { - //| // Trampolines for entering/exiting the JIT. These are a bit tricky to - //| // support full resuming; when we suspend we copy the JIT's portion of - //| // the call stack into the upb_pbdecoder and restore it when we resume. + /*| // Trampolines for entering/exiting the JIT. These are a bit tricky to */ + /*| // support full resuming; when we suspend we copy the JIT's portion of */ + /*| // the call stack into the upb_pbdecoder and restore it when we resume. */ asmlabel(jc, "enterjit"); - //|->enterjit: - //|1: - //| push rbp - //| push r15 - //| push r14 - //| push r13 - //| push r12 - //| push rbx - //| - //| mov rbx, ARG2_64 // Preserve JIT method. - //| - //| mov DECODER, rdi - //| callp upb_pbdecoder_resume // Same args as us; reuse regs. - //| test eax, eax - //| jns >1 - //| mov DECODER->saved_rsp, rsp - //| mov rax, rbx - //| load_regs - //| - //| // Test whether we have a saved stack to resume. - //| mov ARG3_64, DECODER->call_len - //| test ARG3_64, ARG3_64 - //| jnz >2 - //| - //| call rax - //| - //| mov rax, DECODER->size_param - //| mov qword DECODER->call_len, 0 - //|1: - //| pop rbx + /*|->enterjit: */ + /*|1: */ + /*| push rbp */ + /*| push r15 */ + /*| push r14 */ + /*| push r13 */ + /*| push r12 */ + /*| push rbx */ + /*| */ + /*| mov rbx, ARG2_64 // Preserve JIT method. */ + /*| */ + /*| mov DECODER, rdi */ + /*| callp upb_pbdecoder_resume // Same args as us; reuse regs. */ + /*| test eax, eax */ + /*| jns >1 */ + /*| mov DECODER->saved_rsp, rsp */ + /*| mov rax, rbx */ + /*| load_regs */ + /*| */ + /*| // Test whether we have a saved stack to resume. */ + /*| mov ARG3_64, DECODER->call_len */ + /*| test ARG3_64, ARG3_64 */ + /*| jnz >2 */ + /*| */ + /*| call rax */ + /*| */ + /*| mov rax, DECODER->size_param */ + /*| mov qword DECODER->call_len, 0 */ + /*|1: */ + /*| pop rbx */ dasm_put(Dst, 2, (unsigned int)((uintptr_t)upb_pbdecoder_resume), (unsigned int)(((uintptr_t)upb_pbdecoder_resume)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->call_len), Dt2(->size_param), Dt2(->call_len)); -# 236 "upb/pb/compile_decoder_x64.dasc" - //| pop r12 - //| pop r13 - //| pop r14 - //| pop r15 - //| pop rbp - //| ret - //| - //|2: - //| // Resume decoder. - //| mov ARG2_64, DECODER->callstack - //| sub rsp, ARG3_64 - //| mov ARG1_64, rsp - //| callp memcpy // Restore stack. - //| ret // Return to resumed function (not ->enterjit caller). - //| - //| // Other code can call this to suspend the JIT. - //| // To the calling code, it will appear that the function returns when - //| // the JIT resumes, and more buffer space will be available. - //| // Args: eax=the value that decode() should return. +# 238 "upb/pb/compile_decoder_x64.dasc" + /*| pop r12 */ + /*| pop r13 */ + /*| pop r14 */ + /*| pop r15 */ + /*| pop rbp */ + /*| ret */ + /*| */ + /*|2: */ + /*| // Resume decoder. */ + /*| mov ARG2_64, DECODER->callstack */ + /*| sub rsp, ARG3_64 */ + /*| mov ARG1_64, rsp */ + /*| callp memcpy // Restore stack. */ + /*| ret // Return to resumed function (not ->enterjit caller). */ + /*| */ + /*| // Other code can call this to suspend the JIT. */ + /*| // To the calling code, it will appear that the function returns when */ + /*| // the JIT resumes, and more buffer space will be available. */ + /*| // Args: eax=the value that decode() should return. */ dasm_put(Dst, 115, Dt2(->callstack), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL); -# 255 "upb/pb/compile_decoder_x64.dasc" +# 257 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "exitjit"); - //|->exitjit: - //| // Save the stack into DECODER->callstack. - //| mov ARG1_64, DECODER->callstack - //| mov ARG2_64, rsp - //| mov ARG3_64, DECODER->saved_rsp - //| sub ARG3_64, rsp - //| mov DECODER->call_len, ARG3_64 // Preserve len for next resume. - //| mov ebx, eax // Preserve return value across memcpy. - //| callp memcpy // Copy stack into decoder. - //| mov eax, ebx // This will be our return value. - //| - //| // Must NOT do this before the memcpy(), otherwise memcpy() will - //| // clobber the stack we are trying to save! - //| mov rsp, DECODER->saved_rsp - //| pop rbx - //| pop r12 - //| pop r13 - //| pop r14 - //| pop r15 - //| pop rbp - //| ret - //| - //| // Like suspend() in the C decoder, except that the function appears - //| // (from the caller's perspective) not to return until the decoder is - //| // resumed. + /*|->exitjit: */ + /*| // Save the stack into DECODER->callstack. */ + /*| mov ARG1_64, DECODER->callstack */ + /*| mov ARG2_64, rsp */ + /*| mov ARG3_64, DECODER->saved_rsp */ + /*| sub ARG3_64, rsp */ + /*| mov DECODER->call_len, ARG3_64 // Preserve len for next resume. */ + /*| mov ebx, eax // Preserve return value across memcpy. */ + /*| callp memcpy // Copy stack into decoder. */ + /*| mov eax, ebx // This will be our return value. */ + /*| */ + /*| // Must NOT do this before the memcpy(), otherwise memcpy() will */ + /*| // clobber the stack we are trying to save! */ + /*| mov rsp, DECODER->saved_rsp */ + /*| pop rbx */ + /*| pop r12 */ + /*| pop r13 */ + /*| pop r14 */ + /*| pop r15 */ + /*| pop rbp */ + /*| ret */ + /*| */ + /*| // Like suspend() in the C decoder, except that the function appears */ + /*| // (from the caller's perspective) not to return until the decoder is */ + /*| // resumed. */ dasm_put(Dst, 161, Dt2(->callstack), Dt2(->saved_rsp), Dt2(->call_len), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp)); -# 281 "upb/pb/compile_decoder_x64.dasc" +# 283 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "suspend"); - //|->suspend: - //| cmp DECODER->ptr, PTR - //| je >1 - //| mov DECODER->checkpoint, PTR - //|1: - //| commit_regs - //| mov rdi, DECODER - //| callp upb_pbdecoder_suspend - //| jmp ->exitjit - //| + /*|->suspend: */ + /*| cmp DECODER->ptr, PTR */ + /*| je >1 */ + /*| mov DECODER->checkpoint, PTR */ + /*|1: */ + /*| commit_regs */ + /*| mov rdi, DECODER */ + /*| callp upb_pbdecoder_suspend */ + /*| jmp ->exitjit */ + /*| */ dasm_put(Dst, 222, Dt2(->ptr), Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_suspend), (unsigned int)(((uintptr_t)upb_pbdecoder_suspend)>>32), 0xfffffffffffffff0UL); -# 292 "upb/pb/compile_decoder_x64.dasc" +# 294 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "pushlendelim"); - //|->pushlendelim: - //|1: - //| mov FRAME->sink.closure, CLOSURE - //| mov DECODER->checkpoint, PTR - //| dv32 + /*|->pushlendelim: */ + /*|1: */ + /*| mov FRAME->sink.closure, CLOSURE */ + /*| mov DECODER->checkpoint, PTR */ + /*| dv32 */ dasm_put(Dst, 300, Dt1(->sink.closure), Dt2(->checkpoint)); if (1 == 1) { dasm_put(Dst, 313); } else { dasm_put(Dst, 321); } -# 298 "upb/pb/compile_decoder_x64.dasc" - //| mov rcx, DELIMEND - //| sub rcx, PTR - //| sub rcx, rdx - //| jb ->err // Len is greater than enclosing message. - //| mov FRAME->end_ofs, rcx - //| cmp FRAME, DECODER->limit - //| je >3 // Stack overflow - //| add FRAME, sizeof(upb_pbdecoder_frame) - //| mov DELIMEND, PTR - //| add DELIMEND, rdx - //| mov dword FRAME->groupnum, 0 - //| test rcx, rcx - //| jz >2 - //| mov DATAEND, DECODER->end - //| cmp PTR, DELIMEND - //| ja >2 - //| cmp DELIMEND, DATAEND - //| ja >2 - //| mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND +# 300 "upb/pb/compile_decoder_x64.dasc" + /*| mov rcx, DELIMEND */ + /*| sub rcx, PTR */ + /*| sub rcx, rdx */ + /*| jb ->err // Len is greater than enclosing message. */ + /*| mov FRAME->end_ofs, rcx */ + /*| cmp FRAME, DECODER->limit */ + /*| je >3 // Stack overflow */ + /*| add FRAME, sizeof(upb_pbdecoder_frame) */ + /*| mov DELIMEND, PTR */ + /*| add DELIMEND, rdx */ + /*| mov dword FRAME->groupnum, 0 */ + /*| test rcx, rcx */ + /*| jz >2 */ + /*| mov DATAEND, DECODER->end */ + /*| cmp PTR, DELIMEND */ + /*| ja >2 */ + /*| cmp DELIMEND, DATAEND */ + /*| ja >2 */ + /*| mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND */ dasm_put(Dst, 337, Dt1(->end_ofs), Dt2(->limit), sizeof(upb_pbdecoder_frame), Dt1(->groupnum), Dt2(->end)); -# 317 "upb/pb/compile_decoder_x64.dasc" - //|2: - //| ret - //|3: - //| // Error -- call seterr. - //| mov PTR, DECODER->checkpoint // Rollback to before the delim len. - //| // Prepare seterr args. - //| mov ARG1_64, DECODER - //| ld64 kPbDecoderStackOverflow +# 319 "upb/pb/compile_decoder_x64.dasc" + /*|2: */ + /*| ret */ + /*|3: */ + /*| // Error -- call seterr. */ + /*| mov PTR, DECODER->checkpoint // Rollback to before the delim len. */ + /*| // Prepare seterr args. */ + /*| mov ARG1_64, DECODER */ + /*| ld64 kPbDecoderStackOverflow */ dasm_put(Dst, 428, Dt2(->checkpoint)); { uintptr_t v = (uintptr_t)kPbDecoderStackOverflow; @@ -525,258 +537,248 @@ static void emit_static_asm(jitcompiler *jc) { dasm_put(Dst, 454); } } -# 325 "upb/pb/compile_decoder_x64.dasc" - //| callp upb_pbdecoder_seterr - //| call ->suspend - //| jmp <1 - //| - //| // For getting a value that spans a buffer seam. Falls back to C. - //| // Args: rdi=C decoding function (prototype: int f(upb_pbdecoder*, void*)) +# 327 "upb/pb/compile_decoder_x64.dasc" + /*| callp upb_pbdecoder_seterr */ + /*| call ->suspend */ + /*| jmp <1 */ + /*| */ + /*| // For getting a value that spans a buffer seam. Falls back to C. */ + /*|.macro getvalue_slow, func, bytes */ + /*| sub rsp, 8 // Need stack space for func to write value to. */ + /*|1: */ + /*| mov qword [rsp], 0 // For parsing routines that only parse 32 bits. */ + /*| mov ARG1_64, DECODER */ + /*| mov ARG2_64, rsp */ + /*| mov DECODER->checkpoint, PTR */ + /*| commit_regs */ + /*| callp func */ + /*| load_regs */ + /*| test eax, eax */ + /*| jns >2 */ + /*| // Success; return parsed data (in rdx AND xmm0). */ + /*| mov rdx, [rsp] */ + /*| movsd xmm0, qword [rsp] */ + /*| add rsp, 8 */ + /*| sub PTR, bytes // Bias our buffer pointer to rejoin the fast-path. */ + /*| mov DECODER->ptr, PTR */ + /*| ret */ + /*|2: */ + /*| call ->exitjit // Return eax from decode function. */ + /*| jmp <1 */ + /*|.endmacro */ + /*| */ dasm_put(Dst, 458, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32), 0xfffffffffffffff0UL); -# 331 "upb/pb/compile_decoder_x64.dasc" - asmlabel(jc, "getvalue_slow"); - //|->getvalue_slow: - //| sub rsp, 16 // Stack is [8-byte value, 8-byte func pointer] - //| mov [rsp + 8], rdi // Need to preserve fptr across suspends. - //|1: - //| mov qword [rsp], 0 // For parsing routines that only parse 32 bits. - //| mov ARG1_64, DECODER - //| mov ARG2_64, rsp - //| mov DECODER->checkpoint, PTR - //| commit_regs - //| call aword [rsp + 8] - //| load_regs - //| test eax, eax - dasm_put(Dst, 487, 8, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), 8, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); -# 344 "upb/pb/compile_decoder_x64.dasc" - //| jns >2 - //| // Success; return parsed data (in rdx AND xmm0). - //| mov rdx, [rsp] - //| movsd xmm0, qword [rsp] - //| add rsp, 16 - //| ret - //|2: - //| call ->exitjit // Return eax from decode function. - //| jmp <1 - //| - dasm_put(Dst, 588); -# 354 "upb/pb/compile_decoder_x64.dasc" +# 356 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "parse_unknown"); - //| // Args: edx=fieldnum, cl=wire type - //|->parse_unknown: - //| // OPT: handle directly instead of kicking to C. - //| // Check for ENDGROUP. - //| mov ARG1_64, DECODER - //| mov ARG2_32, edx - //| movzx ARG3_32, cl - //| commit_regs - //| callp upb_pbdecoder_skipunknown - //| load_regs - //| cmp eax, DECODE_ENDGROUP - //| jne >1 - dasm_put(Dst, 619, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP); -# 367 "upb/pb/compile_decoder_x64.dasc" - //| ret // Return eax=DECODE_ENDGROUP, not zero - //|1: - //| cmp eax, DECODE_OK - //| je >1 - //| call ->exitjit // Return eax from decode function. - //|1: - //| xor eax, eax - //| ret - //| - //| // Fallback functions for parsing single values. These are used when the - //| // buffer doesn't contain enough remaining data for the fast path. Each - //| // primitive type (v32, v64, f32, f64) has two functions: decode & skip. - //| // Decode functions return their value in rsi/esi. - //| // - //| // These functions leave PTR = value_end - fast_path_bytes, so that we can - //| // re-join the fast path which will add fast_path_bytes after the callback - //| // completes. We also set DECODER->ptr to this value which is a signal to - //| // ->suspend that DECODER->checkpoint is up to date. - dasm_put(Dst, 716, DECODE_OK); -# 385 "upb/pb/compile_decoder_x64.dasc" + /*| // Args: edx=fieldnum, cl=wire type */ + /*|->parse_unknown: */ + /*| // OPT: handle directly instead of kicking to C. */ + /*| // Check for ENDGROUP. */ + /*| mov ARG1_64, DECODER */ + /*| mov ARG2_32, edx */ + /*| movzx ARG3_32, cl */ + /*| commit_regs */ + /*| callp upb_pbdecoder_skipunknown */ + /*| load_regs */ + /*| cmp eax, DECODE_ENDGROUP */ + /*| jne >1 */ + dasm_put(Dst, 487, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP); +# 369 "upb/pb/compile_decoder_x64.dasc" + /*| ret // Return eax=DECODE_ENDGROUP, not zero */ + /*|1: */ + /*| cmp eax, DECODE_OK */ + /*| je >1 */ + /*| call ->exitjit // Return eax from decode function. */ + /*|1: */ + /*| xor eax, eax */ + /*| ret */ + /*| */ + /*| // Fallback functions for parsing single values. These are used when the */ + /*| // buffer doesn't contain enough remaining data for the fast path. Each */ + /*| // primitive type (v32, v64, f32, f64) has two functions: decode & skip. */ + /*| // Decode functions return their value in rsi/esi. */ + /*| // */ + /*| // These functions leave PTR = value_end - fast_path_bytes, so that we can */ + /*| // re-join the fast path which will add fast_path_bytes after the callback */ + /*| // completes. We also set DECODER->ptr to this value which is a signal to */ + /*| // ->suspend that DECODER->checkpoint is up to date. */ + dasm_put(Dst, 584, DECODE_OK); +# 387 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "skip_decode_f32_fallback"); - //|->skipf32_fallback: - //|->decodef32_fallback: - //| mov64 rdi, (uintptr_t)upb_pbdecoder_decode_f32 - //| call ->getvalue_slow - //| sub PTR, 4 - //| mov DECODER->ptr, PTR - //| ret - //| - dasm_put(Dst, 740, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), Dt2(->ptr)); -# 394 "upb/pb/compile_decoder_x64.dasc" + /*|->skipf32_fallback: */ + /*|->decodef32_fallback: */ + /*| getvalue_slow upb_pbdecoder_decode_f32, 4 */ + dasm_put(Dst, 608, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs)); +# 391 "upb/pb/compile_decoder_x64.dasc" + /*| */ + dasm_put(Dst, 712, Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->ptr)); +# 392 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "skip_decode_f64_fallback"); - //|->skipf64_fallback: - //|->decodef64_fallback: - //| mov64 rdi, (uintptr_t)upb_pbdecoder_decode_f64 - //| call ->getvalue_slow - //| sub PTR, 8 - //| mov DECODER->ptr, PTR - //| ret - //| - //| // Called for varint >= 1 byte. - dasm_put(Dst, 762, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), Dt2(->ptr)); -# 404 "upb/pb/compile_decoder_x64.dasc" + /*|->skipf64_fallback: */ + /*|->decodef64_fallback: */ + /*| getvalue_slow upb_pbdecoder_decode_f64, 8 */ + dasm_put(Dst, 760, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs)); +# 396 "upb/pb/compile_decoder_x64.dasc" + /*| */ + /*| // Called for varint >= 1 byte. */ + dasm_put(Dst, 864, Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->ptr)); +# 398 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "skip_decode_v32_fallback"); - //|->skipv32_fallback: - //|->skipv64_fallback: - //| chkeob 16, >1 - dasm_put(Dst, 784); + /*|->skipv32_fallback: */ + /*|->skipv64_fallback: */ + /*| chkeob 16, >1 */ + dasm_put(Dst, 912); if (16 == 1) { - dasm_put(Dst, 789); + dasm_put(Dst, 917); } else { - dasm_put(Dst, 797); + dasm_put(Dst, 925); } -# 408 "upb/pb/compile_decoder_x64.dasc" - //| // With at least 16 bytes left, we can do a branch-less SSE version. - //| movdqu xmm0, [PTR] - //| pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0. - //| not eax - //| bsf eax, eax - //| cmp al, 10 - //| jae ->decode_varint_slow // Error (>10 byte varint). - //| add PTR, rax // bsf result is 0-based, so PTR=end-1, as desired. - //| ret - //| - //|1: - //| // With fewer than 16 bytes, we have to read byte by byte. - //| lea rcx, [PTR + 10] - //| mov rax, PTR // Preserve PTR in case of fallback to slow path. - //| cmp rcx, DATAEND - //| cmova rcx, DATAEND // rcx = MIN(DATAEND, PTR + 10) - //|2: - //| cmp rax, rcx - //| je ->decode_varint_slow - //| test byte [rax], 0x80 - //| jz >3 - //| add rax, 1 - //| jmp <2 - //|3: - //| mov PTR, rax // PTR = varint_end - 1, as desired - //| ret - //| - //| // Returns tag in edx - dasm_put(Dst, 813, 10); -# 436 "upb/pb/compile_decoder_x64.dasc" +# 402 "upb/pb/compile_decoder_x64.dasc" + /*| // With at least 16 bytes left, we can do a branch-less SSE version. */ + /*| movdqu xmm0, [PTR] */ + /*| pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0. */ + /*| not eax */ + /*| bsf eax, eax */ + /*| cmp al, 10 */ + /*| jae ->decode_varint_slow // Error (>10 byte varint). */ + /*| add PTR, rax // bsf result is 0-based, so PTR=end-1, as desired. */ + /*| ret */ + /*| */ + /*|1: */ + /*| // With fewer than 16 bytes, we have to read byte by byte. */ + /*| lea rcx, [PTR + 10] */ + /*| mov rax, PTR // Preserve PTR in case of fallback to slow path. */ + /*| cmp rcx, DATAEND */ + /*| cmova rcx, DATAEND // rcx = MIN(DATAEND, PTR + 10) */ + /*|2: */ + /*| cmp rax, rcx */ + /*| je ->decode_varint_slow */ + /*| test byte [rax], 0x80 */ + /*| jz >3 */ + /*| add rax, 1 */ + /*| jmp <2 */ + /*|3: */ + /*| mov PTR, rax // PTR = varint_end - 1, as desired */ + /*| ret */ + /*| */ + /*| // Returns tag in edx */ + dasm_put(Dst, 941, 10); +# 430 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "decode_unknown_tag_fallback"); - //|->decode_unknown_tag_fallback: - //| sub rsp, 16 - //|1: - //| cmp PTR, DELIMEND - //| jne >2 - //| add rsp, 16 - //| xor eax, eax - //| ret - //|2: - //| // OPT: Have a medium-fast path before falling back to _slow. - //| mov ARG1_64, DECODER - //| mov ARG2_64, rsp - //| commit_regs - //| callp upb_pbdecoder_decode_varint_slow - //| load_regs - dasm_put(Dst, 886, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure)); -# 452 "upb/pb/compile_decoder_x64.dasc" - //| cmp eax, 0 - //| jge >3 - //| mov edx, [rsp] // Success; return parsed data. - //| add rsp, 16 - //| ret - //|3: - //| call ->exitjit // Return eax from decode function. - //| jmp <1 - //| - //| // Called for varint >= 1 byte. - dasm_put(Dst, 989, Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); -# 462 "upb/pb/compile_decoder_x64.dasc" + /*|->decode_unknown_tag_fallback: */ + /*| sub rsp, 16 */ + /*|1: */ + /*| cmp PTR, DELIMEND */ + /*| jne >2 */ + /*| add rsp, 16 */ + /*| xor eax, eax */ + /*| ret */ + /*|2: */ + /*| // OPT: Have a medium-fast path before falling back to _slow. */ + /*| mov ARG1_64, DECODER */ + /*| mov ARG2_64, rsp */ + /*| commit_regs */ + /*| callp upb_pbdecoder_decode_varint_slow */ + /*| load_regs */ + dasm_put(Dst, 1014, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure)); +# 446 "upb/pb/compile_decoder_x64.dasc" + /*| cmp eax, 0 */ + /*| jge >3 */ + /*| mov edx, [rsp] // Success; return parsed data. */ + /*| add rsp, 16 */ + /*| ret */ + /*|3: */ + /*| call ->exitjit // Return eax from decode function. */ + /*| jmp <1 */ + /*| */ + /*| // Called for varint >= 1 byte. */ + dasm_put(Dst, 1117, Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); +# 456 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "decode_v32_v64_fallback"); - //|->decodev32_fallback: - //|->decodev64_fallback: - //| chkeob 10, ->decode_varint_slow - dasm_put(Dst, 1027); + /*|->decodev32_fallback: */ + /*|->decodev64_fallback: */ + /*| chkeob 10, ->decode_varint_slow */ + dasm_put(Dst, 1155); if (10 == 1) { - dasm_put(Dst, 1032); + dasm_put(Dst, 1160); } else { - dasm_put(Dst, 1040); + dasm_put(Dst, 1168); } -# 466 "upb/pb/compile_decoder_x64.dasc" - //| // OPT: do something faster than just calling the C version. - //| mov rdi, PTR - //| callp upb_vdecode_fast - //| test rax, rax - //| je ->decode_varint_slow // Unterminated varint. - //| mov PTR, rax - //| sub PTR, 1 - //| mov DECODER->ptr, PTR - //| ret - //| - dasm_put(Dst, 1056, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32), 0xfffffffffffffff0UL, Dt2(->ptr)); -# 476 "upb/pb/compile_decoder_x64.dasc" +# 460 "upb/pb/compile_decoder_x64.dasc" + /*| // OPT: do something faster than just calling the C version. */ + /*| mov rdi, PTR */ + /*| callp upb_vdecode_fast */ + /*| test rax, rax */ + /*| je ->decode_varint_slow // Unterminated varint. */ + /*| mov PTR, rax */ + /*| sub PTR, 1 */ + /*| mov DECODER->ptr, PTR */ + /*| ret */ + /*| */ + dasm_put(Dst, 1184, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32), 0xfffffffffffffff0UL, Dt2(->ptr)); +# 470 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "decode_varint_slow"); - //|->decode_varint_slow: - //| // Slow path: end of buffer or error (varint length >= 10). - //| mov64 rdi, (uintptr_t)upb_pbdecoder_decode_varint_slow - //| call ->getvalue_slow - //| sub PTR, 1 - //| mov DECODER->ptr, PTR - //| ret - //| - //| // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH}) - dasm_put(Dst, 1101, (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), Dt2(->ptr)); -# 486 "upb/pb/compile_decoder_x64.dasc" + /*|->decode_varint_slow: */ + /*| // Slow path: end of buffer or error (varint length >= 10). */ + /*| getvalue_slow upb_pbdecoder_decode_varint_slow, 1 */ + dasm_put(Dst, 1229, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs)); +# 474 "upb/pb/compile_decoder_x64.dasc" + /*| */ + /*| // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH}) */ + dasm_put(Dst, 1335, Dt2(->buf), Dt2(->ptr)); +# 476 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "checktag_fallback"); - //|->checktag_fallback: - //| sub rsp, 8 - //| mov [rsp], rsi // Preserve expected tag. - //|1: - //| mov ARG1_64, DECODER - //| commit_regs - //| mov DECODER->checkpoint, PTR - //| callp upb_pbdecoder_checktag_slow - //| load_regs - dasm_put(Dst, 1121, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint), (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs)); -# 496 "upb/pb/compile_decoder_x64.dasc" - //| cmp eax, 0 - //| jge >2 - //| add rsp, 8 - //| ret - //|2: - //| call ->exitjit - //| mov rsi, [rsp] - //| cmp PTR, DELIMEND - //| jne <1 - //| mov eax, DECODE_EOF - //| add rsp, 8 - //| ret - //| - //| // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found). - //| // Preserves: rcx, rdx - //| // OPT: Could write this in assembly if it's a hotspot. - dasm_put(Dst, 1220, Dt2(->buf), DECODE_EOF); -# 512 "upb/pb/compile_decoder_x64.dasc" + /*|->checktag_fallback: */ + /*| sub rsp, 8 */ + /*| mov [rsp], rsi // Preserve expected tag. */ + /*|1: */ + /*| mov ARG1_64, DECODER */ + /*| commit_regs */ + /*| mov DECODER->checkpoint, PTR */ + /*| callp upb_pbdecoder_checktag_slow */ + /*| load_regs */ + dasm_put(Dst, 1379, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint), (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs)); +# 486 "upb/pb/compile_decoder_x64.dasc" + /*| cmp eax, 0 */ + /*| jge >2 */ + /*| add rsp, 8 */ + /*| ret */ + /*|2: */ + /*| call ->exitjit */ + /*| mov rsi, [rsp] */ + /*| cmp PTR, DELIMEND */ + /*| jne <1 */ + /*| mov eax, DECODE_EOF */ + /*| add rsp, 8 */ + /*| ret */ + /*| */ + /*| // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found). */ + /*| // Preserves: rcx, rdx */ + /*| // OPT: Could write this in assembly if it's a hotspot. */ + dasm_put(Dst, 1478, Dt2(->buf), DECODE_EOF); +# 502 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "hashlookup"); - //|->hashlookup: - //| push rcx - //| push rdx - //| sub rsp, 16 - //| mov rdi, rsi - //| mov rsi, rdx - //| mov rdx, rsp - //| callp upb_inttable_lookup - //| add rsp, 16 - //| pop rdx - //| pop rcx - //| test al, al - //| jz >2 // Unknown field. - //| mov rax, [rsp-32] // Value from table. - //| ret - //|2: - //| xor rax, rax - //| not rax - //| ret - dasm_put(Dst, 1262, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32), 0xfffffffffffffff0UL); -# 532 "upb/pb/compile_decoder_x64.dasc" + /*|->hashlookup: */ + /*| push rcx */ + /*| push rdx */ + /*| sub rsp, 16 */ + /*| mov rdi, rsi */ + /*| mov rsi, rdx */ + /*| mov rdx, rsp */ + /*| callp upb_inttable_lookup */ + /*| add rsp, 16 */ + /*| pop rdx */ + /*| pop rcx */ + /*| test al, al */ + /*| jz >2 // Unknown field. */ + /*| mov rax, [rsp-32] // Value from table. */ + /*| ret */ + /*|2: */ + /*| xor rax, rax */ + /*| not rax */ + /*| ret */ + dasm_put(Dst, 1520, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32), 0xfffffffffffffff0UL); +# 522 "upb/pb/compile_decoder_x64.dasc" } static void jitprimitive(jitcompiler *jc, opcode op, @@ -786,169 +788,170 @@ static void jitprimitive(jitcompiler *jc, opcode op, X, F64, F32, V64, V64, V32, F64, F32, V64, X, X, X, X, V32, V32, F32, F64, V32, V64 }; static char fastpath_bytes[] = { 1, 1, 4, 8 }; - const valtype_t type = types[op]; - const int fastbytes = fastpath_bytes[type]; + const valtype_t vtype = types[op]; + const int fastbytes = fastpath_bytes[vtype]; upb_func *handler = gethandler(h, sel); + upb_fieldtype_t ftype; + const upb_shim_data *data; if (handler) { - //|1: - //| chkneob fastbytes, >3 + /*|1: */ + /*| chkneob fastbytes, >3 */ dasm_put(Dst, 112); if (fastbytes == 1) { - dasm_put(Dst, 1331); + dasm_put(Dst, 1589); } else { - dasm_put(Dst, 1339, fastbytes); + dasm_put(Dst, 1597, fastbytes); } -# 548 "upb/pb/compile_decoder_x64.dasc" - //|2: - dasm_put(Dst, 1355); -# 549 "upb/pb/compile_decoder_x64.dasc" - switch (type) { +# 540 "upb/pb/compile_decoder_x64.dasc" + /*|2: */ + dasm_put(Dst, 1613); +# 541 "upb/pb/compile_decoder_x64.dasc" + switch (vtype) { case V32: - //| call ->decodev32_fallback - dasm_put(Dst, 1358); -# 552 "upb/pb/compile_decoder_x64.dasc" + /*| call ->decodev32_fallback */ + dasm_put(Dst, 1616); +# 544 "upb/pb/compile_decoder_x64.dasc" break; case V64: - //| call ->decodev64_fallback - dasm_put(Dst, 1362); -# 555 "upb/pb/compile_decoder_x64.dasc" + /*| call ->decodev64_fallback */ + dasm_put(Dst, 1620); +# 547 "upb/pb/compile_decoder_x64.dasc" break; case F32: - //| call ->decodef32_fallback - dasm_put(Dst, 1366); -# 558 "upb/pb/compile_decoder_x64.dasc" + /*| call ->decodef32_fallback */ + dasm_put(Dst, 1624); +# 550 "upb/pb/compile_decoder_x64.dasc" break; case F64: - //| call ->decodef64_fallback - dasm_put(Dst, 1370); -# 561 "upb/pb/compile_decoder_x64.dasc" + /*| call ->decodef64_fallback */ + dasm_put(Dst, 1628); +# 553 "upb/pb/compile_decoder_x64.dasc" break; case X: break; } - //| jmp >4 - dasm_put(Dst, 1374); -# 565 "upb/pb/compile_decoder_x64.dasc" + /*| jmp >4 */ + dasm_put(Dst, 1632); +# 557 "upb/pb/compile_decoder_x64.dasc" - // Fast path decode; for when check_bytes bytes are available. - //|3: - dasm_put(Dst, 1379); -# 568 "upb/pb/compile_decoder_x64.dasc" + /* Fast path decode; for when check_bytes bytes are available. */ + /*|3: */ + dasm_put(Dst, 1637); +# 560 "upb/pb/compile_decoder_x64.dasc" switch (op) { case OP_PARSE_SFIXED32: case OP_PARSE_FIXED32: - //| mov edx, dword [PTR] - dasm_put(Dst, 1382); -# 572 "upb/pb/compile_decoder_x64.dasc" + /*| mov edx, dword [PTR] */ + dasm_put(Dst, 1640); +# 564 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_SFIXED64: case OP_PARSE_FIXED64: - //| mov rdx, qword [PTR] - dasm_put(Dst, 1385); -# 576 "upb/pb/compile_decoder_x64.dasc" + /*| mov rdx, qword [PTR] */ + dasm_put(Dst, 1643); +# 568 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_FLOAT: - //| movss xmm0, dword [PTR] - dasm_put(Dst, 1389); -# 579 "upb/pb/compile_decoder_x64.dasc" + /*| movss xmm0, dword [PTR] */ + dasm_put(Dst, 1647); +# 571 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_DOUBLE: - //| movsd xmm0, qword [PTR] - dasm_put(Dst, 1395); -# 582 "upb/pb/compile_decoder_x64.dasc" + /*| movsd xmm0, qword [PTR] */ + dasm_put(Dst, 1653); +# 574 "upb/pb/compile_decoder_x64.dasc" break; default: - // Inline one byte of varint decoding. - //| movzx edx, byte [PTR] - //| test dl, dl - //| js <2 // Fallback to slow path for >1 byte varint. - dasm_put(Dst, 1401); -# 588 "upb/pb/compile_decoder_x64.dasc" + /* Inline one byte of varint decoding. */ + /*| movzx edx, byte [PTR] */ + /*| test dl, dl */ + /*| js <2 // Fallback to slow path for >1 byte varint. */ + dasm_put(Dst, 1659); +# 580 "upb/pb/compile_decoder_x64.dasc" break; } - // Second-stage decode; used for both fast and slow paths - // (only needed for a few types). - //|4: - dasm_put(Dst, 1411); -# 594 "upb/pb/compile_decoder_x64.dasc" + /* Second-stage decode; used for both fast and slow paths */ + /* (only needed for a few types). */ + /*|4: */ + dasm_put(Dst, 1669); +# 586 "upb/pb/compile_decoder_x64.dasc" switch (op) { case OP_PARSE_SINT32: - // 32-bit zig-zag decode. - //| mov eax, edx - //| shr edx, 1 - //| and eax, 1 - //| neg eax - //| xor edx, eax - dasm_put(Dst, 1414); -# 602 "upb/pb/compile_decoder_x64.dasc" + /* 32-bit zig-zag decode. */ + /*| mov eax, edx */ + /*| shr edx, 1 */ + /*| and eax, 1 */ + /*| neg eax */ + /*| xor edx, eax */ + dasm_put(Dst, 1672); +# 594 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_SINT64: - // 64-bit zig-zag decode. - //| mov rax, rdx - //| shr rdx, 1 - //| and rax, 1 - //| neg rax - //| xor rdx, rax - dasm_put(Dst, 1428); -# 610 "upb/pb/compile_decoder_x64.dasc" + /* 64-bit zig-zag decode. */ + /*| mov rax, rdx */ + /*| shr rdx, 1 */ + /*| and rax, 1 */ + /*| neg rax */ + /*| xor rdx, rax */ + dasm_put(Dst, 1686); +# 602 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_BOOL: - //| test rdx, rdx - //| setne dl - dasm_put(Dst, 1447); -# 614 "upb/pb/compile_decoder_x64.dasc" + /*| test rdx, rdx */ + /*| setne dl */ + dasm_put(Dst, 1705); +# 606 "upb/pb/compile_decoder_x64.dasc" break; default: break; } - // Call callback (or specialize if we can). - upb_fieldtype_t type; - const upb_shim_data *data = upb_shim_getdata(h, sel, &type); + /* Call callback (or specialize if we can). */ + data = upb_shim_getdata(h, sel, &ftype); if (data) { - switch (type) { + switch (ftype) { case UPB_TYPE_INT64: case UPB_TYPE_UINT64: - //| mov [CLOSURE + data->offset], rdx - dasm_put(Dst, 1454, data->offset); -# 626 "upb/pb/compile_decoder_x64.dasc" + /*| mov [CLOSURE + data->offset], rdx */ + dasm_put(Dst, 1712, data->offset); +# 617 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_INT32: case UPB_TYPE_UINT32: case UPB_TYPE_ENUM: - //| mov [CLOSURE + data->offset], edx - dasm_put(Dst, 1459, data->offset); -# 631 "upb/pb/compile_decoder_x64.dasc" + /*| mov [CLOSURE + data->offset], edx */ + dasm_put(Dst, 1717, data->offset); +# 622 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_DOUBLE: - //| movsd qword [CLOSURE + data->offset], XMMARG1 - dasm_put(Dst, 1464, data->offset); -# 634 "upb/pb/compile_decoder_x64.dasc" + /*| movsd qword [CLOSURE + data->offset], XMMARG1 */ + dasm_put(Dst, 1722, data->offset); +# 625 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_FLOAT: - //| movss dword [CLOSURE + data->offset], XMMARG1 - dasm_put(Dst, 1472, data->offset); -# 637 "upb/pb/compile_decoder_x64.dasc" + /*| movss dword [CLOSURE + data->offset], XMMARG1 */ + dasm_put(Dst, 1730, data->offset); +# 628 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_BOOL: - //| mov [CLOSURE + data->offset], dl - dasm_put(Dst, 1480, data->offset); -# 640 "upb/pb/compile_decoder_x64.dasc" + /*| mov [CLOSURE + data->offset], dl */ + dasm_put(Dst, 1738, data->offset); +# 631 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_STRING: case UPB_TYPE_BYTES: case UPB_TYPE_MESSAGE: assert(false); break; } - //| sethas CLOSURE, data->hasbit + /*| sethas CLOSURE, data->hasbit */ if (data->hasbit >= 0) { - dasm_put(Dst, 1485, ((uint32_t)data->hasbit / 8), (1 << ((uint32_t)data->hasbit % 8))); + dasm_put(Dst, 1743, ((uint32_t)data->hasbit / 8), (1 << ((uint32_t)data->hasbit % 8))); } -# 647 "upb/pb/compile_decoder_x64.dasc" +# 638 "upb/pb/compile_decoder_x64.dasc" } else if (handler) { - //| mov ARG1_64, CLOSURE - //| load_handler_data h, sel - dasm_put(Dst, 1491); + /*| mov ARG1_64, CLOSURE */ + /*| load_handler_data h, sel */ + dasm_put(Dst, 1749); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, sel); if (v > 0xffffffff) { @@ -959,211 +962,211 @@ static void jitprimitive(jitcompiler *jc, opcode op, dasm_put(Dst, 454); } } -# 650 "upb/pb/compile_decoder_x64.dasc" - //| callp handler - dasm_put(Dst, 1496, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32), 0xfffffffffffffff0UL); -# 651 "upb/pb/compile_decoder_x64.dasc" +# 641 "upb/pb/compile_decoder_x64.dasc" + /*| callp handler */ + dasm_put(Dst, 1754, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32), 0xfffffffffffffff0UL); +# 642 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, sel)) { - //| test al, al - //| jnz >5 - //| call ->suspend - //| jmp <1 - //|5: - dasm_put(Dst, 1518); -# 657 "upb/pb/compile_decoder_x64.dasc" + /*| test al, al */ + /*| jnz >5 */ + /*| call ->suspend */ + /*| jmp <1 */ + /*|5: */ + dasm_put(Dst, 1776); +# 648 "upb/pb/compile_decoder_x64.dasc" } } - // We do this last so that the checkpoint is not advanced past the user's - // data until the callback has returned success. - //| add PTR, fastbytes - dasm_put(Dst, 1534, fastbytes); -# 663 "upb/pb/compile_decoder_x64.dasc" + /* We do this last so that the checkpoint is not advanced past the user's + * data until the callback has returned success. */ + /*| add PTR, fastbytes */ + dasm_put(Dst, 1792, fastbytes); +# 654 "upb/pb/compile_decoder_x64.dasc" } else { - // No handler registered for this value, just skip it. - //| chkneob fastbytes, >3 + /* No handler registered for this value, just skip it. */ + /*| chkneob fastbytes, >3 */ if (fastbytes == 1) { - dasm_put(Dst, 1331); + dasm_put(Dst, 1589); } else { - dasm_put(Dst, 1339, fastbytes); + dasm_put(Dst, 1597, fastbytes); } -# 666 "upb/pb/compile_decoder_x64.dasc" - //|2: - dasm_put(Dst, 1355); -# 667 "upb/pb/compile_decoder_x64.dasc" - switch (type) { +# 657 "upb/pb/compile_decoder_x64.dasc" + /*|2: */ + dasm_put(Dst, 1613); +# 658 "upb/pb/compile_decoder_x64.dasc" + switch (vtype) { case V32: - //| call ->skipv32_fallback - dasm_put(Dst, 1539); -# 670 "upb/pb/compile_decoder_x64.dasc" + /*| call ->skipv32_fallback */ + dasm_put(Dst, 1797); +# 661 "upb/pb/compile_decoder_x64.dasc" break; case V64: - //| call ->skipv64_fallback - dasm_put(Dst, 1543); -# 673 "upb/pb/compile_decoder_x64.dasc" + /*| call ->skipv64_fallback */ + dasm_put(Dst, 1801); +# 664 "upb/pb/compile_decoder_x64.dasc" break; case F32: - //| call ->skipf32_fallback - dasm_put(Dst, 1547); -# 676 "upb/pb/compile_decoder_x64.dasc" + /*| call ->skipf32_fallback */ + dasm_put(Dst, 1805); +# 667 "upb/pb/compile_decoder_x64.dasc" break; case F64: - //| call ->skipf64_fallback - dasm_put(Dst, 1551); -# 679 "upb/pb/compile_decoder_x64.dasc" + /*| call ->skipf64_fallback */ + dasm_put(Dst, 1809); +# 670 "upb/pb/compile_decoder_x64.dasc" break; case X: break; } - // Fast-path skip. - //|3: - dasm_put(Dst, 1379); -# 685 "upb/pb/compile_decoder_x64.dasc" - if (type == V32 || type == V64) { - //| test byte [PTR], 0x80 - //| jnz <2 - dasm_put(Dst, 1555); -# 688 "upb/pb/compile_decoder_x64.dasc" + /* Fast-path skip. */ + /*|3: */ + dasm_put(Dst, 1637); +# 676 "upb/pb/compile_decoder_x64.dasc" + if (vtype == V32 || vtype == V64) { + /*| test byte [PTR], 0x80 */ + /*| jnz <2 */ + dasm_put(Dst, 1813); +# 679 "upb/pb/compile_decoder_x64.dasc" } - //| add PTR, fastbytes - dasm_put(Dst, 1534, fastbytes); -# 690 "upb/pb/compile_decoder_x64.dasc" + /*| add PTR, fastbytes */ + dasm_put(Dst, 1792, fastbytes); +# 681 "upb/pb/compile_decoder_x64.dasc" } } static void jitdispatch(jitcompiler *jc, const upb_pbdecodermethod *method) { - // Lots of room for tweaking/optimization here. + /* Lots of room for tweaking/optimization here. */ const upb_inttable *dispatch = &method->dispatch; bool has_hash_entries = (dispatch->t.count > 0); - // Whether any of the fields for this message can have two wire types which - // are both valid (packed & non-packed). - // - // OPT: populate this more precisely; not all messages with hash entries have - // this characteristic. + /* Whether any of the fields for this message can have two wire types which + * are both valid (packed & non-packed). + * + * OPT: populate this more precisely; not all messages with hash entries have + * this characteristic. */ bool has_multi_wiretype = has_hash_entries; - //|=>define_jmptarget(jc, &method->dispatch): - //|1: - dasm_put(Dst, 1564, define_jmptarget(jc, &method->dispatch)); -# 709 "upb/pb/compile_decoder_x64.dasc" - // Decode the field tag. - //| mov aword DECODER->checkpoint, PTR - //| chkeob 2, >6 + /*|=>define_jmptarget(jc, &method->dispatch): */ + /*|1: */ + dasm_put(Dst, 1822, define_jmptarget(jc, &method->dispatch)); +# 700 "upb/pb/compile_decoder_x64.dasc" + /* Decode the field tag. */ + /*| mov aword DECODER->checkpoint, PTR */ + /*| chkeob 2, >6 */ dasm_put(Dst, 308, Dt2(->checkpoint)); if (2 == 1) { - dasm_put(Dst, 1568); + dasm_put(Dst, 1826); } else { - dasm_put(Dst, 1576); + dasm_put(Dst, 1834); } -# 712 "upb/pb/compile_decoder_x64.dasc" - //| movzx edx, byte [PTR] - //| test dl, dl - //| jns >7 // Jump if first byte has no continuation bit. - //| movzx ecx, byte [PTR + 1] - //| test cl, cl - //| js >6 // Jump if second byte has continuation bit. - //| // Confirmed two-byte varint. - //| shl ecx, 7 - //| and edx, 0x7f - //| or edx, ecx - //| add PTR, 2 - //| jmp >8 - //|6: - //| call ->decode_unknown_tag_fallback - //| test eax, eax // Hit DELIMEND? - //| jnz >8 - //| ret - //|7: - //| add PTR, 1 - //|8: - //| mov ecx, edx - //| shr edx, 3 - //| and cl, 7 - dasm_put(Dst, 1592, 1); -# 735 "upb/pb/compile_decoder_x64.dasc" +# 703 "upb/pb/compile_decoder_x64.dasc" + /*| movzx edx, byte [PTR] */ + /*| test dl, dl */ + /*| jns >7 // Jump if first byte has no continuation bit. */ + /*| movzx ecx, byte [PTR + 1] */ + /*| test cl, cl */ + /*| js >6 // Jump if second byte has continuation bit. */ + /*| // Confirmed two-byte varint. */ + /*| shl ecx, 7 */ + /*| and edx, 0x7f */ + /*| or edx, ecx */ + /*| add PTR, 2 */ + /*| jmp >8 */ + /*|6: */ + /*| call ->decode_unknown_tag_fallback */ + /*| test eax, eax // Hit DELIMEND? */ + /*| jnz >8 */ + /*| ret */ + /*|7: */ + /*| add PTR, 1 */ + /*|8: */ + /*| mov ecx, edx */ + /*| shr edx, 3 */ + /*| and cl, 7 */ + dasm_put(Dst, 1850, 1); +# 726 "upb/pb/compile_decoder_x64.dasc" - // See comment attached to upb_pbdecodermethod.dispatch for layout of the - // dispatch table. - //|2: - //| cmp edx, dispatch->array_size - dasm_put(Dst, 1657, dispatch->array_size); -# 740 "upb/pb/compile_decoder_x64.dasc" + /* See comment attached to upb_pbdecodermethod.dispatch for layout of the + * dispatch table. */ + /*|2: */ + /*| cmp edx, dispatch->array_size */ + dasm_put(Dst, 1915, dispatch->array_size); +# 731 "upb/pb/compile_decoder_x64.dasc" if (has_hash_entries) { - //| jae >7 - dasm_put(Dst, 1664); -# 742 "upb/pb/compile_decoder_x64.dasc" + /*| jae >7 */ + dasm_put(Dst, 1922); +# 733 "upb/pb/compile_decoder_x64.dasc" } else { - //| jae >5 - dasm_put(Dst, 1669); -# 744 "upb/pb/compile_decoder_x64.dasc" + /*| jae >5 */ + dasm_put(Dst, 1927); +# 735 "upb/pb/compile_decoder_x64.dasc" } - //| // OPT: Compact the lookup arr into 32-bit entries. + /*| // OPT: Compact the lookup arr into 32-bit entries. */ if ((uintptr_t)dispatch->array > 0x7fffffff) { - //| mov64 rax, (uintptr_t)dispatch->array - //| mov rax, qword [rax + rdx * 8] - dasm_put(Dst, 1674, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32)); -# 749 "upb/pb/compile_decoder_x64.dasc" + /*| mov64 rax, (uintptr_t)dispatch->array */ + /*| mov rax, qword [rax + rdx * 8] */ + dasm_put(Dst, 1932, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32)); +# 740 "upb/pb/compile_decoder_x64.dasc" } else { - //| mov rax, qword [rdx * 8 + dispatch->array] - dasm_put(Dst, 1683, dispatch->array); -# 751 "upb/pb/compile_decoder_x64.dasc" + /*| mov rax, qword [rdx * 8 + dispatch->array] */ + dasm_put(Dst, 1941, dispatch->array); +# 742 "upb/pb/compile_decoder_x64.dasc" } - //|3: - //| // We take advantage of the fact that non-present entries are stored - //| // as -1, which will result in wire types that will never match. - //| cmp al, cl - dasm_put(Dst, 1689); -# 756 "upb/pb/compile_decoder_x64.dasc" + /*|3: */ + /*| // We take advantage of the fact that non-present entries are stored */ + /*| // as -1, which will result in wire types that will never match. */ + /*| cmp al, cl */ + dasm_put(Dst, 1947); +# 747 "upb/pb/compile_decoder_x64.dasc" if (has_multi_wiretype) { - //| jne >6 - dasm_put(Dst, 1694); -# 758 "upb/pb/compile_decoder_x64.dasc" + /*| jne >6 */ + dasm_put(Dst, 1952); +# 749 "upb/pb/compile_decoder_x64.dasc" } else { - //| jne >5 - dasm_put(Dst, 1699); -# 760 "upb/pb/compile_decoder_x64.dasc" + /*| jne >5 */ + dasm_put(Dst, 1957); +# 751 "upb/pb/compile_decoder_x64.dasc" } - //| shr rax, 16 - //| - //| // Load the machine code address from the table entry. - //| // The table entry is relative to the dispatch->array jmptarget - //| // (patchdispatch() took care of this) which is the same as - //| // local label "4". The "lea" is really just trying to do - //| // lea rax, [>4 + rax] - //| // - //| // But we can't write that directly for some reason, so we use - //| // rdx as a temporary. - //| lea rdx, [>4] - //|=>define_jmptarget(jc, dispatch->array): - //|4: - //| add rax, rdx - //| ret - //| - //|5: - //| // Field isn't in our table. - //| call ->parse_unknown - //| test eax, eax // ENDGROUP? - //| jz <1 - //| lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG. - //| ret - dasm_put(Dst, 1704, define_jmptarget(jc, dispatch->array)); -# 784 "upb/pb/compile_decoder_x64.dasc" + /*| shr rax, 16 */ + /*| */ + /*| // Load the machine code address from the table entry. */ + /*| // The table entry is relative to the dispatch->array jmptarget */ + /*| // (patchdispatch() took care of this) which is the same as */ + /*| // local label "4". The "lea" is really just trying to do */ + /*| // lea rax, [>4 + rax] */ + /*| // */ + /*| // But we can't write that directly for some reason, so we use */ + /*| // rdx as a temporary. */ + /*| lea rdx, [>4] */ + /*|=>define_jmptarget(jc, dispatch->array): */ + /*|4: */ + /*| add rax, rdx */ + /*| ret */ + /*| */ + /*|5: */ + /*| // Field isn't in our table. */ + /*| call ->parse_unknown */ + /*| test eax, eax // ENDGROUP? */ + /*| jz <1 */ + /*| lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG. */ + /*| ret */ + dasm_put(Dst, 1962, define_jmptarget(jc, dispatch->array)); +# 775 "upb/pb/compile_decoder_x64.dasc" if (has_multi_wiretype) { - //|6: - //| // Primary wire type didn't match, check secondary wire type. - //| cmp ah, cl - //| jne <5 - //| // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER. - //| add rdx, UPB_MAX_FIELDNUMBER - //| // This key will never be in the array part, so do a hash lookup. - dasm_put(Dst, 1738, UPB_MAX_FIELDNUMBER); -# 793 "upb/pb/compile_decoder_x64.dasc" + /*|6: */ + /*| // Primary wire type didn't match, check secondary wire type. */ + /*| cmp ah, cl */ + /*| jne <5 */ + /*| // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER. */ + /*| add rdx, UPB_MAX_FIELDNUMBER */ + /*| // This key will never be in the array part, so do a hash lookup. */ + dasm_put(Dst, 1996, UPB_MAX_FIELDNUMBER); +# 784 "upb/pb/compile_decoder_x64.dasc" assert(has_hash_entries); - //| ld64 dispatch + /*| ld64 dispatch */ { uintptr_t v = (uintptr_t)dispatch; if (v > 0xffffffff) { @@ -1174,17 +1177,17 @@ static void jitdispatch(jitcompiler *jc, dasm_put(Dst, 454); } } -# 795 "upb/pb/compile_decoder_x64.dasc" - //| jmp ->hashlookup // Tail call. - dasm_put(Dst, 1751); -# 796 "upb/pb/compile_decoder_x64.dasc" +# 786 "upb/pb/compile_decoder_x64.dasc" + /*| jmp ->hashlookup // Tail call. */ + dasm_put(Dst, 2009); +# 787 "upb/pb/compile_decoder_x64.dasc" } if (has_hash_entries) { - //|7: - //| // Hash table lookup. - //| ld64 dispatch - dasm_put(Dst, 1756); + /*|7: */ + /*| // Hash table lookup. */ + /*| ld64 dispatch */ + dasm_put(Dst, 2014); { uintptr_t v = (uintptr_t)dispatch; if (v > 0xffffffff) { @@ -1195,41 +1198,42 @@ static void jitdispatch(jitcompiler *jc, dasm_put(Dst, 454); } } -# 802 "upb/pb/compile_decoder_x64.dasc" - //| call ->hashlookup - //| jmp <3 - dasm_put(Dst, 1759); -# 804 "upb/pb/compile_decoder_x64.dasc" +# 793 "upb/pb/compile_decoder_x64.dasc" + /*| call ->hashlookup */ + /*| jmp <3 */ + dasm_put(Dst, 2017); +# 795 "upb/pb/compile_decoder_x64.dasc" } } static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, const upb_pbdecodermethod *method) { - // Internally we parse unknown fields; if this runs us into DELIMEND we jump - // to the corresponding DELIMEND target (either msg end or repeated field - // end), which we find from the OP_CHECKDELIM which must have necessarily - // preceded us. + /* Internally we parse unknown fields; if this runs us into DELIMEND we jump + * to the corresponding DELIMEND target (either msg end or repeated field + * end), which we find from the OP_CHECKDELIM which must have necessarily + * preceded us. */ uint32_t last_instruction = *(jc->pc - 2); int last_arg = (int32_t)last_instruction >> 8; - assert((last_instruction & 0xff) == OP_CHECKDELIM); uint32_t *delimend = (jc->pc - 1) + last_arg; const size_t ptr_words = sizeof(void*) / sizeof(uint32_t); + assert((last_instruction & 0xff) == OP_CHECKDELIM); + if (getop(*(jc->pc - 1)) == OP_TAGN) { jc->pc += ptr_words; } - //| chkneob n, >1 + /*| chkneob n, >1 */ if (n == 1) { - dasm_put(Dst, 1767); + dasm_put(Dst, 2025); } else { - dasm_put(Dst, 1775, n); + dasm_put(Dst, 2033, n); } -# 824 "upb/pb/compile_decoder_x64.dasc" +# 816 "upb/pb/compile_decoder_x64.dasc" - //| // OPT: this is way too much fallback code to put here. - //| // Reduce and/or move to a separate section to make better icache usage. - //| ld64 tag + /*| // OPT: this is way too much fallback code to put here. */ + /*| // Reduce and/or move to a separate section to make better icache usage. */ + /*| ld64 tag */ { uintptr_t v = (uintptr_t)tag; if (v > 0xffffffff) { @@ -1240,78 +1244,78 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, dasm_put(Dst, 454); } } -# 828 "upb/pb/compile_decoder_x64.dasc" - //| call ->checktag_fallback - //| cmp eax, DECODE_MISMATCH - //| je >3 - //| cmp eax, DECODE_EOF - //| je =>jmptarget(jc, delimend) - //| jmp >5 - dasm_put(Dst, 1791, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend)); -# 834 "upb/pb/compile_decoder_x64.dasc" +# 820 "upb/pb/compile_decoder_x64.dasc" + /*| call ->checktag_fallback */ + /*| cmp eax, DECODE_MISMATCH */ + /*| je >3 */ + /*| cmp eax, DECODE_EOF */ + /*| je =>jmptarget(jc, delimend) */ + /*| jmp >5 */ + dasm_put(Dst, 2049, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend)); +# 826 "upb/pb/compile_decoder_x64.dasc" - //|1: + /*|1: */ dasm_put(Dst, 112); -# 836 "upb/pb/compile_decoder_x64.dasc" +# 828 "upb/pb/compile_decoder_x64.dasc" switch (n) { case 1: - //| cmp byte [PTR], tag - dasm_put(Dst, 1814, tag); -# 839 "upb/pb/compile_decoder_x64.dasc" + /*| cmp byte [PTR], tag */ + dasm_put(Dst, 2072, tag); +# 831 "upb/pb/compile_decoder_x64.dasc" break; case 2: - //| cmp word [PTR], tag - dasm_put(Dst, 1818, tag); -# 842 "upb/pb/compile_decoder_x64.dasc" + /*| cmp word [PTR], tag */ + dasm_put(Dst, 2076, tag); +# 834 "upb/pb/compile_decoder_x64.dasc" break; case 3: - //| // OPT: Slightly more efficient code, but depends on an extra byte. - //| // mov eax, dword [PTR] - //| // shl eax, 8 - //| // cmp eax, tag << 8 - //| cmp word [PTR], (tag & 0xffff) - //| jne >2 - //| cmp byte [PTR + 2], (tag >> 16) - //|2: - dasm_put(Dst, 1823, (tag & 0xffff), 2, (tag >> 16)); -# 852 "upb/pb/compile_decoder_x64.dasc" + /*| // OPT: Slightly more efficient code, but depends on an extra byte. */ + /*| // mov eax, dword [PTR] */ + /*| // shl eax, 8 */ + /*| // cmp eax, tag << 8 */ + /*| cmp word [PTR], (tag & 0xffff) */ + /*| jne >2 */ + /*| cmp byte [PTR + 2], (tag >> 16) */ + /*|2: */ + dasm_put(Dst, 2081, (tag & 0xffff), 2, (tag >> 16)); +# 844 "upb/pb/compile_decoder_x64.dasc" break; case 4: - //| cmp dword [PTR], tag - dasm_put(Dst, 1838, tag); -# 855 "upb/pb/compile_decoder_x64.dasc" + /*| cmp dword [PTR], tag */ + dasm_put(Dst, 2096, tag); +# 847 "upb/pb/compile_decoder_x64.dasc" break; case 5: - //| cmp dword [PTR], (tag & 0xffffffff) - //| jne >3 - //| cmp byte [PTR + 4], (tag >> 32) - dasm_put(Dst, 1842, (tag & 0xffffffff), 4, (tag >> 32)); -# 860 "upb/pb/compile_decoder_x64.dasc" + /*| cmp dword [PTR], (tag & 0xffffffff) */ + /*| jne >3 */ + /*| cmp byte [PTR + 4], (tag >> 32) */ + dasm_put(Dst, 2100, (tag & 0xffffffff), 4, (tag >> 32)); +# 852 "upb/pb/compile_decoder_x64.dasc" } - //| je >4 - //|3: - dasm_put(Dst, 1854); -# 863 "upb/pb/compile_decoder_x64.dasc" + /*| je >4 */ + /*|3: */ + dasm_put(Dst, 2112); +# 855 "upb/pb/compile_decoder_x64.dasc" if (ofs == 0) { - //| call =>jmptarget(jc, &method->dispatch) - //| test rax, rax - //| jz =>jmptarget(jc, delimend) - //| jmp rax - dasm_put(Dst, 1861, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend)); -# 868 "upb/pb/compile_decoder_x64.dasc" + /*| call =>jmptarget(jc, &method->dispatch) */ + /*| test rax, rax */ + /*| jz =>jmptarget(jc, delimend) */ + /*| jmp rax */ + dasm_put(Dst, 2119, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend)); +# 860 "upb/pb/compile_decoder_x64.dasc" } else { - //| jmp =>jmptarget(jc, jc->pc + ofs) - dasm_put(Dst, 1873, jmptarget(jc, jc->pc + ofs)); -# 870 "upb/pb/compile_decoder_x64.dasc" + /*| jmp =>jmptarget(jc, jc->pc + ofs) */ + dasm_put(Dst, 2131, jmptarget(jc, jc->pc + ofs)); +# 862 "upb/pb/compile_decoder_x64.dasc" } - //|4: - //| add PTR, n - //|5: - dasm_put(Dst, 1877, n); -# 874 "upb/pb/compile_decoder_x64.dasc" + /*|4: */ + /*| add PTR, n */ + /*|5: */ + dasm_put(Dst, 2135, n); +# 866 "upb/pb/compile_decoder_x64.dasc" } -// Compile the bytecode to x64. +/* Compile the bytecode to x64. */ static void jitbytecode(jitcompiler *jc) { upb_pbdecodermethod *method = NULL; const upb_handlers *h = NULL; @@ -1322,16 +1326,16 @@ static void jitbytecode(jitcompiler *jc) { int32_t longofs = arg; if (op != OP_SETDISPATCH) { - // Skipped for SETDISPATCH because it defines its own asmlabel for the - // dispatch code it emits. + /* Skipped for SETDISPATCH because it defines its own asmlabel for the + * dispatch code it emits. */ asmlabel(jc, "0x%lx.%s", pcofs(jc), upb_pbdecoder_getopname(op)); - // Skipped for SETDISPATCH because it should point at the function - // prologue, not the dispatch function that is emitted first. - // TODO: optimize this to only define pclabels that are actually used. - //|=>define_jmptarget(jc, jc->pc): + /* Skipped for SETDISPATCH because it should point at the function + * prologue, not the dispatch function that is emitted first. + * TODO: optimize this to only define pclabels that are actually used. */ + /*|=>define_jmptarget(jc, jc->pc): */ dasm_put(Dst, 0, define_jmptarget(jc, jc->pc)); -# 895 "upb/pb/compile_decoder_x64.dasc" +# 887 "upb/pb/compile_decoder_x64.dasc" } jc->pc++; @@ -1340,11 +1344,11 @@ static void jitbytecode(jitcompiler *jc) { case OP_STARTMSG: { upb_func *startmsg = gethandler(h, UPB_STARTMSG_SELECTOR); if (startmsg) { - // bool startmsg(void *closure, const void *hd) - //|1: - //| mov ARG1_64, CLOSURE - //| load_handler_data h, UPB_STARTMSG_SELECTOR - dasm_put(Dst, 1886); + /* bool startmsg(void *closure, const void *hd) */ + /*|1: */ + /*| mov ARG1_64, CLOSURE */ + /*| load_handler_data h, UPB_STARTMSG_SELECTOR */ + dasm_put(Dst, 2144); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_STARTMSG_SELECTOR); if (v > 0xffffffff) { @@ -1355,36 +1359,36 @@ static void jitbytecode(jitcompiler *jc) { dasm_put(Dst, 454); } } -# 907 "upb/pb/compile_decoder_x64.dasc" - //| callp startmsg - dasm_put(Dst, 1496, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32), 0xfffffffffffffff0UL); -# 908 "upb/pb/compile_decoder_x64.dasc" +# 899 "upb/pb/compile_decoder_x64.dasc" + /*| callp startmsg */ + dasm_put(Dst, 1754, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32), 0xfffffffffffffff0UL); +# 900 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, UPB_STARTMSG_SELECTOR)) { - //| test al, al - //| jnz >2 - //| call ->suspend - //| jmp <1 - //|2: - dasm_put(Dst, 1893); -# 914 "upb/pb/compile_decoder_x64.dasc" + /*| test al, al */ + /*| jnz >2 */ + /*| call ->suspend */ + /*| jmp <1 */ + /*|2: */ + dasm_put(Dst, 2151); +# 906 "upb/pb/compile_decoder_x64.dasc" } } else { - //| nop - dasm_put(Dst, 1909); -# 917 "upb/pb/compile_decoder_x64.dasc" + /*| nop */ + dasm_put(Dst, 2167); +# 909 "upb/pb/compile_decoder_x64.dasc" } break; } case OP_ENDMSG: { upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR); - //|9: - dasm_put(Dst, 1911); -# 923 "upb/pb/compile_decoder_x64.dasc" + /*|9: */ + dasm_put(Dst, 2169); +# 915 "upb/pb/compile_decoder_x64.dasc" if (endmsg) { - // bool endmsg(void *closure, const void *hd, upb_status *status) - //| mov ARG1_64, CLOSURE - //| load_handler_data h, UPB_ENDMSG_SELECTOR - dasm_put(Dst, 1491); + /* bool endmsg(void *closure, const void *hd, upb_status *status) */ + /*| mov ARG1_64, CLOSURE */ + /*| load_handler_data h, UPB_ENDMSG_SELECTOR */ + dasm_put(Dst, 1749); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_ENDMSG_SELECTOR); if (v > 0xffffffff) { @@ -1395,43 +1399,44 @@ static void jitbytecode(jitcompiler *jc) { dasm_put(Dst, 454); } } -# 927 "upb/pb/compile_decoder_x64.dasc" - //| mov ARG3_64, DECODER->status - //| callp endmsg - dasm_put(Dst, 1914, Dt2(->status), (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32), 0xfffffffffffffff0UL); -# 929 "upb/pb/compile_decoder_x64.dasc" +# 919 "upb/pb/compile_decoder_x64.dasc" + /*| mov ARG3_64, DECODER->status */ + /*| callp endmsg */ + dasm_put(Dst, 2172, Dt2(->status), (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32), 0xfffffffffffffff0UL); +# 921 "upb/pb/compile_decoder_x64.dasc" } break; } case OP_SETDISPATCH: { uint32_t *op_pc = jc->pc - 1; - - // Load info for new method. + const char *msgname; upb_inttable *dispatch; + + /* Load info for new method. */ memcpy(&dispatch, jc->pc, sizeof(void*)); jc->pc += sizeof(void*) / sizeof(uint32_t); - // The OP_SETDISPATCH bytecode contains a pointer that is - // &method->dispatch; we want to go backwards and recover method. + /* The OP_SETDISPATCH bytecode contains a pointer that is + * &method->dispatch; we want to go backwards and recover method. */ method = (void*)((char*)dispatch - offsetof(upb_pbdecodermethod, dispatch)); - // May be NULL, in which case no handlers for this message will be found. - // OPT: we should do better by completely skipping the message in this - // case instead of parsing it field by field. We should also do the skip - // in the containing message's code. + /* May be NULL, in which case no handlers for this message will be found. + * OPT: we should do better by completely skipping the message in this + * case instead of parsing it field by field. We should also do the skip + * in the containing message's code. */ h = method->dest_handlers_; - const char *msgname = upb_msgdef_fullname(upb_handlers_msgdef(h)); + msgname = upb_msgdef_fullname(upb_handlers_msgdef(h)); - // Emit dispatch code for new method. + /* Emit dispatch code for new method. */ asmlabel(jc, "0x%lx.dispatch.%s", pcofs(jc), msgname); jitdispatch(jc, method); - // Emit function prologue for new method. + /* Emit function prologue for new method. */ asmlabel(jc, "0x%lx.parse.%s", pcofs(jc), msgname); - //|=>define_jmptarget(jc, op_pc): - //|=>define_jmptarget(jc, method): - //| sub rsp, 8 - dasm_put(Dst, 1940, define_jmptarget(jc, op_pc), define_jmptarget(jc, method)); -# 959 "upb/pb/compile_decoder_x64.dasc" + /*|=>define_jmptarget(jc, op_pc): */ + /*|=>define_jmptarget(jc, method): */ + /*| sub rsp, 8 */ + dasm_put(Dst, 2198, define_jmptarget(jc, op_pc), define_jmptarget(jc, method)); +# 952 "upb/pb/compile_decoder_x64.dasc" break; } @@ -1455,13 +1460,13 @@ static void jitbytecode(jitcompiler *jc) { case OP_STARTSTR: { upb_func *start = gethandler(h, arg); if (start) { - // void *startseq(void *closure, const void *hd) - // void *startsubmsg(void *closure, const void *hd) - // void *startstr(void *closure, const void *hd, size_t size_hint) - //|1: - //| mov ARG1_64, CLOSURE - //| load_handler_data h, arg - dasm_put(Dst, 1886); + /* void *startseq(void *closure, const void *hd) + * void *startsubmsg(void *closure, const void *hd) + * void *startstr(void *closure, const void *hd, size_t size_hint) */ + /*|1: */ + /*| mov ARG1_64, CLOSURE */ + /*| load_handler_data h, arg */ + dasm_put(Dst, 2144); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); if (v > 0xffffffff) { @@ -1472,33 +1477,33 @@ static void jitbytecode(jitcompiler *jc) { dasm_put(Dst, 454); } } -# 988 "upb/pb/compile_decoder_x64.dasc" +# 981 "upb/pb/compile_decoder_x64.dasc" if (op == OP_STARTSTR) { - //| mov ARG3_64, DELIMEND - //| sub ARG3_64, PTR - dasm_put(Dst, 1948); -# 991 "upb/pb/compile_decoder_x64.dasc" + /*| mov ARG3_64, DELIMEND */ + /*| sub ARG3_64, PTR */ + dasm_put(Dst, 2206); +# 984 "upb/pb/compile_decoder_x64.dasc" } - //| callp start - dasm_put(Dst, 1496, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32), 0xfffffffffffffff0UL); -# 993 "upb/pb/compile_decoder_x64.dasc" + /*| callp start */ + dasm_put(Dst, 1754, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32), 0xfffffffffffffff0UL); +# 986 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, arg)) { - //| test rax, rax - //| jnz >2 - //| call ->suspend - //| jmp <1 - //|2: - dasm_put(Dst, 1956); -# 999 "upb/pb/compile_decoder_x64.dasc" + /*| test rax, rax */ + /*| jnz >2 */ + /*| call ->suspend */ + /*| jmp <1 */ + /*|2: */ + dasm_put(Dst, 2214); +# 992 "upb/pb/compile_decoder_x64.dasc" } - //| mov CLOSURE, rax - dasm_put(Dst, 1973); -# 1001 "upb/pb/compile_decoder_x64.dasc" + /*| mov CLOSURE, rax */ + dasm_put(Dst, 2231); +# 994 "upb/pb/compile_decoder_x64.dasc" } else { - // TODO: nop is only required because of asmlabel(). - //| nop - dasm_put(Dst, 1909); -# 1004 "upb/pb/compile_decoder_x64.dasc" + /* TODO: nop is only required because of asmlabel(). */ + /*| nop */ + dasm_put(Dst, 2167); +# 997 "upb/pb/compile_decoder_x64.dasc" } break; } @@ -1507,13 +1512,13 @@ static void jitbytecode(jitcompiler *jc) { case OP_ENDSTR: { upb_func *end = gethandler(h, arg); if (end) { - // bool endseq(void *closure, const void *hd) - // bool endsubmsg(void *closure, const void *hd) - // bool endstr(void *closure, const void *hd) - //|1: - //| mov ARG1_64, CLOSURE - //| load_handler_data h, arg - dasm_put(Dst, 1886); + /* bool endseq(void *closure, const void *hd) + * bool endsubmsg(void *closure, const void *hd) + * bool endstr(void *closure, const void *hd) */ + /*|1: */ + /*| mov ARG1_64, CLOSURE */ + /*| load_handler_data h, arg */ + dasm_put(Dst, 2144); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); if (v > 0xffffffff) { @@ -1524,44 +1529,45 @@ static void jitbytecode(jitcompiler *jc) { dasm_put(Dst, 454); } } -# 1018 "upb/pb/compile_decoder_x64.dasc" - //| callp end - dasm_put(Dst, 1496, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32), 0xfffffffffffffff0UL); -# 1019 "upb/pb/compile_decoder_x64.dasc" +# 1011 "upb/pb/compile_decoder_x64.dasc" + /*| callp end */ + dasm_put(Dst, 1754, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32), 0xfffffffffffffff0UL); +# 1012 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, arg)) { - //| test al, al - //| jnz >2 - //| call ->suspend - //| jmp <1 - //|2: - dasm_put(Dst, 1893); -# 1025 "upb/pb/compile_decoder_x64.dasc" + /*| test al, al */ + /*| jnz >2 */ + /*| call ->suspend */ + /*| jmp <1 */ + /*|2: */ + dasm_put(Dst, 2151); +# 1018 "upb/pb/compile_decoder_x64.dasc" } } else { - // TODO: nop is only required because of asmlabel(). - //| nop - dasm_put(Dst, 1909); -# 1029 "upb/pb/compile_decoder_x64.dasc" + /* TODO: nop is only required because of asmlabel(). */ + /*| nop */ + dasm_put(Dst, 2167); +# 1022 "upb/pb/compile_decoder_x64.dasc" } break; } case OP_STRING: { upb_func *str = gethandler(h, arg); - //| cmp PTR, DELIMEND - //| je >4 - //|1: - //| cmp PTR, DATAEND - //| jne >2 - //| call ->suspend - //| jmp <1 - //|2: - dasm_put(Dst, 1977); -# 1042 "upb/pb/compile_decoder_x64.dasc" + /*| cmp PTR, DELIMEND */ + /*| je >4 */ + /*|1: */ + /*| cmp PTR, DATAEND */ + /*| jne >2 */ + /*| call ->suspend */ + /*| jmp <1 */ + /*|2: */ + dasm_put(Dst, 2235); +# 1035 "upb/pb/compile_decoder_x64.dasc" if (str) { - // size_t str(void *closure, const void *hd, const char *str, size_t n) - //| mov ARG1_64, CLOSURE - //| load_handler_data h, arg - dasm_put(Dst, 1491); + /* size_t str(void *closure, const void *hd, const char *str, + * size_t n) */ + /*| mov ARG1_64, CLOSURE */ + /*| load_handler_data h, arg */ + dasm_put(Dst, 1749); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); if (v > 0xffffffff) { @@ -1572,101 +1578,101 @@ static void jitbytecode(jitcompiler *jc) { dasm_put(Dst, 454); } } +# 1040 "upb/pb/compile_decoder_x64.dasc" + /*| mov ARG3_64, PTR */ + /*| mov ARG4_64, DATAEND */ + /*| sub ARG4_64, PTR */ + /*| mov ARG5_64, qword DECODER->handle */ + /*| callp str */ + /*| add PTR, rax */ + dasm_put(Dst, 2262, Dt2(->handle), (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32), 0xfffffffffffffff0UL); # 1046 "upb/pb/compile_decoder_x64.dasc" - //| mov ARG3_64, PTR - //| mov ARG4_64, DATAEND - //| sub ARG4_64, PTR - //| mov ARG5_64, qword DECODER->handle - //| callp str - //| add PTR, rax - dasm_put(Dst, 2004, Dt2(->handle), (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32), 0xfffffffffffffff0UL); -# 1052 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, arg)) { - //| cmp PTR, DATAEND - //| je >3 - //| call ->strret_fallback - //|3: - dasm_put(Dst, 2042); -# 1057 "upb/pb/compile_decoder_x64.dasc" + /*| cmp PTR, DATAEND */ + /*| je >3 */ + /*| call ->strret_fallback */ + /*|3: */ + dasm_put(Dst, 2300); +# 1051 "upb/pb/compile_decoder_x64.dasc" } } else { - //| mov PTR, DATAEND - dasm_put(Dst, 2055); -# 1060 "upb/pb/compile_decoder_x64.dasc" + /*| mov PTR, DATAEND */ + dasm_put(Dst, 2313); +# 1054 "upb/pb/compile_decoder_x64.dasc" } - //| cmp PTR, DELIMEND - //| jne <1 - //|4: - dasm_put(Dst, 2059); -# 1064 "upb/pb/compile_decoder_x64.dasc" + /*| cmp PTR, DELIMEND */ + /*| jne <1 */ + /*|4: */ + dasm_put(Dst, 2317); +# 1058 "upb/pb/compile_decoder_x64.dasc" break; } case OP_PUSHTAGDELIM: - //| mov FRAME->sink.closure, CLOSURE - //| // This shouldn't need to be read, because tag-delimited fields - //| // shouldn't have an OP_SETDELIM after them. But for the moment - //| // non-packed repeated fields do OP_SETDELIM so they can share more - //| // code with the packed code-path. If this is changed later, this - //| // store can be removed. - //| mov qword FRAME->end_ofs, 0 - //| cmp FRAME, DECODER->limit - //| je ->err - //| add FRAME, sizeof(upb_pbdecoder_frame) - //| mov dword FRAME->groupnum, arg - dasm_put(Dst, 2070, Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->limit), sizeof(upb_pbdecoder_frame), Dt1(->groupnum), arg); -# 1078 "upb/pb/compile_decoder_x64.dasc" + /*| mov FRAME->sink.closure, CLOSURE */ + /*| // This shouldn't need to be read, because tag-delimited fields */ + /*| // shouldn't have an OP_SETDELIM after them. But for the moment */ + /*| // non-packed repeated fields do OP_SETDELIM so they can share more */ + /*| // code with the packed code-path. If this is changed later, this */ + /*| // store can be removed. */ + /*| mov qword FRAME->end_ofs, 0 */ + /*| cmp FRAME, DECODER->limit */ + /*| je ->err */ + /*| add FRAME, sizeof(upb_pbdecoder_frame) */ + /*| mov dword FRAME->groupnum, arg */ + dasm_put(Dst, 2328, Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->limit), sizeof(upb_pbdecoder_frame), Dt1(->groupnum), arg); +# 1072 "upb/pb/compile_decoder_x64.dasc" break; case OP_PUSHLENDELIM: - //| call ->pushlendelim - dasm_put(Dst, 2100); -# 1081 "upb/pb/compile_decoder_x64.dasc" + /*| call ->pushlendelim */ + dasm_put(Dst, 2358); +# 1075 "upb/pb/compile_decoder_x64.dasc" break; case OP_POP: - //| sub FRAME, sizeof(upb_pbdecoder_frame) - //| mov CLOSURE, FRAME->sink.closure - dasm_put(Dst, 2104, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure)); -# 1085 "upb/pb/compile_decoder_x64.dasc" + /*| sub FRAME, sizeof(upb_pbdecoder_frame) */ + /*| mov CLOSURE, FRAME->sink.closure */ + dasm_put(Dst, 2362, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure)); +# 1079 "upb/pb/compile_decoder_x64.dasc" break; case OP_SETDELIM: - // OPT: experiment with testing vs old offset to optimize away. - //| mov DATAEND, DECODER->end - //| add DELIMEND, FRAME->end_ofs - //| cmp DELIMEND, DECODER->buf - //| jb >1 - //| cmp DELIMEND, DATAEND - //| ja >1 // OPT: try cmov. - //| mov DATAEND, DELIMEND - //|1: - dasm_put(Dst, 2114, Dt2(->end), Dt1(->end_ofs), Dt2(->buf)); -# 1096 "upb/pb/compile_decoder_x64.dasc" + /* OPT: experiment with testing vs old offset to optimize away. */ + /*| mov DATAEND, DECODER->end */ + /*| add DELIMEND, FRAME->end_ofs */ + /*| cmp DELIMEND, DECODER->buf */ + /*| jb >1 */ + /*| cmp DELIMEND, DATAEND */ + /*| ja >1 // OPT: try cmov. */ + /*| mov DATAEND, DELIMEND */ + /*|1: */ + dasm_put(Dst, 2372, Dt2(->end), Dt1(->end_ofs), Dt2(->buf)); +# 1090 "upb/pb/compile_decoder_x64.dasc" break; case OP_SETBIGGROUPNUM: - //| mov dword FRAME->groupnum, *jc->pc++ - dasm_put(Dst, 2094, Dt1(->groupnum), *jc->pc++); -# 1099 "upb/pb/compile_decoder_x64.dasc" + /*| mov dword FRAME->groupnum, *jc->pc++ */ + dasm_put(Dst, 2352, Dt1(->groupnum), *jc->pc++); +# 1093 "upb/pb/compile_decoder_x64.dasc" break; case OP_CHECKDELIM: - //| cmp DELIMEND, PTR - //| je =>jmptarget(jc, jc->pc + longofs) - dasm_put(Dst, 2144, jmptarget(jc, jc->pc + longofs)); -# 1103 "upb/pb/compile_decoder_x64.dasc" + /*| cmp DELIMEND, PTR */ + /*| je =>jmptarget(jc, jc->pc + longofs) */ + dasm_put(Dst, 2402, jmptarget(jc, jc->pc + longofs)); +# 1097 "upb/pb/compile_decoder_x64.dasc" break; case OP_CALL: - //| call =>jmptarget(jc, jc->pc + longofs) - dasm_put(Dst, 2151, jmptarget(jc, jc->pc + longofs)); -# 1106 "upb/pb/compile_decoder_x64.dasc" + /*| call =>jmptarget(jc, jc->pc + longofs) */ + dasm_put(Dst, 2409, jmptarget(jc, jc->pc + longofs)); +# 1100 "upb/pb/compile_decoder_x64.dasc" break; case OP_BRANCH: - //| jmp =>jmptarget(jc, jc->pc + longofs); - dasm_put(Dst, 1873, jmptarget(jc, jc->pc + longofs)); -# 1109 "upb/pb/compile_decoder_x64.dasc" + /*| jmp =>jmptarget(jc, jc->pc + longofs); */ + dasm_put(Dst, 2131, jmptarget(jc, jc->pc + longofs)); +# 1103 "upb/pb/compile_decoder_x64.dasc" break; case OP_RET: - //|9: - //| add rsp, 8 - //| ret - dasm_put(Dst, 2154); -# 1114 "upb/pb/compile_decoder_x64.dasc" + /*|9: */ + /*| add rsp, 8 */ + /*| ret */ + dasm_put(Dst, 2412); +# 1108 "upb/pb/compile_decoder_x64.dasc" break; case OP_TAG1: jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method); @@ -1681,9 +1687,9 @@ static void jitbytecode(jitcompiler *jc) { break; } case OP_DISPATCH: - //| call =>jmptarget(jc, &method->dispatch) - dasm_put(Dst, 2151, jmptarget(jc, &method->dispatch)); -# 1129 "upb/pb/compile_decoder_x64.dasc" + /*| call =>jmptarget(jc, &method->dispatch) */ + dasm_put(Dst, 2409, jmptarget(jc, &method->dispatch)); +# 1123 "upb/pb/compile_decoder_x64.dasc" break; case OP_HALT: assert(false); @@ -1691,7 +1697,7 @@ static void jitbytecode(jitcompiler *jc) { } asmlabel(jc, "eof"); - //| nop - dasm_put(Dst, 1909); -# 1137 "upb/pb/compile_decoder_x64.dasc" + /*| nop */ + dasm_put(Dst, 2167); +# 1131 "upb/pb/compile_decoder_x64.dasc" } |