From 39c9a8bd5a278873e8ca00b0f50e2494a60deb26 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Wed, 3 Dec 2014 14:18:28 -0800 Subject: Sync from Google-internal development. Most notably, a JSON parser is now provided. --- upb/pb/compile_decoder_x64.h | 859 +++++++++++++++++++------------------------ 1 file changed, 373 insertions(+), 486 deletions(-) (limited to 'upb/pb') diff --git a/upb/pb/compile_decoder_x64.h b/upb/pb/compile_decoder_x64.h index 9787a57..095d2ef 100644 --- a/upb/pb/compile_decoder_x64.h +++ b/upb/pb/compile_decoder_x64.h @@ -21,19 +21,22 @@ //| //|.arch x64 //|.actionlist upb_jit_actionlist -static const unsigned char upb_jit_actionlist[1914] = { - 249,255,248,10,248,1,85,65,87,65,86,65,85,65,84,83,72,131,252,236,8,72,137, - 252,243,73,137,252,255,255,72,184,237,237,252,255,208,255,133,192,15,137, - 244,247,73,137,167,233,72,137,216,77,139,183,233,73,139,159,233,77,139,167, - 233,77,139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,73,139,151,233, - 72,133,210,15,133,244,248,252,255,208,73,139,135,233,73,199,135,233,0,0,0, - 0,248,1,72,131,196,8,91,65,92,65,93,65,94,65,95,93,195,248,2,73,141,183,233, - 72,41,212,72,137,231,255,195,255,248,11,73,141,191,233,72,137,230,73,139, - 151,233,72,41,226,73,137,151,233,137,195,255,137,216,73,139,167,233,72,131, - 196,8,91,65,92,65,93,65,94,65,95,93,195,255,248,12,73,57,159,233,15,132,244, - 247,73,137,159,233,248,1,77,137,183,233,73,137,159,233,77,137,167,233,73, - 137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,76,137, - 252,255,255,252,233,244,11,255,248,13,248,1,77,137,174,233,73,137,159,233, +static const unsigned char upb_jit_actionlist[2158] = { + 249,255,248,10,248,1,85,65,87,65,86,65,85,65,84,83,72,137,252,243,73,137, + 252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137, + 228,65,92,133,192,15,137,244,247,73,137,167,233,72,137,216,77,139,183,233, + 73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233, + 73,3,175,233,73,139,151,233,72,133,210,15,133,244,248,252,255,208,73,139, + 135,233,73,199,135,233,0,0,0,0,248,1,255,91,65,92,65,93,65,94,65,95,93,195, + 248,2,73,141,183,233,72,41,212,72,137,231,72,184,237,237,65,84,73,137,228, + 72,129,228,239,252,255,208,76,137,228,65,92,195,255,248,11,73,141,191,233, + 72,137,230,73,139,151,233,72,41,226,73,137,151,233,137,195,72,184,237,237, + 65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,137,216,73,139, + 167,233,91,65,92,65,93,65,94,65,95,93,195,255,248,12,73,57,159,233,15,132, + 244,247,73,137,159,233,248,1,77,137,183,233,73,137,159,233,77,137,167,233, + 73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,76, + 137,252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76, + 137,228,65,92,252,233,244,11,255,248,13,248,1,77,137,174,233,73,137,159,233, 255,76,57,227,15,132,244,253,255,76,137,225,72,41,217,72,131,252,249,1,15, 130,244,253,255,15,182,19,132,210,15,137,244,254,248,7,232,244,14,248,8,72, 131,195,1,72,137,252,233,72,41,217,72,41,209,15,130,244,15,73,137,142,233, @@ -41,51 +44,58 @@ static const unsigned char upb_jit_actionlist[1914] = { 233,0,0,0,0,72,133,201,15,132,244,248,77,139,167,233,72,57,252,235,15,135, 244,248,76,57,229,15,135,244,248,255,73,137,252,236,248,2,195,248,3,73,139, 159,233,76,137,252,255,255,72,190,237,237,255,190,237,255,49,252,246,255, - 232,244,12,252,233,244,1,255,248,16,72,131,252,236,16,72,137,188,253,36,233, - 248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137, - 183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175, - 233,73,137,174,233,77,137,174,233,252,255,148,253,36,233,77,139,183,233,73, - 139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,73, - 3,175,233,255,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72,131, - 196,16,195,248,2,232,244,11,252,233,244,1,255,248,17,76,137,252,255,137,214, - 15,182,209,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73, - 43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,255,77,139,183,233, - 73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233, - 73,3,175,233,129,252,248,239,15,133,244,247,195,248,1,129,252,248,239,15, - 132,244,247,232,244,11,248,1,49,192,195,255,248,18,248,19,72,191,237,237, - 232,244,16,72,131,252,235,4,73,137,159,233,195,255,248,20,248,21,72,191,237, - 237,232,244,16,72,131,252,235,8,73,137,159,233,195,255,248,22,248,23,255, - 76,57,227,15,132,244,247,255,76,137,225,72,41,217,72,131,252,249,16,15,130, - 244,247,255,252,243,15,111,3,102,15,215,192,252,247,208,15,188,192,60,10, - 15,131,244,24,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73,15, - 71,204,248,2,72,131,192,1,72,57,200,15,132,244,24,252,246,0,128,15,133,244, - 2,248,3,72,137,195,195,255,248,25,72,131,252,236,16,248,1,72,57,252,235,15, - 133,244,248,72,131,196,16,49,192,195,248,2,76,137,252,255,72,137,230,77,137, - 183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175, - 233,73,137,174,233,77,137,174,233,255,77,139,183,233,73,139,159,233,77,139, - 167,233,77,139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,131,252, - 248,0,15,141,244,249,139,20,36,72,131,196,16,195,248,3,232,244,11,252,233, - 244,1,255,248,14,248,26,255,76,57,227,15,132,244,24,255,76,137,225,72,41, - 217,72,131,252,249,10,15,130,244,24,255,72,137,223,255,72,133,192,15,132, + 72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65, + 92,232,244,12,252,233,244,1,255,248,16,72,131,252,236,16,72,137,188,253,36, + 233,248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77, + 137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73, + 3,175,233,73,137,174,233,77,137,174,233,252,255,148,253,36,233,77,139,183, + 233,73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175, + 233,73,3,175,233,255,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4, + 36,72,131,196,16,195,248,2,232,244,11,252,233,244,1,255,248,17,76,137,252, + 255,137,214,15,182,209,77,137,183,233,73,137,159,233,77,137,167,233,73,137, + 175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237, + 237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183, + 233,73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175, + 233,73,3,175,233,129,252,248,239,255,15,133,244,247,195,248,1,129,252,248, + 239,15,132,244,247,232,244,11,248,1,49,192,195,255,248,18,248,19,72,191,237, + 237,232,244,16,72,131,252,235,4,73,137,159,233,195,255,248,20,248,21,72,191, + 237,237,232,244,16,72,131,252,235,8,73,137,159,233,195,255,248,22,248,23, + 255,76,57,227,15,132,244,247,255,76,137,225,72,41,217,72,131,252,249,16,15, + 130,244,247,255,252,243,15,111,3,102,15,215,192,252,247,208,15,188,192,60, + 10,15,131,244,24,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73, + 15,71,204,248,2,72,131,192,1,72,57,200,15,132,244,24,252,246,0,128,15,133, + 244,2,248,3,72,137,195,195,255,248,25,72,131,252,236,16,248,1,72,57,252,235, + 15,133,244,248,72,131,196,16,49,192,195,248,2,76,137,252,255,72,137,230,77, + 137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73, + 3,175,233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228,72, + 129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77, + 139,167,233,77,139,174,233,255,73,139,174,233,73,43,175,233,73,3,175,233, + 131,252,248,0,15,141,244,249,139,20,36,72,131,196,16,195,248,3,232,244,11, + 252,233,244,1,255,248,14,248,26,255,76,57,227,15,132,244,24,255,76,137,225, + 72,41,217,72,131,252,249,10,15,130,244,24,255,72,137,223,72,184,237,237,65, + 84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,72,133,192,15,132, 244,24,72,137,195,72,131,252,235,1,73,137,159,233,195,255,248,24,72,191,237, 237,232,244,16,72,131,252,235,1,73,137,159,233,195,255,248,27,72,131,252, 236,8,72,137,52,36,248,1,76,137,252,255,77,137,183,233,73,137,159,233,77, 137,167,233,73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137, - 174,233,73,137,159,233,255,77,139,183,233,73,139,159,233,77,139,167,233,77, - 139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,131,252,248,0,15,141, - 244,248,72,131,196,8,195,248,2,232,244,11,72,139,52,36,72,57,252,235,15,133, - 244,1,184,237,72,131,196,8,195,255,248,28,81,82,72,131,252,236,16,72,137, - 252,247,72,137,214,72,137,226,255,72,131,196,16,90,89,132,192,15,132,244, - 248,72,139,68,36,224,195,248,2,72,49,192,72,252,247,208,195,255,248,1,255, - 76,57,227,15,133,244,249,255,76,137,225,72,41,217,72,129,252,249,239,15,131, - 244,249,255,248,2,255,232,244,14,255,232,244,26,255,232,244,19,255,232,244, - 21,255,252,233,244,250,255,248,3,255,139,19,255,72,139,19,255,252,243,15, - 16,3,255,252,242,15,16,3,255,15,182,19,132,210,15,136,244,2,255,248,4,255, - 137,208,209,252,234,131,224,1,252,247,216,49,194,255,72,137,208,72,209,252, - 234,72,131,224,1,72,252,247,216,72,49,194,255,72,133,210,15,149,210,255,73, - 137,149,233,255,65,137,149,233,255,252,242,65,15,17,133,233,255,252,243,65, - 15,17,133,233,255,65,136,149,233,255,65,128,141,233,235,255,76,137,252,239, - 255,132,192,15,133,244,251,232,244,12,252,233,244,1,248,5,255,72,129,195, + 174,233,73,137,159,233,72,184,237,237,65,84,73,137,228,72,129,228,239,252, + 255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139,167,233,77, + 139,174,233,73,139,174,233,73,43,175,233,255,73,3,175,233,131,252,248,0,15, + 141,244,248,72,131,196,8,195,248,2,232,244,11,72,139,52,36,72,57,252,235, + 15,133,244,1,184,237,72,131,196,8,195,255,248,28,81,82,72,131,252,236,16, + 72,137,252,247,72,137,214,72,137,226,72,184,237,237,65,84,73,137,228,72,129, + 228,239,252,255,208,76,137,228,65,92,72,131,196,16,90,89,132,192,15,132,244, + 248,72,139,68,36,224,195,248,2,72,49,192,72,252,247,208,195,255,76,57,227, + 15,133,244,249,255,76,137,225,72,41,217,72,129,252,249,239,15,131,244,249, + 255,248,2,255,232,244,14,255,232,244,26,255,232,244,19,255,232,244,21,255, + 252,233,244,250,255,248,3,255,139,19,255,72,139,19,255,252,243,15,16,3,255, + 252,242,15,16,3,255,15,182,19,132,210,15,136,244,2,255,248,4,255,137,208, + 209,252,234,131,224,1,252,247,216,49,194,255,72,137,208,72,209,252,234,72, + 131,224,1,72,252,247,216,72,49,194,255,72,133,210,15,149,210,255,73,137,149, + 233,255,65,137,149,233,255,252,242,65,15,17,133,233,255,252,243,65,15,17, + 133,233,255,65,136,149,233,255,65,128,141,233,235,255,76,137,252,239,255, + 72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65, + 92,255,132,192,15,133,244,251,232,244,12,252,233,244,1,248,5,255,72,129,195, 239,255,232,244,22,255,232,244,23,255,232,244,18,255,232,244,20,255,252,246, 3,128,15,133,244,2,255,249,248,1,255,76,57,227,15,132,244,252,255,76,137, 225,72,41,217,72,131,252,249,2,15,130,244,252,255,15,182,19,132,210,15,137, @@ -104,11 +114,13 @@ static const unsigned char upb_jit_actionlist[1914] = { 59,239,15,133,244,249,128,187,233,235,255,15,132,244,250,248,3,255,232,245, 72,133,192,15,132,245,252,255,224,255,252,233,245,255,248,4,72,129,195,239, 248,5,255,248,1,76,137,252,239,255,132,192,15,133,244,248,232,244,12,252, - 233,244,1,248,2,255,144,255,248,9,255,73,139,151,233,255,249,249,72,131,252, - 236,8,255,72,137,252,234,72,41,218,255,72,133,192,15,133,244,248,232,244, - 12,252,233,244,1,248,2,255,73,137,197,255,72,57,252,235,15,132,244,250,248, - 1,76,57,227,15,133,244,248,232,244,12,252,233,244,1,248,2,255,72,137,218, - 76,137,225,72,41,217,77,139,135,233,255,72,1,195,255,76,57,227,15,132,244, + 233,244,1,248,2,255,144,255,248,9,255,73,139,151,233,72,184,237,237,65,84, + 73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,255,249,249,72,131, + 252,236,8,255,72,137,252,234,72,41,218,255,72,133,192,15,133,244,248,232, + 244,12,252,233,244,1,248,2,255,73,137,197,255,72,57,252,235,15,132,244,250, + 248,1,76,57,227,15,133,244,248,232,244,12,252,233,244,1,248,2,255,72,137, + 218,76,137,225,72,41,217,77,139,135,233,72,184,237,237,65,84,73,137,228,72, + 129,228,239,252,255,208,76,137,228,65,92,72,1,195,255,76,57,227,15,132,244, 249,232,244,29,248,3,255,76,137,227,255,72,57,252,235,15,133,244,1,248,4, 255,77,137,174,233,73,199,134,233,0,0,0,0,73,129,198,239,77,59,183,233,15, 132,244,15,65,199,134,233,237,255,232,244,13,255,73,129,252,238,239,77,139, @@ -222,17 +234,21 @@ static const char *const upb_jit_globalnames[] = { //| add DELIMEND, DECODER->buf //|.endmacro //| -//| // OPT: use "call rel32" where possible. +//| // Calls an external C function at address "addr". //|.macro callp, addr -//|| { -//|| //int64_t ofs = (int64_t)addr - (int64_t)upb_status_init; -//|| //if (ofs > (1 << 30) || ofs < -(1 << 30)) { //| mov64 rax, (uintptr_t)addr +//| +//| // Stack must be 16-byte aligned (x86-64 ABI requires this). +//| // +//| // OPT: possibly remove this by statically ensuring correct alignment. +//| // +//| // OPT: use "call rel32" where possible. +//| push r12 +//| mov r12, rsp +//| and rsp, 0xfffffffffffffff0UL // Align stack. //| call rax -//|| //} else { -//| // call &addr -//|| //} -//|| } +//| mov rsp, r12 +//| pop r12 //|.endmacro //| //|.macro ld64, val @@ -331,7 +347,7 @@ static void asmlabel(jitcompiler *jc, const char *fmt, ...) { // we do it here instead. //|=>pclabel: dasm_put(Dst, 0, pclabel); -# 172 "upb/pb/compile_decoder_x64.dasc" +# 176 "upb/pb/compile_decoder_x64.dasc" upb_inttable_insert(&jc->asmlabels, pclabel, upb_value_ptr(str)); #endif } @@ -371,25 +387,10 @@ static void emit_static_asm(jitcompiler *jc) { //| push r12 //| push rbx //| - //| // Align stack. - //| // Since the JIT can call other functions (the JIT'ted code is not a leaf - //| // function) we must respect alignment rules. All x86-64 systems require - //| // 16-byte stack alignment. - //| sub rsp, 8 - //| //| mov rbx, ARG2_64 // Preserve JIT method. //| //| mov DECODER, rdi //| callp upb_pbdecoder_resume // Same args as us; reuse regs. - dasm_put(Dst, 2); - { - //int64_t ofs = (int64_t)upb_pbdecoder_resume - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_resume), (unsigned int)(((uintptr_t)upb_pbdecoder_resume)>>32)); - //} else { - //} - } -# 221 "upb/pb/compile_decoder_x64.dasc" //| test eax, eax //| jns >1 //| mov DECODER->saved_rsp, rsp @@ -406,8 +407,9 @@ static void emit_static_asm(jitcompiler *jc) { //| mov rax, DECODER->size_param //| mov qword DECODER->call_len, 0 //|1: - //| add rsp, 8 // Counter previous alignment. //| pop rbx + dasm_put(Dst, 2, (unsigned int)((uintptr_t)upb_pbdecoder_resume), (unsigned int)(((uintptr_t)upb_pbdecoder_resume)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->call_len), Dt2(->size_param), Dt2(->call_len)); +# 236 "upb/pb/compile_decoder_x64.dasc" //| pop r12 //| pop r13 //| pop r14 @@ -421,23 +423,14 @@ static void emit_static_asm(jitcompiler *jc) { //| sub rsp, ARG3_64 //| mov ARG1_64, rsp //| callp memcpy // Restore stack. - dasm_put(Dst, 38, Dt2(->saved_rsp), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->call_len), Dt2(->size_param), Dt2(->call_len), Dt2(->callstack)); - { - //int64_t ofs = (int64_t)memcpy - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32)); - //} else { - //} - } -# 252 "upb/pb/compile_decoder_x64.dasc" //| ret // Return to resumed function (not ->enterjit caller). //| //| // Other code can call this to suspend the JIT. //| // To the calling code, it will appear that the function returns when //| // the JIT resumes, and more buffer space will be available. //| // Args: eax=the value that decode() should return. - dasm_put(Dst, 135); -# 258 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 115, Dt2(->callstack), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL); +# 255 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "exitjit"); //|->exitjit: //| // Save the stack into DECODER->callstack. @@ -448,21 +441,11 @@ static void emit_static_asm(jitcompiler *jc) { //| mov DECODER->call_len, ARG3_64 // Preserve len for next resume. //| mov ebx, eax // Preserve return value across memcpy. //| callp memcpy // Copy stack into decoder. - dasm_put(Dst, 137, Dt2(->callstack), Dt2(->saved_rsp), Dt2(->call_len)); - { - //int64_t ofs = (int64_t)memcpy - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32)); - //} else { - //} - } -# 268 "upb/pb/compile_decoder_x64.dasc" //| mov eax, ebx // This will be our return value. //| //| // Must NOT do this before the memcpy(), otherwise memcpy() will //| // clobber the stack we are trying to save! //| mov rsp, DECODER->saved_rsp - //| add rsp, 8 // Counter previous alignment. //| pop rbx //| pop r12 //| pop r13 @@ -474,8 +457,8 @@ static void emit_static_asm(jitcompiler *jc) { //| // Like suspend() in the C decoder, except that the function appears //| // (from the caller's perspective) not to return until the decoder is //| // resumed. - dasm_put(Dst, 160, Dt2(->saved_rsp)); -# 285 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 161, Dt2(->callstack), Dt2(->saved_rsp), Dt2(->call_len), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp)); +# 281 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "suspend"); //|->suspend: //| cmp DECODER->ptr, PTR @@ -485,32 +468,23 @@ static void emit_static_asm(jitcompiler *jc) { //| commit_regs //| mov rdi, DECODER //| callp upb_pbdecoder_suspend - dasm_put(Dst, 182, Dt2(->ptr), Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure)); - { - //int64_t ofs = (int64_t)upb_pbdecoder_suspend - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_suspend), (unsigned int)(((uintptr_t)upb_pbdecoder_suspend)>>32)); - //} else { - //} - } -# 294 "upb/pb/compile_decoder_x64.dasc" //| jmp ->exitjit //| - dasm_put(Dst, 235); -# 296 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 222, Dt2(->ptr), Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_suspend), (unsigned int)(((uintptr_t)upb_pbdecoder_suspend)>>32), 0xfffffffffffffff0UL); +# 292 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "pushlendelim"); //|->pushlendelim: //|1: //| mov FRAME->sink.closure, CLOSURE //| mov DECODER->checkpoint, PTR //| dv32 - dasm_put(Dst, 240, Dt1(->sink.closure), Dt2(->checkpoint)); + dasm_put(Dst, 300, Dt1(->sink.closure), Dt2(->checkpoint)); if (1 == 1) { - dasm_put(Dst, 253); + dasm_put(Dst, 313); } else { - dasm_put(Dst, 261); + dasm_put(Dst, 321); } -# 302 "upb/pb/compile_decoder_x64.dasc" +# 298 "upb/pb/compile_decoder_x64.dasc" //| mov rcx, DELIMEND //| sub rcx, PTR //| sub rcx, rdx @@ -530,8 +504,8 @@ static void emit_static_asm(jitcompiler *jc) { //| cmp DELIMEND, DATAEND //| ja >2 //| mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND - dasm_put(Dst, 277, Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), Dt2(->end)); -# 321 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 337, Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), Dt2(->end)); +# 317 "upb/pb/compile_decoder_x64.dasc" //|2: //| ret //|3: @@ -540,34 +514,26 @@ static void emit_static_asm(jitcompiler *jc) { //| // Prepare seterr args. //| mov ARG1_64, DECODER //| ld64 kPbDecoderStackOverflow - dasm_put(Dst, 368, Dt2(->checkpoint)); + dasm_put(Dst, 428, Dt2(->checkpoint)); { uintptr_t v = (uintptr_t)kPbDecoderStackOverflow; if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 329 "upb/pb/compile_decoder_x64.dasc" +# 325 "upb/pb/compile_decoder_x64.dasc" //| callp upb_pbdecoder_seterr - { - //int64_t ofs = (int64_t)upb_pbdecoder_seterr - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32)); - //} else { - //} - } -# 330 "upb/pb/compile_decoder_x64.dasc" //| call ->suspend //| jmp <1 //| //| // For getting a value that spans a buffer seam. Falls back to C. //| // Args: rdi=C decoding function (prototype: int f(upb_pbdecoder*, void*)) - dasm_put(Dst, 398); -# 335 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 458, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32), 0xfffffffffffffff0UL); +# 331 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "getvalue_slow"); //|->getvalue_slow: //| sub rsp, 16 // Stack is [8-byte value, 8-byte func pointer] @@ -581,8 +547,8 @@ static void emit_static_asm(jitcompiler *jc) { //| call aword [rsp + 8] //| load_regs //| test eax, eax - dasm_put(Dst, 406, 8, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), 8, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); -# 348 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 487, 8, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), 8, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); +# 344 "upb/pb/compile_decoder_x64.dasc" //| jns >2 //| // Success; return parsed data (in rdx AND xmm0). //| mov rdx, [rsp] @@ -593,8 +559,8 @@ static void emit_static_asm(jitcompiler *jc) { //| call ->exitjit // Return eax from decode function. //| jmp <1 //| - dasm_put(Dst, 507); -# 358 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 588); +# 354 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "parse_unknown"); //| // Args: edx=fieldnum, cl=wire type //|->parse_unknown: @@ -605,18 +571,11 @@ static void emit_static_asm(jitcompiler *jc) { //| movzx ARG3_32, cl //| commit_regs //| callp upb_pbdecoder_skipunknown - dasm_put(Dst, 538, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure)); - { - //int64_t ofs = (int64_t)upb_pbdecoder_skipunknown - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32)); - //} else { - //} - } -# 368 "upb/pb/compile_decoder_x64.dasc" //| load_regs //| cmp eax, DECODE_ENDGROUP //| jne >1 + dasm_put(Dst, 619, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP); +# 367 "upb/pb/compile_decoder_x64.dasc" //| ret // Return eax=DECODE_ENDGROUP, not zero //|1: //| cmp eax, DECODE_OK @@ -635,8 +594,8 @@ static void emit_static_asm(jitcompiler *jc) { //| // re-join the fast path which will add fast_path_bytes after the callback //| // completes. We also set DECODER->ptr to this value which is a signal to //| // ->suspend that DECODER->checkpoint is up to date. - dasm_put(Dst, 582, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP, DECODE_OK); -# 389 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 716, DECODE_OK); +# 385 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "skip_decode_f32_fallback"); //|->skipf32_fallback: //|->decodef32_fallback: @@ -646,8 +605,8 @@ static void emit_static_asm(jitcompiler *jc) { //| mov DECODER->ptr, PTR //| ret //| - dasm_put(Dst, 638, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), Dt2(->ptr)); -# 398 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 740, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), Dt2(->ptr)); +# 394 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "skip_decode_f64_fallback"); //|->skipf64_fallback: //|->decodef64_fallback: @@ -658,19 +617,19 @@ static void emit_static_asm(jitcompiler *jc) { //| ret //| //| // Called for varint >= 1 byte. - dasm_put(Dst, 660, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), Dt2(->ptr)); -# 408 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 762, (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), Dt2(->ptr)); +# 404 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "skip_decode_v32_fallback"); //|->skipv32_fallback: //|->skipv64_fallback: //| chkeob 16, >1 - dasm_put(Dst, 682); + dasm_put(Dst, 784); if (16 == 1) { - dasm_put(Dst, 687); + dasm_put(Dst, 789); } else { - dasm_put(Dst, 695); + dasm_put(Dst, 797); } -# 412 "upb/pb/compile_decoder_x64.dasc" +# 408 "upb/pb/compile_decoder_x64.dasc" //| // With at least 16 bytes left, we can do a branch-less SSE version. //| movdqu xmm0, [PTR] //| pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0. @@ -698,8 +657,8 @@ static void emit_static_asm(jitcompiler *jc) { //| ret //| //| // Returns tag in edx - dasm_put(Dst, 711, 10); -# 439 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 813, 10); +# 435 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "decode_unknown_tag_fallback"); //|->decode_unknown_tag_fallback: //| sub rsp, 16 @@ -715,16 +674,9 @@ static void emit_static_asm(jitcompiler *jc) { //| mov ARG2_64, rsp //| commit_regs //| callp upb_pbdecoder_decode_varint_slow - dasm_put(Dst, 780, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure)); - { - //int64_t ofs = (int64_t)upb_pbdecoder_decode_varint_slow - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32)); - //} else { - //} - } -# 454 "upb/pb/compile_decoder_x64.dasc" //| load_regs + dasm_put(Dst, 882, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure)); +# 451 "upb/pb/compile_decoder_x64.dasc" //| cmp eax, 0 //| jge >3 //| mov edx, [rsp] // Success; return parsed data. @@ -735,31 +687,22 @@ static void emit_static_asm(jitcompiler *jc) { //| jmp <1 //| //| // Called for varint >= 1 byte. - dasm_put(Dst, 846, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); -# 465 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 985, Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf)); +# 461 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "decode_v32_v64_fallback"); //|->decodev32_fallback: //|->decodev64_fallback: //| chkeob 10, ->decode_varint_slow - dasm_put(Dst, 900); + dasm_put(Dst, 1023); if (10 == 1) { - dasm_put(Dst, 905); + dasm_put(Dst, 1028); } else { - dasm_put(Dst, 913); + dasm_put(Dst, 1036); } -# 469 "upb/pb/compile_decoder_x64.dasc" +# 465 "upb/pb/compile_decoder_x64.dasc" //| // OPT: do something faster than just calling the C version. //| mov rdi, PTR //| callp upb_vdecode_fast - dasm_put(Dst, 929); - { - //int64_t ofs = (int64_t)upb_vdecode_fast - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32)); - //} else { - //} - } -# 472 "upb/pb/compile_decoder_x64.dasc" //| test rax, rax //| je ->decode_varint_slow // Unterminated varint. //| mov PTR, rax @@ -767,8 +710,8 @@ static void emit_static_asm(jitcompiler *jc) { //| mov DECODER->ptr, PTR //| ret //| - dasm_put(Dst, 933, Dt2(->ptr)); -# 479 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1052, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32), 0xfffffffffffffff0UL, Dt2(->ptr)); +# 475 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "decode_varint_slow"); //|->decode_varint_slow: //| // Slow path: end of buffer or error (varint length >= 10). @@ -779,8 +722,8 @@ static void emit_static_asm(jitcompiler *jc) { //| ret //| //| // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH}) - dasm_put(Dst, 954, (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), Dt2(->ptr)); -# 489 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1097, (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), Dt2(->ptr)); +# 485 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "checktag_fallback"); //|->checktag_fallback: //| sub rsp, 8 @@ -790,16 +733,9 @@ static void emit_static_asm(jitcompiler *jc) { //| commit_regs //| mov DECODER->checkpoint, PTR //| callp upb_pbdecoder_checktag_slow - dasm_put(Dst, 974, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint)); - { - //int64_t ofs = (int64_t)upb_pbdecoder_checktag_slow - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32)); - //} else { - //} - } -# 498 "upb/pb/compile_decoder_x64.dasc" //| load_regs + dasm_put(Dst, 1117, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint), (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs)); +# 495 "upb/pb/compile_decoder_x64.dasc" //| cmp eax, 0 //| jge >2 //| add rsp, 8 @@ -816,8 +752,8 @@ static void emit_static_asm(jitcompiler *jc) { //| // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found). //| // Preserves: rcx, rdx //| // OPT: Could write this in assembly if it's a hotspot. - dasm_put(Dst, 1028, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_EOF); -# 515 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1216, Dt2(->buf), DECODE_EOF); +# 511 "upb/pb/compile_decoder_x64.dasc" asmlabel(jc, "hashlookup"); //|->hashlookup: //| push rcx @@ -827,15 +763,6 @@ static void emit_static_asm(jitcompiler *jc) { //| mov rsi, rdx //| mov rdx, rsp //| callp upb_inttable_lookup - dasm_put(Dst, 1094); - { - //int64_t ofs = (int64_t)upb_inttable_lookup - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32)); - //} else { - //} - } -# 524 "upb/pb/compile_decoder_x64.dasc" //| add rsp, 16 //| pop rdx //| pop rcx @@ -847,8 +774,8 @@ static void emit_static_asm(jitcompiler *jc) { //| xor rax, rax //| not rax //| ret - dasm_put(Dst, 1114); -# 535 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1258, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32), 0xfffffffffffffff0UL); +# 531 "upb/pb/compile_decoder_x64.dasc" } static void jitprimitive(jitcompiler *jc, opcode op, @@ -865,85 +792,85 @@ static void jitprimitive(jitcompiler *jc, opcode op, if (handler) { //|1: //| chkneob fastbytes, >3 - dasm_put(Dst, 1143); + dasm_put(Dst, 112); if (fastbytes == 1) { - dasm_put(Dst, 1146); + dasm_put(Dst, 1327); } else { - dasm_put(Dst, 1154, fastbytes); + dasm_put(Dst, 1335, fastbytes); } -# 551 "upb/pb/compile_decoder_x64.dasc" +# 547 "upb/pb/compile_decoder_x64.dasc" //|2: - dasm_put(Dst, 1170); -# 552 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1351); +# 548 "upb/pb/compile_decoder_x64.dasc" switch (type) { case V32: //| call ->decodev32_fallback - dasm_put(Dst, 1173); -# 555 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1354); +# 551 "upb/pb/compile_decoder_x64.dasc" break; case V64: //| call ->decodev64_fallback - dasm_put(Dst, 1177); -# 558 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1358); +# 554 "upb/pb/compile_decoder_x64.dasc" break; case F32: //| call ->decodef32_fallback - dasm_put(Dst, 1181); -# 561 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1362); +# 557 "upb/pb/compile_decoder_x64.dasc" break; case F64: //| call ->decodef64_fallback - dasm_put(Dst, 1185); -# 564 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1366); +# 560 "upb/pb/compile_decoder_x64.dasc" break; case X: break; } //| jmp >4 - dasm_put(Dst, 1189); -# 568 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1370); +# 564 "upb/pb/compile_decoder_x64.dasc" // Fast path decode; for when check_bytes bytes are available. //|3: - dasm_put(Dst, 1194); -# 571 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1375); +# 567 "upb/pb/compile_decoder_x64.dasc" switch (op) { case OP_PARSE_SFIXED32: case OP_PARSE_FIXED32: //| mov edx, dword [PTR] - dasm_put(Dst, 1197); -# 575 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1378); +# 571 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_SFIXED64: case OP_PARSE_FIXED64: //| mov rdx, qword [PTR] - dasm_put(Dst, 1200); -# 579 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1381); +# 575 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_FLOAT: //| movss xmm0, dword [PTR] - dasm_put(Dst, 1204); -# 582 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1385); +# 578 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_DOUBLE: //| movsd xmm0, qword [PTR] - dasm_put(Dst, 1210); -# 585 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1391); +# 581 "upb/pb/compile_decoder_x64.dasc" break; default: // Inline one byte of varint decoding. //| movzx edx, byte [PTR] //| test dl, dl //| js <2 // Fallback to slow path for >1 byte varint. - dasm_put(Dst, 1216); -# 591 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1397); +# 587 "upb/pb/compile_decoder_x64.dasc" break; } // Second-stage decode; used for both fast and slow paths // (only needed for a few types). //|4: - dasm_put(Dst, 1226); -# 597 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1407); +# 593 "upb/pb/compile_decoder_x64.dasc" switch (op) { case OP_PARSE_SINT32: // 32-bit zig-zag decode. @@ -952,8 +879,8 @@ static void jitprimitive(jitcompiler *jc, opcode op, //| and eax, 1 //| neg eax //| xor edx, eax - dasm_put(Dst, 1229); -# 605 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1410); +# 601 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_SINT64: // 64-bit zig-zag decode. @@ -962,14 +889,14 @@ static void jitprimitive(jitcompiler *jc, opcode op, //| and rax, 1 //| neg rax //| xor rdx, rax - dasm_put(Dst, 1243); -# 613 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1424); +# 609 "upb/pb/compile_decoder_x64.dasc" break; case OP_PARSE_BOOL: //| test rdx, rdx //| setne dl - dasm_put(Dst, 1262); -# 617 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1443); +# 613 "upb/pb/compile_decoder_x64.dasc" break; default: break; } @@ -982,30 +909,30 @@ static void jitprimitive(jitcompiler *jc, opcode op, case UPB_TYPE_INT64: case UPB_TYPE_UINT64: //| mov [CLOSURE + data->offset], rdx - dasm_put(Dst, 1269, data->offset); -# 629 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1450, data->offset); +# 625 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_INT32: case UPB_TYPE_UINT32: case UPB_TYPE_ENUM: //| mov [CLOSURE + data->offset], edx - dasm_put(Dst, 1274, data->offset); -# 634 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1455, data->offset); +# 630 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_DOUBLE: //| movsd qword [CLOSURE + data->offset], XMMARG1 - dasm_put(Dst, 1279, data->offset); -# 637 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1460, data->offset); +# 633 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_FLOAT: //| movss dword [CLOSURE + data->offset], XMMARG1 - dasm_put(Dst, 1287, data->offset); -# 640 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1468, data->offset); +# 636 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_BOOL: //| mov [CLOSURE + data->offset], dl - dasm_put(Dst, 1295, data->offset); -# 643 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1476, data->offset); +# 639 "upb/pb/compile_decoder_x64.dasc" break; case UPB_TYPE_STRING: case UPB_TYPE_BYTES: @@ -1014,98 +941,92 @@ static void jitprimitive(jitcompiler *jc, opcode op, } //| sethas CLOSURE, data->hasbit if (data->hasbit >= 0) { - dasm_put(Dst, 1300, ((uint32_t)data->hasbit / 8), (1 << ((uint32_t)data->hasbit % 8))); + dasm_put(Dst, 1481, ((uint32_t)data->hasbit / 8), (1 << ((uint32_t)data->hasbit % 8))); } -# 650 "upb/pb/compile_decoder_x64.dasc" +# 646 "upb/pb/compile_decoder_x64.dasc" } else if (handler) { //| mov ARG1_64, CLOSURE //| load_handler_data h, sel - dasm_put(Dst, 1306); + dasm_put(Dst, 1487); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, sel); if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 653 "upb/pb/compile_decoder_x64.dasc" +# 649 "upb/pb/compile_decoder_x64.dasc" //| callp handler - { - //int64_t ofs = (int64_t)handler - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32)); - //} else { - //} - } -# 654 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1492, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32), 0xfffffffffffffff0UL); +# 650 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, sel)) { //| test al, al //| jnz >5 //| call ->suspend //| jmp <1 //|5: - dasm_put(Dst, 1311); -# 660 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1514); +# 656 "upb/pb/compile_decoder_x64.dasc" } } // We do this last so that the checkpoint is not advanced past the user's // data until the callback has returned success. //| add PTR, fastbytes - dasm_put(Dst, 1327, fastbytes); -# 666 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1530, fastbytes); +# 662 "upb/pb/compile_decoder_x64.dasc" } else { // No handler registered for this value, just skip it. //| chkneob fastbytes, >3 if (fastbytes == 1) { - dasm_put(Dst, 1146); + dasm_put(Dst, 1327); } else { - dasm_put(Dst, 1154, fastbytes); + dasm_put(Dst, 1335, fastbytes); } -# 669 "upb/pb/compile_decoder_x64.dasc" +# 665 "upb/pb/compile_decoder_x64.dasc" //|2: - dasm_put(Dst, 1170); -# 670 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1351); +# 666 "upb/pb/compile_decoder_x64.dasc" switch (type) { case V32: //| call ->skipv32_fallback - dasm_put(Dst, 1332); -# 673 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1535); +# 669 "upb/pb/compile_decoder_x64.dasc" break; case V64: //| call ->skipv64_fallback - dasm_put(Dst, 1336); -# 676 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1539); +# 672 "upb/pb/compile_decoder_x64.dasc" break; case F32: //| call ->skipf32_fallback - dasm_put(Dst, 1340); -# 679 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1543); +# 675 "upb/pb/compile_decoder_x64.dasc" break; case F64: //| call ->skipf64_fallback - dasm_put(Dst, 1344); -# 682 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1547); +# 678 "upb/pb/compile_decoder_x64.dasc" break; case X: break; } // Fast-path skip. //|3: - dasm_put(Dst, 1194); -# 688 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1375); +# 684 "upb/pb/compile_decoder_x64.dasc" if (type == V32 || type == V64) { //| test byte [PTR], 0x80 //| jnz <2 - dasm_put(Dst, 1348); -# 691 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1551); +# 687 "upb/pb/compile_decoder_x64.dasc" } //| add PTR, fastbytes - dasm_put(Dst, 1327, fastbytes); -# 693 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1530, fastbytes); +# 689 "upb/pb/compile_decoder_x64.dasc" } } @@ -1125,18 +1046,18 @@ static void jitdispatch(jitcompiler *jc, //|=>define_jmptarget(jc, &method->dispatch): //|1: - dasm_put(Dst, 1357, define_jmptarget(jc, &method->dispatch)); -# 712 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1560, define_jmptarget(jc, &method->dispatch)); +# 708 "upb/pb/compile_decoder_x64.dasc" // Decode the field tag. //| mov aword DECODER->checkpoint, PTR //| chkeob 2, >6 - dasm_put(Dst, 248, Dt2(->checkpoint)); + dasm_put(Dst, 308, Dt2(->checkpoint)); if (2 == 1) { - dasm_put(Dst, 1361); + dasm_put(Dst, 1564); } else { - dasm_put(Dst, 1369); + dasm_put(Dst, 1572); } -# 715 "upb/pb/compile_decoder_x64.dasc" +# 711 "upb/pb/compile_decoder_x64.dasc" //| movzx edx, byte [PTR] //| test dl, dl //| jns >7 // Jump if first byte has no continuation bit. @@ -1160,49 +1081,49 @@ static void jitdispatch(jitcompiler *jc, //| mov ecx, edx //| shr edx, 3 //| and cl, 7 - dasm_put(Dst, 1385, 1); -# 738 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1588, 1); +# 734 "upb/pb/compile_decoder_x64.dasc" // See comment attached to upb_pbdecodermethod.dispatch for layout of the // dispatch table. //|2: //| cmp edx, dispatch->array_size - dasm_put(Dst, 1450, dispatch->array_size); -# 743 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1653, dispatch->array_size); +# 739 "upb/pb/compile_decoder_x64.dasc" if (has_hash_entries) { //| jae >7 - dasm_put(Dst, 1457); -# 745 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1660); +# 741 "upb/pb/compile_decoder_x64.dasc" } else { //| jae >5 - dasm_put(Dst, 1462); -# 747 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1665); +# 743 "upb/pb/compile_decoder_x64.dasc" } //| // OPT: Compact the lookup arr into 32-bit entries. if ((uintptr_t)dispatch->array > 0x7fffffff) { //| mov64 rax, (uintptr_t)dispatch->array //| mov rax, qword [rax + rdx * 8] - dasm_put(Dst, 1467, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32)); -# 752 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1670, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32)); +# 748 "upb/pb/compile_decoder_x64.dasc" } else { //| mov rax, qword [rdx * 8 + dispatch->array] - dasm_put(Dst, 1476, dispatch->array); -# 754 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1679, dispatch->array); +# 750 "upb/pb/compile_decoder_x64.dasc" } //|3: //| // We take advantage of the fact that non-present entries are stored //| // as -1, which will result in wire types that will never match. //| cmp al, cl - dasm_put(Dst, 1482); -# 759 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1685); +# 755 "upb/pb/compile_decoder_x64.dasc" if (has_multi_wiretype) { //| jne >6 - dasm_put(Dst, 1487); -# 761 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1690); +# 757 "upb/pb/compile_decoder_x64.dasc" } else { //| jne >5 - dasm_put(Dst, 1492); -# 763 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1695); +# 759 "upb/pb/compile_decoder_x64.dasc" } //| shr rax, 16 //| @@ -1227,8 +1148,8 @@ static void jitdispatch(jitcompiler *jc, //| jz <1 //| lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG. //| ret - dasm_put(Dst, 1497, define_jmptarget(jc, dispatch->array)); -# 787 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1700, define_jmptarget(jc, dispatch->array)); +# 783 "upb/pb/compile_decoder_x64.dasc" if (has_multi_wiretype) { //|6: @@ -1238,46 +1159,46 @@ static void jitdispatch(jitcompiler *jc, //| // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER. //| add rdx, UPB_MAX_FIELDNUMBER //| // This key will never be in the array part, so do a hash lookup. - dasm_put(Dst, 1531, UPB_MAX_FIELDNUMBER); -# 796 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1734, UPB_MAX_FIELDNUMBER); +# 792 "upb/pb/compile_decoder_x64.dasc" assert(has_hash_entries); //| ld64 dispatch { uintptr_t v = (uintptr_t)dispatch; if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 798 "upb/pb/compile_decoder_x64.dasc" +# 794 "upb/pb/compile_decoder_x64.dasc" //| jmp ->hashlookup // Tail call. - dasm_put(Dst, 1544); -# 799 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1747); +# 795 "upb/pb/compile_decoder_x64.dasc" } if (has_hash_entries) { //|7: //| // Hash table lookup. //| ld64 dispatch - dasm_put(Dst, 1549); + dasm_put(Dst, 1752); { uintptr_t v = (uintptr_t)dispatch; if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 805 "upb/pb/compile_decoder_x64.dasc" +# 801 "upb/pb/compile_decoder_x64.dasc" //| call ->hashlookup //| jmp <3 - dasm_put(Dst, 1552); -# 807 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1755); +# 803 "upb/pb/compile_decoder_x64.dasc" } } @@ -1299,11 +1220,11 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, //| chkneob n, >1 if (n == 1) { - dasm_put(Dst, 1560); + dasm_put(Dst, 1763); } else { - dasm_put(Dst, 1568, n); + dasm_put(Dst, 1771, n); } -# 827 "upb/pb/compile_decoder_x64.dasc" +# 823 "upb/pb/compile_decoder_x64.dasc" //| // OPT: this is way too much fallback code to put here. //| // Reduce and/or move to a separate section to make better icache usage. @@ -1311,36 +1232,36 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, { uintptr_t v = (uintptr_t)tag; if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 831 "upb/pb/compile_decoder_x64.dasc" +# 827 "upb/pb/compile_decoder_x64.dasc" //| call ->checktag_fallback //| cmp eax, DECODE_MISMATCH //| je >3 //| cmp eax, DECODE_EOF //| je =>jmptarget(jc, delimend) //| jmp >5 - dasm_put(Dst, 1584, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend)); -# 837 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1787, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend)); +# 833 "upb/pb/compile_decoder_x64.dasc" //|1: - dasm_put(Dst, 1143); -# 839 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 112); +# 835 "upb/pb/compile_decoder_x64.dasc" switch (n) { case 1: //| cmp byte [PTR], tag - dasm_put(Dst, 1607, tag); -# 842 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1810, tag); +# 838 "upb/pb/compile_decoder_x64.dasc" break; case 2: //| cmp word [PTR], tag - dasm_put(Dst, 1611, tag); -# 845 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1814, tag); +# 841 "upb/pb/compile_decoder_x64.dasc" break; case 3: //| // OPT: Slightly more efficient code, but depends on an extra byte. @@ -1351,42 +1272,42 @@ static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs, //| jne >2 //| cmp byte [PTR + 2], (tag >> 16) //|2: - dasm_put(Dst, 1616, (tag & 0xffff), 2, (tag >> 16)); -# 855 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1819, (tag & 0xffff), 2, (tag >> 16)); +# 851 "upb/pb/compile_decoder_x64.dasc" break; case 4: //| cmp dword [PTR], tag - dasm_put(Dst, 1631, tag); -# 858 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1834, tag); +# 854 "upb/pb/compile_decoder_x64.dasc" break; case 5: //| cmp dword [PTR], (tag & 0xffffffff) //| jne >3 //| cmp byte [PTR + 4], (tag >> 32) - dasm_put(Dst, 1635, (tag & 0xffffffff), 4, (tag >> 32)); -# 863 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1838, (tag & 0xffffffff), 4, (tag >> 32)); +# 859 "upb/pb/compile_decoder_x64.dasc" } //| je >4 //|3: - dasm_put(Dst, 1647); -# 866 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1850); +# 862 "upb/pb/compile_decoder_x64.dasc" if (ofs == 0) { //| call =>jmptarget(jc, &method->dispatch) //| test rax, rax //| jz =>jmptarget(jc, delimend) //| jmp rax - dasm_put(Dst, 1654, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend)); -# 871 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1857, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend)); +# 867 "upb/pb/compile_decoder_x64.dasc" } else { //| jmp =>jmptarget(jc, jc->pc + ofs) - dasm_put(Dst, 1666, jmptarget(jc, jc->pc + ofs)); -# 873 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1869, jmptarget(jc, jc->pc + ofs)); +# 869 "upb/pb/compile_decoder_x64.dasc" } //|4: //| add PTR, n //|5: - dasm_put(Dst, 1670, n); -# 877 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1873, n); +# 873 "upb/pb/compile_decoder_x64.dasc" } // Compile the bytecode to x64. @@ -1409,7 +1330,7 @@ static void jitbytecode(jitcompiler *jc) { // TODO: optimize this to only define pclabels that are actually used. //|=>define_jmptarget(jc, jc->pc): dasm_put(Dst, 0, define_jmptarget(jc, jc->pc)); -# 898 "upb/pb/compile_decoder_x64.dasc" +# 894 "upb/pb/compile_decoder_x64.dasc" } jc->pc++; @@ -1422,75 +1343,62 @@ static void jitbytecode(jitcompiler *jc) { //|1: //| mov ARG1_64, CLOSURE //| load_handler_data h, UPB_STARTMSG_SELECTOR - dasm_put(Dst, 1679); + dasm_put(Dst, 1882); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_STARTMSG_SELECTOR); if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 910 "upb/pb/compile_decoder_x64.dasc" +# 906 "upb/pb/compile_decoder_x64.dasc" //| callp startmsg - { - //int64_t ofs = (int64_t)startmsg - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32)); - //} else { - //} - } -# 911 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1492, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32), 0xfffffffffffffff0UL); +# 907 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, UPB_STARTMSG_SELECTOR)) { //| test al, al //| jnz >2 //| call ->suspend //| jmp <1 //|2: - dasm_put(Dst, 1686); -# 917 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1889); +# 913 "upb/pb/compile_decoder_x64.dasc" } } else { //| nop - dasm_put(Dst, 1702); -# 920 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1905); +# 916 "upb/pb/compile_decoder_x64.dasc" } break; } case OP_ENDMSG: { upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR); //|9: - dasm_put(Dst, 1704); -# 926 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1907); +# 922 "upb/pb/compile_decoder_x64.dasc" if (endmsg) { // bool endmsg(void *closure, const void *hd, upb_status *status) //| mov ARG1_64, CLOSURE //| load_handler_data h, UPB_ENDMSG_SELECTOR - dasm_put(Dst, 1306); + dasm_put(Dst, 1487); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_ENDMSG_SELECTOR); if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 930 "upb/pb/compile_decoder_x64.dasc" +# 926 "upb/pb/compile_decoder_x64.dasc" //| mov ARG3_64, DECODER->status //| callp endmsg - dasm_put(Dst, 1707, Dt2(->status)); - { - //int64_t ofs = (int64_t)endmsg - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32)); - //} else { - //} - } -# 932 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1910, Dt2(->status), (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32), 0xfffffffffffffff0UL); +# 928 "upb/pb/compile_decoder_x64.dasc" } break; } @@ -1521,8 +1429,8 @@ static void jitbytecode(jitcompiler *jc) { //|=>define_jmptarget(jc, op_pc): //|=>define_jmptarget(jc, method): //| sub rsp, 8 - dasm_put(Dst, 1712, define_jmptarget(jc, op_pc), define_jmptarget(jc, method)); -# 962 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1936, define_jmptarget(jc, op_pc), define_jmptarget(jc, method)); +# 958 "upb/pb/compile_decoder_x64.dasc" break; } @@ -1552,50 +1460,44 @@ static void jitbytecode(jitcompiler *jc) { //|1: //| mov ARG1_64, CLOSURE //| load_handler_data h, arg - dasm_put(Dst, 1679); + dasm_put(Dst, 1882); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 991 "upb/pb/compile_decoder_x64.dasc" +# 987 "upb/pb/compile_decoder_x64.dasc" if (op == OP_STARTSTR) { //| mov ARG3_64, DELIMEND //| sub ARG3_64, PTR - dasm_put(Dst, 1720); -# 994 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1944); +# 990 "upb/pb/compile_decoder_x64.dasc" } //| callp start - { - //int64_t ofs = (int64_t)start - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32)); - //} else { - //} - } -# 996 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1492, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32), 0xfffffffffffffff0UL); +# 992 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, arg)) { //| test rax, rax //| jnz >2 //| call ->suspend //| jmp <1 //|2: - dasm_put(Dst, 1728); -# 1002 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1952); +# 998 "upb/pb/compile_decoder_x64.dasc" } //| mov CLOSURE, rax - dasm_put(Dst, 1745); -# 1004 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1969); +# 1000 "upb/pb/compile_decoder_x64.dasc" } else { // TODO: nop is only required because of asmlabel(). //| nop - dasm_put(Dst, 1702); -# 1007 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1905); +# 1003 "upb/pb/compile_decoder_x64.dasc" } break; } @@ -1610,41 +1512,35 @@ static void jitbytecode(jitcompiler *jc) { //|1: //| mov ARG1_64, CLOSURE //| load_handler_data h, arg - dasm_put(Dst, 1679); + dasm_put(Dst, 1882); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 1021 "upb/pb/compile_decoder_x64.dasc" +# 1017 "upb/pb/compile_decoder_x64.dasc" //| callp end - { - //int64_t ofs = (int64_t)end - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32)); - //} else { - //} - } -# 1022 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1492, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32), 0xfffffffffffffff0UL); +# 1018 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, arg)) { //| test al, al //| jnz >2 //| call ->suspend //| jmp <1 //|2: - dasm_put(Dst, 1686); -# 1028 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1889); +# 1024 "upb/pb/compile_decoder_x64.dasc" } } else { // TODO: nop is only required because of asmlabel(). //| nop - dasm_put(Dst, 1702); -# 1032 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1905); +# 1028 "upb/pb/compile_decoder_x64.dasc" } break; } @@ -1658,59 +1554,50 @@ static void jitbytecode(jitcompiler *jc) { //| call ->suspend //| jmp <1 //|2: - dasm_put(Dst, 1749); -# 1045 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1973); +# 1041 "upb/pb/compile_decoder_x64.dasc" if (str) { // size_t str(void *closure, const void *hd, const char *str, size_t n) //| mov ARG1_64, CLOSURE //| load_handler_data h, arg - dasm_put(Dst, 1306); + dasm_put(Dst, 1487); { uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg); if (v > 0xffffffff) { - dasm_put(Dst, 386, (unsigned int)(v), (unsigned int)((v)>>32)); + dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32)); } else if (v) { - dasm_put(Dst, 391, v); + dasm_put(Dst, 451, v); } else { - dasm_put(Dst, 394); + dasm_put(Dst, 454); } } -# 1049 "upb/pb/compile_decoder_x64.dasc" +# 1045 "upb/pb/compile_decoder_x64.dasc" //| mov ARG3_64, PTR //| mov ARG4_64, DATAEND //| sub ARG4_64, PTR //| mov ARG5_64, qword DECODER->handle //| callp str - dasm_put(Dst, 1776, Dt2(->handle)); - { - //int64_t ofs = (int64_t)str - (int64_t)upb_status_init; - //if (ofs > (1 << 30) || ofs < -(1 << 30)) { - dasm_put(Dst, 30, (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32)); - //} else { - //} - } -# 1054 "upb/pb/compile_decoder_x64.dasc" //| add PTR, rax - dasm_put(Dst, 1790); -# 1055 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2000, Dt2(->handle), (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32), 0xfffffffffffffff0UL); +# 1051 "upb/pb/compile_decoder_x64.dasc" if (!alwaysok(h, arg)) { //| cmp PTR, DATAEND //| je >3 //| call ->strret_fallback //|3: - dasm_put(Dst, 1794); -# 1060 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2038); +# 1056 "upb/pb/compile_decoder_x64.dasc" } } else { //| mov PTR, DATAEND - dasm_put(Dst, 1807); -# 1063 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2051); +# 1059 "upb/pb/compile_decoder_x64.dasc" } //| cmp PTR, DELIMEND //| jne <1 //|4: - dasm_put(Dst, 1811); -# 1067 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2055); +# 1063 "upb/pb/compile_decoder_x64.dasc" break; } case OP_PUSHTAGDELIM: @@ -1725,19 +1612,19 @@ static void jitbytecode(jitcompiler *jc) { //| cmp FRAME, DECODER->limit //| je ->err //| mov dword FRAME->groupnum, arg - dasm_put(Dst, 1822, Dt1(->sink.closure), Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), arg); -# 1081 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2066, Dt1(->sink.closure), Dt1(->end_ofs), sizeof(upb_pbdecoder_frame), Dt2(->limit), Dt1(->groupnum), arg); +# 1077 "upb/pb/compile_decoder_x64.dasc" break; case OP_PUSHLENDELIM: //| call ->pushlendelim - dasm_put(Dst, 1852); -# 1084 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2096); +# 1080 "upb/pb/compile_decoder_x64.dasc" break; case OP_POP: //| sub FRAME, sizeof(upb_pbdecoder_frame) //| mov CLOSURE, FRAME->sink.closure - dasm_put(Dst, 1856, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure)); -# 1088 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2100, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure)); +# 1084 "upb/pb/compile_decoder_x64.dasc" break; case OP_SETDELIM: // OPT: experiment with testing vs old offset to optimize away. @@ -1749,36 +1636,36 @@ static void jitbytecode(jitcompiler *jc) { //| ja >1 // OPT: try cmov. //| mov DATAEND, DELIMEND //|1: - dasm_put(Dst, 1866, Dt2(->end), Dt1(->end_ofs), Dt2(->buf)); -# 1099 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2110, Dt2(->end), Dt1(->end_ofs), Dt2(->buf)); +# 1095 "upb/pb/compile_decoder_x64.dasc" break; case OP_SETBIGGROUPNUM: //| mov dword FRAME->groupnum, *jc->pc++ - dasm_put(Dst, 1846, Dt1(->groupnum), *jc->pc++); -# 1102 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2090, Dt1(->groupnum), *jc->pc++); +# 1098 "upb/pb/compile_decoder_x64.dasc" break; case OP_CHECKDELIM: //| cmp DELIMEND, PTR //| je =>jmptarget(jc, jc->pc + longofs) - dasm_put(Dst, 1896, jmptarget(jc, jc->pc + longofs)); -# 1106 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2140, jmptarget(jc, jc->pc + longofs)); +# 1102 "upb/pb/compile_decoder_x64.dasc" break; case OP_CALL: //| call =>jmptarget(jc, jc->pc + longofs) - dasm_put(Dst, 1903, jmptarget(jc, jc->pc + longofs)); -# 1109 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2147, jmptarget(jc, jc->pc + longofs)); +# 1105 "upb/pb/compile_decoder_x64.dasc" break; case OP_BRANCH: //| jmp =>jmptarget(jc, jc->pc + longofs); - dasm_put(Dst, 1666, jmptarget(jc, jc->pc + longofs)); -# 1112 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1869, jmptarget(jc, jc->pc + longofs)); +# 1108 "upb/pb/compile_decoder_x64.dasc" break; case OP_RET: //|9: //| add rsp, 8 //| ret - dasm_put(Dst, 1906); -# 1117 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 2150); +# 1113 "upb/pb/compile_decoder_x64.dasc" break; case OP_TAG1: jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method); @@ -1799,6 +1686,6 @@ static void jitbytecode(jitcompiler *jc) { asmlabel(jc, "eof"); //| nop - dasm_put(Dst, 1702); -# 1137 "upb/pb/compile_decoder_x64.dasc" + dasm_put(Dst, 1905); +# 1133 "upb/pb/compile_decoder_x64.dasc" } -- cgit v1.2.3