1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
|
/*
* upb - a minimalist implementation of protocol buffers.
*
* Copyright (c) 2013 Google Inc. See LICENSE for details.
* Author: Josh Haberman <jhaberman@gmail.com>
*
* Driver code for the x64 JIT compiler.
*/
#include <dlfcn.h>
#include <stdio.h>
#include <sys/mman.h>
#include "upb/pb/decoder.h"
#include "upb/pb/decoder.int.h"
#include "upb/pb/varint.int.h"
#include "upb/shim/shim.h"
// These defines are necessary for DynASM codegen.
// See dynasm/dasm_proto.h for more info.
#define Dst_DECL jitcompiler *jc
#define Dst_REF (jc->dynasm)
#define Dst (jc)
// In debug mode, make DynASM do internal checks (must be defined before any
// dasm header is included.
#ifndef NDEBUG
#define DASM_CHECKS
#endif
#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif
#define DECODE_EOF -3
typedef struct {
mgroup *group;
uint32_t *pc;
// This pointer is allocated by dasm_init() and freed by dasm_free().
struct dasm_State *dynasm;
// Maps arbitrary void* -> pclabel.
upb_inttable pclabels;
upb_inttable pcdefined;
// For marking labels that should go into the generated code.
// Maps pclabel -> char* label (string is owned by the table).
upb_inttable asmlabels;
// For checking that two asmlabels aren't defined for the same byte.
int lastlabelofs;
// The total number of pclabels currently defined.
uint32_t pclabel_count;
// Used by DynASM to store globals.
void **globals;
bool chkret;
} jitcompiler;
// Functions called by codegen.
static int pclabel(jitcompiler *jc, const void *here);
static int define_pclabel(jitcompiler *jc, const void *here);
static void asmlabel(jitcompiler *jc, const char *fmt, ...);
static int pcofs(jitcompiler* jc);
#include "dynasm/dasm_proto.h"
#include "dynasm/dasm_x86.h"
#include "upb/pb/compile_decoder_x64.h"
static jitcompiler *newjitcompiler(mgroup *group) {
jitcompiler *jc = malloc(sizeof(jitcompiler));
jc->chkret = false;
jc->group = group;
jc->pclabel_count = 0;
jc->lastlabelofs = -1;
upb_inttable_init(&jc->pclabels, UPB_CTYPE_UINT32);
upb_inttable_init(&jc->pcdefined, UPB_CTYPE_BOOL);
upb_inttable_init(&jc->asmlabels, UPB_CTYPE_PTR);
jc->globals = malloc(UPB_JIT_GLOBAL__MAX * sizeof(*jc->globals));
dasm_init(jc, 1);
dasm_setupglobal(jc, jc->globals, UPB_JIT_GLOBAL__MAX);
dasm_setup(jc, upb_jit_actionlist);
return jc;
}
static void freejitcompiler(jitcompiler *jc) {
upb_inttable_iter i;
upb_inttable_begin(&i, &jc->asmlabels);
for (; !upb_inttable_done(&i); upb_inttable_next(&i)) {
free(upb_value_getptr(upb_inttable_iter_value(&i)));
}
upb_inttable_uninit(&jc->asmlabels);
upb_inttable_uninit(&jc->pclabels);
upb_inttable_uninit(&jc->pcdefined);
dasm_free(jc);
free(jc->globals);
free(jc);
}
// Returns a pclabel associated with the given arbitrary pointer.
static int pclabel(jitcompiler *jc, const void *here) {
upb_value v;
bool found = upb_inttable_lookupptr(&jc->pclabels, here, &v);
if (!found) {
upb_value_setuint32(&v, jc->pclabel_count++);
dasm_growpc(jc, jc->pclabel_count);
upb_inttable_insertptr(&jc->pclabels, here, v);
}
return upb_value_getuint32(v);
}
// Defines a pclabel associated with the given arbitrary pointer.
// May only be called once (to avoid redefining the pclabel).
static int define_pclabel(jitcompiler *jc, const void *here) {
// Will assert-fail if it already exists.
upb_inttable_insertptr(&jc->pcdefined, here, upb_value_bool(true));
return pclabel(jc, here);
}
// Returns a bytecode pc offset relative to the beginning of the group's code.
static int pcofs(jitcompiler *jc) {
return jc->pc - jc->group->bytecode;
}
static void upb_reg_jit_gdb(jitcompiler *jc);
static int getpclabel(jitcompiler *jc, const void *target) {
return dasm_getpclabel(jc, pclabel(jc, target));
}
// Given a pcofs relative to method, returns the machine code offset for it
// (relative to the beginning of the machine code).
int nativeofs(jitcompiler *jc, const upb_pbdecodermethod *method, int pcofs) {
void *target = jc->group->bytecode + method->code_base.ofs + pcofs;
return getpclabel(jc, target);
}
// Given a pcofs relative to this method's base, returns a machine code offset
// relative to pclabel(dispatch->array) (which is used in jitdispatch as the
// machine code base for dispatch table lookups).
uint32_t dispatchofs(jitcompiler *jc, const upb_pbdecodermethod *method,
int pcofs) {
int ofs1 = getpclabel(jc, method->dispatch.array);
int ofs2 = nativeofs(jc, method, pcofs);
assert(ofs1 > 0);
assert(ofs2 > 0);
int ret = ofs2 - ofs1;
assert(ret > 0);
return ret;
}
// Rewrites the dispatch tables into machine code offsets.
static void patchdispatch(jitcompiler *jc) {
upb_inttable_iter i;
upb_inttable_begin(&i, &jc->group->methods);
for (; !upb_inttable_done(&i); upb_inttable_next(&i)) {
upb_pbdecodermethod *method = upb_value_getptr(upb_inttable_iter_value(&i));
method->is_native_ = true;
upb_inttable *dispatch = &method->dispatch;
upb_inttable_iter i2;
upb_inttable_begin(&i2, dispatch);
for (; !upb_inttable_done(&i2); upb_inttable_next(&i2)) {
uintptr_t key = upb_inttable_iter_key(&i2);
if (key == 0) continue;
uint64_t val = upb_value_getuint64(upb_inttable_iter_value(&i2));
uint64_t newval;
if (key <= UPB_MAX_FIELDNUMBER) {
// Primary slot.
uint64_t oldofs = val >> 16;
uint64_t newofs = dispatchofs(jc, method, oldofs);
newval = (val & 0xffff) | (newofs << 16);
assert((int64_t)newval > 0);
} else {
// Secondary slot. Since we have 64 bits for the value, we use an
// absolute offset.
newval = (uint64_t)(jc->group->jit_code + nativeofs(jc, method, val));
}
bool ok = upb_inttable_replace(dispatch, key, upb_value_uint64(newval));
UPB_ASSERT_VAR(ok, ok);
}
// Set this only *after* we have patched the offsets (nativeofs() above
// reads this).
method->code_base.ptr = jc->group->jit_code + getpclabel(jc, method);
upb_byteshandler *h = &method->input_handler_;
upb_byteshandler_setstartstr(h, upb_pbdecoder_startjit, NULL);
upb_byteshandler_setstring(h, jc->group->jit_code, method->code_base.ptr);
upb_byteshandler_setendstr(h, upb_pbdecoder_end, method);
}
}
// Define for JIT debugging.
#ifdef UPB_JIT_LOAD_SO
static void load_so(jitcompiler *jc) {
// Dump to a .so file in /tmp and load that, so all the tooling works right
// (for example, debuggers and profilers will see symbol names for the JIT-ted
// code). This is the same goal of the GDB JIT code below, but the GDB JIT
// interface is only used/understood by GDB. Hopefully a standard will
// develop for registering JIT-ted code that all tools will recognize,
// rendering this obsolete.
//
// Requires that gcc is available from the command-line.
// Convert all asm labels from pclabel offsets to machine code offsets.
upb_inttable_iter i;
upb_inttable mclabels;
upb_inttable_init(&mclabels, UPB_CTYPE_PTR);
upb_inttable_begin(&i, &jc->asmlabels);
for (; !upb_inttable_done(&i); upb_inttable_next(&i)) {
upb_inttable_insert(&mclabels,
dasm_getpclabel(jc, upb_inttable_iter_key(&i)),
upb_inttable_iter_value(&i));
}
FILE *f = fopen("/tmp/upb-jit-code.s", "w");
if (f) {
uint8_t *jit_code = (uint8_t*)jc->group->jit_code;
fputs(" .text\n\n", f);
size_t linelen = 0;
for (size_t i = 0; i < jc->group->jit_size; i++) {
upb_value v;
if (upb_inttable_lookup(&mclabels, i, &v)) {
const char *label = upb_value_getptr(v);
// "X." makes our JIT syms recognizable as such, which we build into
// other tooling.
fprintf(f, "\n\nX.%s:\n", label);
fprintf(f, " .globl X.%s", label);
linelen = 1000;
}
if (linelen >= 77) {
linelen = fprintf(f, "\n .byte %u", jit_code[i]);
} else {
linelen += fprintf(f, ",%u", jit_code[i]);
}
}
fputs("\n", f);
fclose(f);
} else {
fprintf(stderr, "Couldn't open /tmp/upb-jit-code.s for writing\n");
abort();
}
// TODO: racy
if (system("gcc -shared -o /tmp/upb-jit-code.so /tmp/upb-jit-code.s") != 0) {
fprintf(stderr, "Error compiling upb-jit-code.s\n");
abort();
}
jc->group->dl = dlopen("/tmp/upb-jit-code.so", RTLD_LAZY);
if (!jc->group->dl) {
fprintf(stderr, "Couldn't dlopen(): %s\n", dlerror());
abort();
}
munmap(jc->group->jit_code, jc->group->jit_size);
jc->group->jit_code = dlsym(jc->group->dl, "X.enterjit");
if (!jc->group->jit_code) {
fprintf(stderr, "Couldn't find enterjit sym\n");
abort();
}
upb_inttable_uninit(&mclabels);
}
#endif
void upb_pbdecoder_jit(mgroup *group) {
group->debug_info = NULL;
group->dl = NULL;
assert(group->bytecode);
jitcompiler *jc = newjitcompiler(group);
emit_static_asm(jc);
jitbytecode(jc);
int dasm_status = dasm_link(jc, &jc->group->jit_size);
if (dasm_status != DASM_S_OK) {
fprintf(stderr, "DynASM error; returned status: 0x%08x\n", dasm_status);
abort();
}
char *jit_code = mmap(NULL, jc->group->jit_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
dasm_encode(jc, jit_code);
mprotect(jit_code, jc->group->jit_size, PROT_EXEC | PROT_READ);
upb_reg_jit_gdb(jc);
jc->group->jit_code = (upb_string_handlerfunc *)jit_code;
#ifdef UPB_JIT_LOAD_SO
load_so(jc);
#endif
patchdispatch(jc);
freejitcompiler(jc);
// Now the bytecode is no longer needed.
free(group->bytecode);
group->bytecode = NULL;
}
void upb_pbdecoder_freejit(mgroup *group) {
if (!group->jit_code) return;
if (group->dl) {
#ifdef UPB_JIT_LOAD_SO
dlclose(group->dl);
#endif
} else {
munmap(group->jit_code, group->jit_size);
}
free(group->debug_info);
// TODO: unregister GDB JIT interface.
}
// To debug JIT-ted code with GDB we need to tell GDB about the JIT-ted code
// at runtime. GDB 7.x+ has defined an interface for doing this, and these
// structure/function defintions are copied out of gdb/jit.h
//
// We need to give GDB an ELF file at runtime describing the symbols we have
// generated. To avoid implementing the ELF format, we generate an ELF file
// at compile-time and compile it in as a character string. We can replace
// a few key constants (address of JIT-ted function and its size) by looking
// for a few magic numbers and doing a dumb string replacement.
//
// Unfortunately this approach is showing its limits; we can only define one
// symbol, and this approach only works with GDB. The .so approach above is
// more reliable.
#ifndef __APPLE__
const unsigned char upb_jit_debug_elf_file[] = {
#include "upb/pb/jit_debug_elf_file.h"
};
typedef enum {
GDB_JIT_NOACTION = 0,
GDB_JIT_REGISTER,
GDB_JIT_UNREGISTER
} jit_actions_t;
typedef struct gdb_jit_entry {
struct gdb_jit_entry *next_entry;
struct gdb_jit_entry *prev_entry;
const char *symfile_addr;
uint64_t symfile_size;
} gdb_jit_entry;
typedef struct {
uint32_t version;
uint32_t action_flag;
gdb_jit_entry *relevant_entry;
gdb_jit_entry *first_entry;
} gdb_jit_descriptor;
gdb_jit_descriptor __jit_debug_descriptor = {1, GDB_JIT_NOACTION, NULL, NULL};
void __attribute__((noinline)) __jit_debug_register_code() {
__asm__ __volatile__("");
}
static void upb_reg_jit_gdb(jitcompiler *jc) {
// Create debug info.
size_t elf_len = sizeof(upb_jit_debug_elf_file);
jc->group->debug_info = malloc(elf_len);
memcpy(jc->group->debug_info, upb_jit_debug_elf_file, elf_len);
uint64_t *p = (void *)jc->group->debug_info;
for (; (void *)(p + 1) <= (void *)jc->group->debug_info + elf_len; ++p) {
if (*p == 0x12345678) {
*p = (uintptr_t)jc->group->jit_code;
}
if (*p == 0x321) {
*p = jc->group->jit_size;
}
}
// Register the JIT-ted code with GDB.
gdb_jit_entry *e = malloc(sizeof(gdb_jit_entry));
e->next_entry = __jit_debug_descriptor.first_entry;
e->prev_entry = NULL;
if (e->next_entry) e->next_entry->prev_entry = e;
e->symfile_addr = jc->group->debug_info;
e->symfile_size = elf_len;
__jit_debug_descriptor.first_entry = e;
__jit_debug_descriptor.relevant_entry = e;
__jit_debug_descriptor.action_flag = GDB_JIT_REGISTER;
__jit_debug_register_code();
}
#else
static void upb_reg_jit_gdb(jitcompiler *jc) { (void)jc; }
#endif
|