1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
|
--[[
Code to generate a C API in:
foo.proto -> foo.upb.h
foo.upb.c
This code is evolving very quickly and so there are lots of little things
that aren't perfect right now. As it settles a little more, the code
quality should improve.
--]]
local upb = require "upb"
local dump_cinit = require "dump_cinit"
local export = {}
local typemap = {
[upb.TYPE_BOOL] = "bool",
[upb.TYPE_FLOAT] = "float",
[upb.TYPE_INT32] = "int32_t",
[upb.TYPE_UINT32] = "uint32_t",
[upb.TYPE_DOUBLE] = "double",
[upb.TYPE_INT64] = "int64_t",
[upb.TYPE_UINT64] = "uint64_t",
[upb.TYPE_STRING] = "upb_stringview",
[upb.TYPE_BYTES] = "upb_stringview",
}
function strip_proto(filename)
return string.gsub(filename, '%.proto$','')
end
local function join(...)
return table.concat({...}, ".")
end
local function to_cident(...)
return string.gsub(join(...), "[%./]", "_")
end
local function to_preproc(...)
return string.upper(to_cident(...))
end
-- Strips away last path element, ie:
-- foo.Bar.Baz -> foo.Bar
local function remove_name(name)
local package_end = 0
for i=1,string.len(name) do
if string.byte(name, i) == string.byte(".", 1) then
package_end = i - 1
end
end
return string.sub(name, 1, package_end)
end
local function enum_value_symbol(enumdef, name)
return to_cident(remove_name(enumdef:full_name())) .. "_" .. name
end
local function dump_enum_vals(enumdef, append)
local enum_vals = {}
for k, v in enumdef:values() do
enum_vals[#enum_vals + 1] = {k, v}
end
table.sort(enum_vals, function(a, b) return a[2] < b[2] end)
-- protobuf convention is that enum values are scoped at the level of the
-- enum itself, to follow C++. Ie, if you have the enum:
-- message Foo {
-- enum E {
-- VAL1 = 1;
-- VAL2 = 2;
-- }
-- }
--
-- The name of VAL1 is Foo.VAL1, not Foo.E.VAL1.
--
-- This seems a bit sketchy, but people often name their enum values
-- accordingly, ie:
--
-- enum Foo {
-- FOO_VAL1 = 1;
-- FOO_VAL2 = 2;
-- }
--
-- So if we don't respect this also, we end up with constants that look like:
--
-- GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_TYPE_TYPE_DOUBLE = 1
--
-- (notice the duplicated "TYPE").
local cident = to_cident(remove_name(enumdef:full_name()))
for i, pair in ipairs(enum_vals) do
k, v = pair[1], pair[2]
append(' %s = %d', enum_value_symbol(enumdef, k), v)
if i == #enum_vals then
append('\n')
else
append(',\n')
end
end
end
local function field_default(field)
if field:type() == upb.TYPE_MESSAGE then
return "NULL"
elseif field:type() == upb.TYPE_STRING or
field:type() == upb.TYPE_BYTES then
local default = field:default() or ""
return string.format('upb_stringview_make("%s", strlen("%s"))', field:default(), field:default())
elseif field:type() == upb.TYPE_ENUM then
return enum_value_symbol(field:subdef(), field:default())
else
return field:default();
end
end
local function ctype(field, const)
if const then
const = "const "
else
const = ""
end
if field:label() == upb.LABEL_REPEATED then
return const .. "upb_array*"
elseif field:type() == upb.TYPE_MESSAGE then
if field:containing_type():file() == field:subdef():file() then
return const .. to_cident(field:subdef():full_name()) .. "*"
else
return const .. "struct " .. to_cident(field:subdef():full_name()) .. "*"
end
elseif field:type() == upb.TYPE_ENUM then
return to_cident(field:subdef():full_name())
else
return typemap[field:type()] or "void*"
end
end
local function emit_file_warning(filedef, append)
append('/* This file was generated by upbc (the upb compiler) from the input\n')
append(' * file:\n')
append(' *\n')
append(' * %s\n', filedef:name())
append(' *\n')
append(' * Do not edit -- your changes will be discarded when the file is\n')
append(' * regenerated. */\n\n')
end
local function field_layout_rank(field)
-- Order:
-- 1, 2, 3. primitive fields (8, 4, 1 byte)
-- 4. string fields
-- 5. submessage fields
-- 6. repeated fields
--
-- This has the following nice properties:
--
-- 1. padding alignment is (nearly) minimized.
-- 2. fields that might have defaults (1-4) are segregated
-- from fields that are always zero-initialized (5-7).
--
-- We skip oneof fields, because they are emitted in a separate pass.
local rank
if field:containing_oneof() then
rank = 100 -- These go last (actually we skip them).
elseif field:label() == upb.LABEL_REPEATED then
rank = 6
elseif field:type() == upb.TYPE_MESSAGE then
rank = 5
elseif field:type() == upb.TYPE_STRING or field:type() == upb.TYPE_BYTES then
rank = 4
elseif field:type() == upb.TYPE_BOOL then
rank = 3
elseif field:type() == upb.TYPE_FLOAT or
field:type() == upb.TYPE_INT32 or
field:type() == upb.TYPE_UINT32 then
rank = 2
else
rank = 1
end
-- Break ties with field number.
return (rank * 2^29) + field:number()
end
local function has_hasbit(field)
if field:containing_type():file():syntax() == upb.SYNTAX_PROTO2 then
return field:label() ~= upb.LABEL_REPEATED and not field:containing_oneof()
else
return false
end
end
local function write_h_file(filedef, append)
emit_file_warning(filedef, append)
local basename_preproc = to_preproc(filedef:name())
append('#ifndef %s_UPB_H_\n', basename_preproc)
append('#define %s_UPB_H_\n\n', basename_preproc)
append('#include "upb/msg.h"\n\n')
append('UPB_BEGIN_EXTERN_C\n\n')
for msg in filedef:defs(upb.DEF_MSG) do
-- TODO(haberman): forward declare C++ type names so we can use
-- UPB_DECLARE_TYPE().
local msgname = to_cident(msg:full_name())
append('struct %s;\n', msgname)
append('typedef struct %s %s;\n', msgname, msgname)
end
append("/* Enums */\n\n")
for _, def in ipairs(sorted_defs(filedef:defs(upb.DEF_ENUM))) do
local cident = to_cident(def:full_name())
append('typedef enum {\n')
dump_enum_vals(def, append)
append('} %s;\n\n', cident)
end
for msg in filedef:defs(upb.DEF_MSG) do
local msgname = to_cident(msg:full_name())
append('/* %s */\n', msgname)
append('extern const upb_msglayout_msginit_v1 %s_msginit;\n', msgname)
append('%s *%s_new(upb_env *env);\n', msgname, msgname)
append('%s *%s_parsenew(upb_stringview buf, upb_env *env);\n',
msgname, msgname)
append('char *%s_serialize(%s *msg, upb_env *env, size_t *len);\n',
msgname, msgname)
append('void %s_free(%s *msg, upb_env *env);\n', msgname, msgname)
append('\n')
append('/* getters. */\n')
local setters, get_setters = dump_cinit.str_appender()
for field in msg:fields() do
local fieldname = to_cident(field:name())
if field:type() == upb.TYPE_MESSAGE and
field:subdef():file() ~= filedef then
-- Forward declaration for message type declared in another file.
append('struct %s;\n', to_cident(field:subdef():full_name()))
end
append('%s %s_%s(const %s *msg);\n',
ctype(field, true), msgname, fieldname, msgname)
setters('void %s_set_%s(%s *msg, %s value);\n',
msgname, fieldname, msgname, ctype(field))
end
for oneof in msg:oneofs() do
local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name())
append('typedef enum {\n')
for field in oneof:fields() do
append(' %s = %d,\n', fullname .. "_" .. field:name(), field:number())
end
append(' %s_NOT_SET = 0,\n', fullname)
append('} %s_oneofcases;\n', fullname)
append('%s_oneofcases %s_case(const %s *msg);\n', fullname, fullname, msgname)
end
append('\n')
append('/* setters. */\n')
append(get_setters())
append('\n')
append('\n')
end
append('UPB_END_EXTERN_C')
append('\n')
append('\n')
append('#endif /* %s_UPB_H_ */\n', basename_preproc)
end
local function write_c_file(filedef, hfilename, append)
emit_file_warning(filedef, append)
append('#include <stddef.h>\n')
append('#include "upb/decode.h"\n')
append('#include "upb/encode.h"\n')
append('#include "upb/msg.h"\n')
append('#include "upb/upb.h"\n')
append('#include "%s"\n\n', hfilename)
for dep in filedef:dependencies() do
local outbase = strip_proto(dep:name())
append('#include "%s.upb.h"\n', outbase)
end
append('\n')
for msg in filedef:defs(upb.DEF_MSG) do
local msgname = to_cident(msg:full_name())
local fields_array_ref = "NULL"
local submsgs_array_ref = "NULL"
local oneofs_array_ref = "NULL"
local field_count = 0
local submsg_count = 0
local submsg_set = {}
local submsg_indexes = {}
local hasbit_count = 0
local hasbit_indexes = {}
local oneof_count = 0
local oneof_indexes = {}
-- Create a layout order for oneofs.
local oneofs_layout_order = {}
for oneof in msg:oneofs() do
table.insert(oneofs_layout_order, oneof)
end
table.sort(oneofs_layout_order, function(a, b)
return a:name() < b:name()
end)
for _, oneof in ipairs(oneofs_layout_order) do
oneof_indexes[oneof] = oneof_count
oneof_count = oneof_count + 1
end
-- Create a layout order for fields. We use this order for the struct and
-- for offsets, but our list of fields we keep in field number order.
local fields_layout_order = {}
for field in msg:fields() do
table.insert(fields_layout_order, field)
end
table.sort(fields_layout_order, function(a, b)
return field_layout_rank(a) < field_layout_rank(b)
end)
-- Another sorted array in field number order.
local fields_number_order = {}
for field in msg:fields() do
table.insert(fields_number_order, field)
end
table.sort(fields_number_order, function(a, b)
return a:number() < b:number()
end)
append('struct %s {\n', msgname)
-- Non-oneof fields.
for _, field in ipairs(fields_layout_order) do
field_count = field_count + 1
if field:type() == upb.TYPE_MESSAGE then
submsg_count = submsg_count + 1
submsg_set[field:subdef()] = true
end
if field:containing_oneof() then
-- Handled below.
else
if has_hasbit(field) then
hasbit_indexes[field] = hasbit_count
hasbit_count = hasbit_count + 1
end
append(' %s %s;\n', ctype(field), field:name())
end
end
-- Oneof fields.
for oneof in msg:oneofs() do
local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name())
append(' union {\n')
for field in oneof:fields() do
append(' %s %s;\n', ctype(field), field:name())
end
append(' } %s;\n', oneof:name())
append(' %s_oneofcases %s_case;\n', fullname, oneof:name())
end
append('};\n\n')
if oneof_count > 0 then
local oneofs_array_name = msgname .. "_oneofs"
oneofs_array_ref = "&" .. oneofs_array_name .. "[0]"
append('static const upb_msglayout_oneofinit_v1 %s[%s] = {\n',
oneofs_array_name, oneof_count)
for _, oneof in ipairs(oneofs_layout_order) do
append(' {offsetof(%s, %s), offsetof(%s, %s_case)},\n',
msgname, oneof:name(), msgname, oneof:name())
end
append('};\n\n')
end
if submsg_count > 0 then
-- TODO(haberman): could save a little bit of space by only generating a
-- "submsgs" array for every strongly-connected component.
local submsgs_array_name = msgname .. "_submsgs"
submsgs_array_ref = "&" .. submsgs_array_name .. "[0]"
append('static const upb_msglayout_msginit_v1 *const %s[%s] = {\n',
submsgs_array_name, submsg_count)
-- Create a deterministically-sorted array of submessage entries.
local submsg_array = {}
for k, v in pairs(submsg_set) do
table.insert(submsg_array, k)
end
table.sort(submsg_array, function(a, b)
return a:full_name() < b:full_name()
end)
for i, submsg in ipairs(submsg_array) do
append(' &%s_msginit,\n', to_cident(submsg:full_name()))
submsg_indexes[submsg] = i - 1
end
append('};\n\n')
end
if field_count > 0 then
local fields_array_name = msgname .. "__fields"
fields_array_ref = "&" .. fields_array_name .. "[0]"
append('static const upb_msglayout_fieldinit_v1 %s[%s] = {\n',
fields_array_name, field_count)
for _, field in ipairs(fields_number_order) do
local submsg_index = "UPB_NO_SUBMSG"
local oneof_index = "UPB_NOT_IN_ONEOF"
if field:type() == upb.TYPE_MESSAGE then
submsg_index = submsg_indexes[field:subdef()]
end
if field:containing_oneof() then
oneof_index = oneof_indexes[field:containing_oneof()]
end
append(' {%s, offsetof(%s, %s), %s, %s, %s, %s, %s},\n',
field:number(),
msgname,
(field:containing_oneof() and field:containing_oneof():name()) or field:name(),
hasbit_indexes[field] or "UPB_NO_HASBIT",
oneof_index,
submsg_index,
field:descriptor_type(),
field:label())
end
append('};\n\n')
end
append('const upb_msglayout_msginit_v1 %s_msginit = {\n', msgname)
append(' %s,\n', submsgs_array_ref)
append(' %s,\n', fields_array_ref)
append(' %s,\n', oneofs_array_ref)
append(' NULL, /* TODO. default_msg */\n')
append(' UPB_ALIGNED_SIZEOF(%s), %s, %s, %s, %s\n',
msgname, field_count,
0, -- TODO: oneof_count
'false', -- TODO: extendable
msg:file():syntax() == upb.SYNTAX_PROTO2
)
append('};\n\n')
append('%s *%s_new(upb_env *env) {\n', msgname, msgname)
append(' %s *msg = upb_env_malloc(env, sizeof(*msg));\n',
msgname)
append(' memset(msg, 0, sizeof(*msg)); /* TODO: defaults */\n')
append(' return msg;\n')
append('}\n')
append('%s *%s_parsenew(upb_stringview buf, upb_env *env) {\n',
msgname, msgname)
append(' %s *msg = %s_new(env);\n', msgname, msgname)
append(' if (upb_decode(buf, msg, &%s_msginit, env)) {\n', msgname)
append(' return msg;\n')
append(' } else {\n')
append(' return NULL;\n')
append(' }\n')
append('}\n')
append('char *%s_serialize(%s *msg, upb_env *env, size_t *size) {\n',
msgname, msgname)
append(' return upb_encode(msg, &%s_msginit, env, size);\n', msgname)
append('}\n')
for field in msg:fields() do
append('%s %s_%s(const %s *msg) {\n',
ctype(field, true), msgname, field:name(), msgname);
if field:containing_oneof() then
local oneof = field:containing_oneof()
append(' return msg->%s_case == %s ? msg->%s.%s : %s;\n',
oneof:name(), field:number(), oneof:name(), field:name(),
field_default(field))
else
append(' return msg->%s;\n', field:name())
end
append('}\n')
append('void %s_set_%s(%s *msg, %s value) {\n',
msgname, field:name(), msgname, ctype(field));
if field:containing_oneof() then
local oneof = field:containing_oneof()
append(' msg->%s.%s = value;\n', oneof:name(), field:name())
append(' msg->%s_case = %s;\n', oneof:name(), field:number())
else
append(' msg->%s = value;\n', field:name())
end
append('}\n')
end
for oneof in msg:oneofs() do
local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name())
append('%s_oneofcases %s_case(const %s *msg) {\n', fullname, fullname, msgname)
append(' return msg->%s_case;\n', oneof:name())
append('}\n')
end
end
end
function export.write_gencode(filedef, hfilename, append_h, append_c)
write_h_file(filedef, append_h)
write_c_file(filedef, hfilename, append_c)
end
return export
|