From fa82e4fbf065bb474b0f87023f66edffb8a5b850 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Wed, 17 Aug 2011 08:15:57 -0700 Subject: Benchmark JIT vs no JIT without forcing a whole-project recompile. --- Makefile | 98 ++++++++++++++++++++++++------------ benchmarks/parsestream.upb.c | 87 ++++++++++++++++++++++++++++++++ benchmarks/parsestream.upb_table.c | 86 ------------------------------- benchmarks/parsetostruct.upb.c | 79 +++++++++++++++++++++++++++++ benchmarks/parsetostruct.upb_table.c | 75 --------------------------- perf-tests.sh | 7 +-- 6 files changed, 235 insertions(+), 197 deletions(-) create mode 100644 benchmarks/parsestream.upb.c delete mode 100644 benchmarks/parsestream.upb_table.c create mode 100644 benchmarks/parsetostruct.upb.c delete mode 100644 benchmarks/parsetostruct.upb_table.c diff --git a/Makefile b/Makefile index 0a8f62a..b0e3d9a 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,13 @@ ifeq (, $(findstring -O, $(USER_CFLAGS))) DEF_OPT = -Os endif +ifneq (, $(findstring DUPB_USE_JIT_X64, $(USER_CFLAGS))) + ifeq (, $(findstring DNDEBUG, $(USER_CFLAGS))) + $(error "JIT only works with -DNDEBUG enabled!") + endif + USE_JIT=true +endif + # Basic compiler/flag setup. CC=gcc CXX=g++ @@ -91,8 +98,8 @@ OTHERSRC=upb/pb/encoder.c BENCHMARKS_SRC= \ benchmarks/main.c \ - benchmarks/parsestream.upb_table.c \ - benchmarks/parsetostruct.upb_table.c + benchmarks/parsestream.upb.c \ + benchmarks/parsetostruct.upb.c TESTS_SRC= \ tests/test_decoder.c \ @@ -135,11 +142,8 @@ lib: $(LIBUPB) OBJ=$(patsubst %.c,%.o,$(SRC)) PICOBJ=$(patsubst %.c,%.lo,$(SRC)) -ifneq (, $(findstring DUPB_USE_JIT_X64, $(USER_CFLAGS))) +ifdef USE_JIT upb/pb/decoder.o upb/pb/decoder.lo: upb/pb/decoder_x86.h - ifeq (, $(findstring DNDEBUG, $(USER_CFLAGS))) - $(error "JIT only works with -DNDEBUG enabled!") - endif endif $(LIBUPB): $(OBJ) $(E) AR $(LIBUPB) @@ -277,8 +281,16 @@ tests/tests: upb/libupb.a # Benchmarks UPB_BENCHMARKS=benchmarks/b.parsestream_googlemessage1.upb_table \ benchmarks/b.parsestream_googlemessage2.upb_table \ - benchmarks/b.parsetostruct_googlemessage1.upb_table_byref \ - benchmarks/b.parsetostruct_googlemessage2.upb_table_byref + benchmarks/b.parsetostruct_googlemessage1.upb_table_byval \ + benchmarks/b.parsetostruct_googlemessage2.upb_table_byval \ + +ifdef USE_JIT +UPB_BENCHMARKS += \ + benchmarks/b.parsestream_googlemessage1.upb_jit \ + benchmarks/b.parsestream_googlemessage2.upb_jit \ + benchmarks/b.parsetostruct_googlemessage1.upb_jit_byval \ + benchmarks/b.parsetostruct_googlemessage2.upb_jit_byval +endif BENCHMARKS=$(UPB_BENCHMARKS) \ benchmarks/b.parsetostruct_googlemessage1.proto2_table \ @@ -300,51 +312,71 @@ benchmarks/google_messages.pb.cc: benchmarks/google_messages.proto protoc benchmarks/google_messages.proto --cpp_out=. benchmarks/b.parsetostruct_googlemessage1.upb_table_byval \ -benchmarks/b.parsetostruct_googlemessage1.upb_table_byref \ -benchmarks/b.parsetostruct_googlemessage2.upb_table_byval \ -benchmarks/b.parsetostruct_googlemessage2.upb_table_byref: \ - benchmarks/parsetostruct.upb_table.c $(LIBUPB) benchmarks/google_messages.proto.pb - $(E) 'CC benchmarks/parsetostruct.upb_table.c (benchmarks.SpeedMessage1, byval)' +benchmarks/b.parsetostruct_googlemessage2.upb_table_byval: \ + benchmarks/parsetostruct.upb.c $(LIBUPB) benchmarks/google_messages.proto.pb + $(E) 'CC benchmarks/parsetostruct.upb.c (benchmarks.SpeedMessage1, byval, nojit)' $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsetostruct_googlemessage1.upb_table_byval $< \ -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \ -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ -DMESSAGE_FILE=\"google_message1.dat\" \ - -DBYREF=false $(LIBUPB) - $(E) 'CC benchmarks/parsetostruct.upb_table.c (benchmarks.SpeedMessage1, byref)' - $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsetostruct_googlemessage1.upb_table_byref $< \ - -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \ - -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ - -DMESSAGE_FILE=\"google_message1.dat\" \ - -DBYREF=true $(LIBUPB) - $(E) 'CC benchmarks/parsetostruct.upb_table.c (benchmarks.SpeedMessage2, byval)' + -DBYREF=false -DJIT=false $(LIBUPB) + $(E) 'CC benchmarks/parsetostruct.upb.c (benchmarks.SpeedMessage2, byref, nojit)' $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsetostruct_googlemessage2.upb_table_byval $< \ -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \ -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ -DMESSAGE_FILE=\"google_message2.dat\" \ - -DBYREF=false $(LIBUPB) - $(E) 'CC benchmarks/parsetostruct.upb_table.c (benchmarks.SpeedMessage2, byref)' - $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsetostruct_googlemessage2.upb_table_byref $< \ - -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \ - -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ - -DMESSAGE_FILE=\"google_message2.dat\" \ - -DBYREF=true $(LIBUPB) + -DBYREF=false -DJIT=false $(LIBUPB) benchmarks/b.parsestream_googlemessage1.upb_table \ benchmarks/b.parsestream_googlemessage2.upb_table: \ - benchmarks/parsestream.upb_table.c $(LIBUPB) benchmarks/google_messages.proto.pb - $(E) 'CC benchmarks/parsestream.upb_table.c (benchmarks.SpeedMessage1)' + benchmarks/parsestream.upb.c $(LIBUPB) benchmarks/google_messages.proto.pb + $(E) 'CC benchmarks/parsestream.upb.c (benchmarks.SpeedMessage1, nojit)' $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsestream_googlemessage1.upb_table $< \ -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \ -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ - -DMESSAGE_FILE=\"google_message1.dat\" \ + -DMESSAGE_FILE=\"google_message1.dat\" -DJIT=false \ $(LIBUPB) - $(E) 'CC benchmarks/parsestream.upb_table.c (benchmarks.SpeedMessage2)' + $(E) 'CC benchmarks/parsestream.upb.c (benchmarks.SpeedMessage2, nojit)' $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsestream_googlemessage2.upb_table $< \ -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \ -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ - -DMESSAGE_FILE=\"google_message2.dat\" \ + -DMESSAGE_FILE=\"google_message2.dat\" -DJIT=false \ $(LIBUPB) +ifdef USE_JIT +benchmarks/b.parsetostruct_googlemessage1.upb_jit_byval \ +benchmarks/b.parsetostruct_googlemessage2.upb_jit_byval: \ + benchmarks/parsetostruct.upb.c $(LIBUPB) benchmarks/google_messages.proto.pb + $(E) 'CC benchmarks/parsetostruct.upb.c (benchmarks.SpeedMessage1, byref, jit)' + $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsetostruct_googlemessage1.upb_jit_byval $< \ + -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message1.dat\" -DJIT=true \ + -DBYREF=true -DJIT=true $(LIBUPB) + $(E) 'CC benchmarks/parsetostruct.upb.c (benchmarks.SpeedMessage2, byval, jit)' + $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsetostruct_googlemessage2.upb_jit_byval $< \ + -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message2.dat\" -DJIT=true \ + -DBYREF=false -DJIT=true $(LIBUPB) + +benchmarks/b.parsestream_googlemessage1.upb_jit \ +benchmarks/b.parsestream_googlemessage2.upb_jit: \ + benchmarks/parsestream.upb.c $(LIBUPB) benchmarks/google_messages.proto.pb + $(E) 'CC benchmarks/parsestream.upb.c (benchmarks.SpeedMessage1, jit)' + $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsestream_googlemessage1.upb_jit $< \ + -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message1.dat\" -DJIT=true \ + $(LIBUPB) + $(E) 'CC benchmarks/parsestream.upb.c (benchmarks.SpeedMessage2, jit)' + $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o benchmarks/b.parsestream_googlemessage2.upb_jit $< \ + -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \ + -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \ + -DMESSAGE_FILE=\"google_message2.dat\" -DJIT=true \ + $(LIBUPB) +endif + benchmarks/b.parsetostruct_googlemessage1.proto2_table \ benchmarks/b.parsetostruct_googlemessage2.proto2_table: \ benchmarks/parsetostruct.proto2_table.cc benchmarks/google_messages.pb.cc diff --git a/benchmarks/parsestream.upb.c b/benchmarks/parsestream.upb.c new file mode 100644 index 0000000..37ccb42 --- /dev/null +++ b/benchmarks/parsestream.upb.c @@ -0,0 +1,87 @@ + +#include "main.c" + +#include +#include "upb/bytestream.h" +#include "upb/def.h" +#include "upb/pb/decoder.h" +#include "upb/pb/glue.h" + +static char *input_str; +static size_t input_len; +static upb_msgdef *def; +static upb_decoder decoder; +static upb_stringsrc stringsrc; + +static upb_sflow_t startsubmsg(void *_m, upb_value fval) { + (void)_m; + (void)fval; + return UPB_CONTINUE_WITH(NULL); +} + +static upb_flow_t value(void *closure, upb_value fval, upb_value val) { + (void)closure; + (void)fval; + (void)val; + return UPB_CONTINUE; +} + +static bool initialize() +{ + // Initialize upb state, decode descriptor. + upb_status status = UPB_STATUS_INIT; + upb_symtab *s = upb_symtab_new(); + upb_read_descriptorfile(s, MESSAGE_DESCRIPTOR_FILE, &status); + if(!upb_ok(&status)) { + upb_status_print(&status, stderr); + return false; + } + + def = upb_dyncast_msgdef(upb_symtab_lookup(s, MESSAGE_NAME)); + if(!def) { + fprintf(stderr, "Error finding symbol '%s'.\n", MESSAGE_NAME); + return false; + } + upb_symtab_unref(s); + + // Read the message data itself. + input_str = upb_readfile(MESSAGE_FILE, &input_len); + if(input_str == NULL) { + fprintf(stderr, "Error reading " MESSAGE_FILE "\n"); + return false; + } + + upb_handlers *handlers = upb_handlers_new(); + if (!JIT) handlers->should_jit = false; + // Cause all messages to be read, but do nothing when they are. + upb_handlerset hset = {NULL, NULL, value, startsubmsg, NULL, NULL, NULL}; + upb_handlers_reghandlerset(handlers, def, &hset); + upb_decoder_initforhandlers(&decoder, handlers); + upb_handlers_unref(handlers); + upb_stringsrc_init(&stringsrc); + return true; +} + +static void cleanup() +{ + free(input_str); + upb_def_unref(UPB_UPCAST(def)); + upb_decoder_uninit(&decoder); + upb_stringsrc_uninit(&stringsrc); +} + +static size_t run(int i) +{ + (void)i; + upb_status status = UPB_STATUS_INIT; + upb_stringsrc_reset(&stringsrc, input_str, input_len); + upb_decoder_reset(&decoder, upb_stringsrc_bytesrc(&stringsrc), 0, UINT64_MAX, NULL); + upb_decoder_decode(&decoder, &status); + if(!upb_ok(&status)) goto err; + return input_len; + +err: + fprintf(stderr, "Decode error: "); + upb_status_print(&status, stderr); + return 0; +} diff --git a/benchmarks/parsestream.upb_table.c b/benchmarks/parsestream.upb_table.c deleted file mode 100644 index a4022b1..0000000 --- a/benchmarks/parsestream.upb_table.c +++ /dev/null @@ -1,86 +0,0 @@ - -#include "main.c" - -#include -#include "upb/bytestream.h" -#include "upb/def.h" -#include "upb/pb/decoder.h" -#include "upb/pb/glue.h" - -static char *input_str; -static size_t input_len; -static upb_msgdef *def; -static upb_decoder decoder; -static upb_stringsrc stringsrc; - -static upb_sflow_t startsubmsg(void *_m, upb_value fval) { - (void)_m; - (void)fval; - return UPB_CONTINUE_WITH(NULL); -} - -static upb_flow_t value(void *closure, upb_value fval, upb_value val) { - (void)closure; - (void)fval; - (void)val; - return UPB_CONTINUE; -} - -static bool initialize() -{ - // Initialize upb state, decode descriptor. - upb_status status = UPB_STATUS_INIT; - upb_symtab *s = upb_symtab_new(); - upb_read_descriptorfile(s, MESSAGE_DESCRIPTOR_FILE, &status); - if(!upb_ok(&status)) { - upb_status_print(&status, stderr); - return false; - } - - def = upb_dyncast_msgdef(upb_symtab_lookup(s, MESSAGE_NAME)); - if(!def) { - fprintf(stderr, "Error finding symbol '%s'.\n", MESSAGE_NAME); - return false; - } - upb_symtab_unref(s); - - // Read the message data itself. - input_str = upb_readfile(MESSAGE_FILE, &input_len); - if(input_str == NULL) { - fprintf(stderr, "Error reading " MESSAGE_FILE "\n"); - return false; - } - - upb_handlers *handlers = upb_handlers_new(); - // Cause all messages to be read, but do nothing when they are. - upb_handlerset hset = {NULL, NULL, value, startsubmsg, NULL, NULL, NULL}; - upb_handlers_reghandlerset(handlers, def, &hset); - upb_decoder_initforhandlers(&decoder, handlers); - upb_handlers_unref(handlers); - upb_stringsrc_init(&stringsrc); - return true; -} - -static void cleanup() -{ - free(input_str); - upb_def_unref(UPB_UPCAST(def)); - upb_decoder_uninit(&decoder); - upb_stringsrc_uninit(&stringsrc); -} - -static size_t run(int i) -{ - (void)i; - upb_status status = UPB_STATUS_INIT; - upb_stringsrc_reset(&stringsrc, input_str, input_len); - upb_decoder_reset(&decoder, upb_stringsrc_bytesrc(&stringsrc), 0, UINT64_MAX, NULL); - upb_decoder_decode(&decoder, &status); - if(!upb_ok(&status)) goto err; - return input_len; - -err: - fprintf(stderr, "Decode error: "); - upb_status_print(&status, stderr); - return 0; -} diff --git a/benchmarks/parsetostruct.upb.c b/benchmarks/parsetostruct.upb.c new file mode 100644 index 0000000..a436fd4 --- /dev/null +++ b/benchmarks/parsetostruct.upb.c @@ -0,0 +1,79 @@ + +#include "main.c" + +#include "upb/bytestream.h" +#include "upb/def.h" +#include "upb/msg.h" +#include "upb/pb/decoder.h" +#include "upb/pb/glue.h" + +static upb_msgdef *def; +static size_t len; +static void *msg; +static upb_stringsrc strsrc; +static upb_decoder d; + +static bool initialize() +{ + // Initialize upb state, decode descriptor. + upb_status status = UPB_STATUS_INIT; + upb_symtab *s = upb_symtab_new(); + upb_read_descriptorfile(s, MESSAGE_DESCRIPTOR_FILE, &status); + if(!upb_ok(&status)) { + upb_status_print(&status, stderr); + return false; + } + + def = upb_dyncast_msgdef(upb_symtab_lookup(s, MESSAGE_NAME)); + if(!def) { + fprintf(stderr, "Error finding symbol '%s'.\n", MESSAGE_NAME); + return false; + } + upb_symtab_unref(s); + + // Read the message data itself. + char *str = upb_readfile(MESSAGE_FILE, &len); + if(str == NULL) { + fprintf(stderr, "Error reading " MESSAGE_FILE "\n"); + return false; + } + upb_status_uninit(&status); + msg = upb_stdmsg_new(def); + + upb_stringsrc_init(&strsrc); + upb_stringsrc_reset(&strsrc, str, len); + upb_handlers *h = upb_handlers_new(); + upb_accessors_reghandlers(h, def); + if (!JIT) h->should_jit = false; + upb_decoder_initforhandlers(&d, h); + upb_handlers_unref(h); + + if (!BYREF) { + // TODO: use byref/byval accessors. + } + return true; +} + +static void cleanup() +{ + upb_stdmsg_free(msg, def); + upb_def_unref(UPB_UPCAST(def)); + upb_stringsrc_uninit(&strsrc); + upb_decoder_uninit(&d); +} + +static size_t run(int i) +{ + (void)i; + upb_status status = UPB_STATUS_INIT; + upb_msg_clear(msg, def); + upb_decoder_reset(&d, upb_stringsrc_bytesrc(&strsrc), 0, UINT64_MAX, msg); + upb_decoder_decode(&d, &status); + if(!upb_ok(&status)) goto err; + return len; + +err: + fprintf(stderr, "Decode error: "); + upb_status_print(&status, stderr); + return 0; +} diff --git a/benchmarks/parsetostruct.upb_table.c b/benchmarks/parsetostruct.upb_table.c deleted file mode 100644 index 88f355d..0000000 --- a/benchmarks/parsetostruct.upb_table.c +++ /dev/null @@ -1,75 +0,0 @@ - -#include "main.c" - -#include "upb/bytestream.h" -#include "upb/def.h" -#include "upb/msg.h" -#include "upb/pb/decoder.h" -#include "upb/pb/glue.h" - -static upb_msgdef *def; -static size_t len; -static void *msg; -static upb_stringsrc strsrc; -static upb_decoder d; - -static bool initialize() -{ - // Initialize upb state, decode descriptor. - upb_status status = UPB_STATUS_INIT; - upb_symtab *s = upb_symtab_new(); - upb_read_descriptorfile(s, MESSAGE_DESCRIPTOR_FILE, &status); - if(!upb_ok(&status)) { - upb_status_print(&status, stderr); - return false; - } - - def = upb_dyncast_msgdef(upb_symtab_lookup(s, MESSAGE_NAME)); - if(!def) { - fprintf(stderr, "Error finding symbol '%s'.\n", MESSAGE_NAME); - return false; - } - upb_symtab_unref(s); - - // Read the message data itself. - char *str = upb_readfile(MESSAGE_FILE, &len); - if(str == NULL) { - fprintf(stderr, "Error reading " MESSAGE_FILE "\n"); - return false; - } - upb_status_uninit(&status); - msg = upb_stdmsg_new(def); - - upb_stringsrc_init(&strsrc); - upb_stringsrc_reset(&strsrc, str, len); - upb_decoder_initformsgdef(&d, def); - - if (!BYREF) { - // TODO: use byref/byval accessors. - } - return true; -} - -static void cleanup() -{ - upb_stdmsg_free(msg, def); - upb_def_unref(UPB_UPCAST(def)); - upb_stringsrc_uninit(&strsrc); - upb_decoder_uninit(&d); -} - -static size_t run(int i) -{ - (void)i; - upb_status status = UPB_STATUS_INIT; - upb_msg_clear(msg, def); - upb_decoder_reset(&d, upb_stringsrc_bytesrc(&strsrc), 0, UINT64_MAX, msg); - upb_decoder_decode(&d, &status); - if(!upb_ok(&status)) goto err; - return len; - -err: - fprintf(stderr, "Decode error: "); - upb_status_print(&status, stderr); - return 0; -} diff --git a/perf-tests.sh b/perf-tests.sh index 118922d..7bdbee0 100755 --- a/perf-tests.sh +++ b/perf-tests.sh @@ -26,6 +26,7 @@ run_with_flags () { run_with_flags "-DNDEBUG -fomit-frame-pointer -m32" "omitfp32" #fi -run_with_flags "-DNDEBUG " "plain" -run_with_flags "-DNDEBUG -fomit-frame-pointer" "omitfp" -run_with_flags "-DNDEBUG -DUPB_USE_JIT_X64" "jit" +# Ideally we could test for x86-64 in deciding whether to compile with +# the JIT flag. +run_with_flags "-DNDEBUG -DUPB_USE_JIT_X64" "plain" +run_with_flags "-DNDEBUG -fomit-frame-pointer -DUPB_USE_JIT_X64" "omitfp" -- cgit v1.2.3