Integrated benchmarks into main Makefile.

8 files changed, 365 insertions, 0 deletions

diff --git a/benchmarks/getgraphs.py b/benchmarks/getgraphs.py
new file mode 100644
index 0000000..aacf6b2
--- /dev/null
+++ b/benchmarks/getgraphs.py
@@ -0,0 +1,35 @@
+
+import sys
+
+benchmarks = {}
+color_map = {'proto2_compiled': 'FF0000',
+             'proto2_table': 'FF00FF',
+             'upb_table_byref': '0000FF',
+             'upb_table_byval': '00FF00'}
+for line in sys.stdin:
+  name, val = line.split(': ')
+  components = name.split('_')
+  benchmark = '_'.join(components[1:3])
+  variant = '_'.join(components[3:])
+  if benchmark not in benchmarks:
+    benchmarks[benchmark] = []
+  benchmarks[benchmark].append((variant, int(val)))
+
+def encode(x):
+  digits = (range(ord("A"), ord("Z")+1) + range(ord("a"), ord("z")+1) +
+            range(ord("0"), ord("9")+1) + [ord("."), ord("-")])
+  return chr(digits[x / 64]) + chr(digits[x % 64])
+
+for benchmark, values in benchmarks.items():
+  def cmp(a, b):
+    return b[1] - a[1]
+  values.sort(cmp)
+  variants = [x[0] for x in values]
+  values = [x[1] for x in values]
+  scaling = 400
+  encoded_values = [encode((x * 4096 / scaling) - 1) for x in values]
+  legend = "chdl=%s" % ("|".join(variants))
+  colors = "chco=%s" % ("|".join([color_map[x] for x in variants]))
+  data = "chd=e:%s" % ("".join(encoded_values))
+  url = "http://chart.apis.google.com/chart?cht=bhs&chs=500x200&chtt=%s+(MB/s)&chxt=x&chxr=0,0,%d&%s" % (benchmark, scaling, "&".join([legend, data, colors]))
+  print url
diff --git a/benchmarks/google_message1.dat b/benchmarks/google_message1.dat
new file mode 100644
index 0000000..bc0f064
--- /dev/null
+++ b/benchmarks/google_message1.dat
diff --git a/benchmarks/google_message2.dat b/benchmarks/google_message2.dat
new file mode 100644
index 0000000..06c0944
--- /dev/null
+++ b/benchmarks/google_message2.dat
diff --git a/benchmarks/google_messages.proto b/benchmarks/google_messages.proto
new file mode 100644
index 0000000..17cace7
--- /dev/null
+++ b/benchmarks/google_messages.proto
@@ -0,0 +1,136 @@
+
+package benchmarks;
+
+option optimize_for = SPEED;
+
+message SpeedMessage1 {
+  required string field1 = 1;
+  optional string field9 = 9;
+  optional string field18 = 18;
+  optional bool field80 = 80 [default=false];
+  optional bool field81 = 81 [default=true];
+  required int32 field2 = 2;
+  required int32 field3 = 3;
+  optional int32 field280 = 280;
+  optional int32 field6 = 6 [default=0];
+  optional int64 field22 = 22;
+  optional string field4 = 4;
+  repeated fixed64 field5 = 5;
+  optional bool field59 = 59 [default=false];
+  optional string field7 = 7;
+  optional int32 field16 = 16;
+  optional int32 field130 = 130 [default=0];
+  optional bool field12 = 12 [default=true];
+  optional bool field17 = 17 [default=true];
+  optional bool field13 = 13 [default=true];
+  optional bool field14 = 14 [default=true];
+  optional int32 field104 = 104 [default=0];
+  optional int32 field100 = 100 [default=0];
+  optional int32 field101 = 101 [default=0];
+  optional string field102 = 102;
+  optional string field103 = 103;
+  optional int32 field29 = 29 [default=0];
+  optional bool field30 = 30 [default=false];
+  optional int32 field60 = 60 [default=-1];
+  optional int32 field271 = 271 [default=-1];
+  optional int32 field272 = 272 [default=-1];
+  optional int32 field150 = 150;
+  optional int32 field23 = 23 [default=0];
+  optional bool field24 = 24 [default=false];
+  optional int32 field25 = 25 [default=0];
+  optional SpeedMessage1SubMessage field15 = 15;
+  optional bool field78 = 78;
+  optional int32 field67 = 67 [default=0];
+  optional int32 field68 = 68;
+  optional int32 field128 = 128 [default=0];
+  optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"];
+  optional int32 field131 = 131 [default=0];
+}
+
+message SpeedMessage1SubMessage {
+  optional int32 field1 = 1 [default=0];
+  optional int32 field2 = 2 [default=0];
+  optional int32 field3 = 3 [default=0];
+  optional string field15 = 15;
+  optional bool field12 = 12 [default=true];
+  optional int64 field13 = 13;
+  optional int64 field14 = 14;
+  optional int32 field16 = 16;
+  optional int32 field19 = 19 [default=2];
+  optional bool field20  = 20 [default=true];
+  optional bool field28 = 28 [default=true];
+  optional fixed64 field21 = 21;
+  optional int32 field22 = 22;
+  optional bool field23 = 23 [ default=false ];
+  optional bool field206 = 206 [default=false];
+  optional fixed32 field203 = 203;
+  optional int32 field204 = 204;
+  optional string field205 = 205;
+  optional uint64 field207 = 207;
+  optional uint64 field300 = 300;
+}
+
+message SpeedMessage2 {
+  optional string field1 = 1;
+  optional int64 field3 = 3;
+  optional int64 field4 = 4;
+  optional int64 field30 = 30;
+  optional bool field75  = 75 [default=false];
+  optional string field6 = 6;
+  optional bytes field2 = 2;
+  optional int32 field21 = 21 [default=0];
+  optional int32 field71 = 71;
+  optional float field25 = 25;
+  optional int32 field109 = 109 [default=0];
+  optional int32 field210 = 210 [default=0];
+  optional int32 field211 = 211 [default=0];
+  optional int32 field212 = 212 [default=0];
+  optional int32 field213 = 213 [default=0];
+  optional int32 field216 = 216 [default=0];
+  optional int32 field217 = 217 [default=0];
+  optional int32 field218 = 218 [default=0];
+  optional int32 field220 = 220 [default=0];
+  optional int32 field221 = 221 [default=0];
+  optional float field222 = 222 [default=0.0];
+  optional int32 field63 = 63;
+
+  repeated group Group1 = 10 {
+    required float field11 = 11;
+    optional float field26 = 26;
+    optional string field12 = 12;
+    optional string field13 = 13;
+    repeated string field14 = 14;
+    required uint64 field15 = 15;
+    optional int32 field5 = 5;
+    optional string field27 = 27;
+    optional int32 field28 = 28;
+    optional string field29 = 29;
+    optional string field16 = 16;
+    repeated string field22 = 22;
+    repeated int32 field73 = 73;
+    optional int32 field20 = 20 [default=0];
+    optional string field24 = 24;
+    optional SpeedMessage2GroupedMessage field31 = 31;
+  }
+  repeated string field128 = 128;
+  optional int64 field131 = 131;
+  repeated string field127 = 127;
+  optional int32 field129 = 129;
+  repeated int64 field130 = 130;
+  optional bool field205 = 205 [default=false];
+  optional bool field206 = 206 [default=false];
+}
+
+message SpeedMessage2GroupedMessage {
+  optional float field1 = 1;
+  optional float field2 = 2;
+  optional float field3 = 3 [default=0.0];
+  optional bool field4 = 4;
+  optional bool field5 = 5;
+  optional bool field6 = 6 [default=true];
+  optional bool field7 = 7 [default=false];
+  optional float field8 = 8;
+  optional bool field9 = 9;
+  optional float field10 = 10;
+  optional int64 field11 = 11;
+}
diff --git a/benchmarks/main.c b/benchmarks/main.c
new file mode 100644
index 0000000..f66b4d0
--- /dev/null
+++ b/benchmarks/main.c
@@ -0,0 +1,46 @@
+
+#include <stdbool.h>
+#include <time.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+
+static bool initialize();
+static void cleanup();
+static size_t run();
+
+int main (int argc, char *argv[])
+{
+  (void)argc;
+
+  /* Change cwd to where the binary is. */
+  char *lastslash = strrchr(argv[0], '/');
+  char *progname = argv[0];
+  if(lastslash) {
+    *lastslash = '\0';
+    chdir(argv[0]);
+    *lastslash = '/';
+    progname = lastslash + 3;  /* "/b_" */
+  }
+
+  if(!initialize()) {
+    fprintf(stderr, "%s: failed to initialize\n", argv[0]);
+    return 1;
+  }
+
+  size_t total_bytes = 0;
+  clock_t before = clock();
+  for(int i = 0; true; i++) {
+    if((i & 0xFF) == 0 && (clock() - before > CLOCKS_PER_SEC)) break;
+    size_t bytes = run();
+    if(bytes == 0) {
+      fprintf(stderr, "%s: failed.\n", argv[0]);
+      return 2;
+    }
+    total_bytes += bytes;
+  }
+  double elapsed = ((double)clock() - before) / CLOCKS_PER_SEC;
+  printf("%s:%d\n", progname, (int)(total_bytes / elapsed / (1 << 20)));
+  cleanup();
+  return 0;
+}
diff --git a/benchmarks/parsetostruct.proto2_compiled.cc b/benchmarks/parsetostruct.proto2_compiled.cc
new file mode 100644
index 0000000..339e184
--- /dev/null
+++ b/benchmarks/parsetostruct.proto2_compiled.cc
@@ -0,0 +1,37 @@
+
+#include "main.c"
+#include MESSAGE_HFILE
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+
+static std::string str;
+MESSAGE_CIDENT msg;
+
+static bool initialize()
+{
+  /* Read the message data itself. */
+  std::ifstream stream(MESSAGE_FILE);
+  if(!stream.is_open()) {
+    fprintf(stderr, "Error opening " MESSAGE_FILE ".\n");
+    return false;
+  }
+  std::stringstream stringstream;
+  stringstream << stream.rdbuf();
+  str = stringstream.str();
+  return true;
+}
+
+static void cleanup()
+{
+}
+
+static size_t run()
+{
+  if(!msg.ParseFromString(str)) {
+    fprintf(stderr, "Error parsing with proto2.\n");
+    return 0;
+  }
+  return str.size();
+}
diff --git a/benchmarks/parsetostruct.proto2_table.cc b/benchmarks/parsetostruct.proto2_table.cc
new file mode 100644
index 0000000..815834e
--- /dev/null
+++ b/benchmarks/parsetostruct.proto2_table.cc
@@ -0,0 +1,44 @@
+
+#include "main.c"
+#include <google/protobuf/dynamic_message.h>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+#include MESSAGE_HFILE
+
+static std::string str;
+static google::protobuf::DynamicMessageFactory factory;
+static google::protobuf::Message *msg;
+
+static bool initialize()
+{
+  /* Read the message data itself. */
+  std::ifstream stream(MESSAGE_FILE);
+  if(!stream.is_open()) {
+    fprintf(stderr, "Error opening " MESSAGE_FILE ".\n");
+    return false;
+  }
+  std::stringstream stringstream;
+  stringstream << stream.rdbuf();
+  str = stringstream.str();
+
+  /* Create the DynamicMessage. */
+  const google::protobuf::Message *dynamic_msg_prototype =
+      factory.GetPrototype(MESSAGE_CIDENT::descriptor());
+  msg = dynamic_msg_prototype->New();
+  return true;
+}
+
+static void cleanup()
+{
+  delete msg;
+}
+
+static size_t run()
+{
+  if(!msg->ParseFromString(str)) {
+    fprintf(stderr, "Error parsing with proto2.\n");
+    return 0;
+  }
+  return str.size();
+}
diff --git a/benchmarks/parsetostruct.upb_table.c b/benchmarks/parsetostruct.upb_table.c
new file mode 100644
index 0000000..f93a6a7
--- /dev/null
+++ b/benchmarks/parsetostruct.upb_table.c
@@ -0,0 +1,67 @@
+
+#include "main.c"
+
+#include "upb_context.h"
+#include "upb_msg.h"
+
+static struct upb_context c;
+static struct upb_string str;
+static struct upb_msg_parse_state s;
+static struct upb_msg *m;
+static void *data;
+
+static bool initialize()
+{
+  /* Initialize upb state, parse descriptor. */
+  upb_context_init(&c);
+  struct upb_string fds;
+  if(!upb_strreadfile(MESSAGE_DESCRIPTOR_FILE, &fds)) {
+    fprintf(stderr, "Couldn't read " MESSAGE_DESCRIPTOR_FILE ".\n");
+    return false;
+  }
+  if(!upb_context_parsefds(&c, &fds)) {
+    fprintf(stderr, "Error importing " MESSAGE_DESCRIPTOR_FILE ".\n");
+    return false;
+  }
+  upb_strfree(fds);
+
+  char class_name[] = MESSAGE_NAME;
+  struct upb_string proto_name;
+  proto_name.ptr = class_name;
+  proto_name.byte_len = sizeof(class_name)-1;
+  struct upb_symtab_entry *e = upb_context_lookup(&c, &proto_name);
+  if(!e || e->type != UPB_SYM_MESSAGE) {
+    fprintf(stderr, "Error finding symbol '" UPB_STRFMT "'.\n",
+            UPB_STRARG(proto_name));
+    return false;
+  }
+
+  m = e->ref.msg;
+  data = upb_msgdata_new(m);
+  upb_msg_parse_init(&s, data, m, false, true);
+
+  /* Read the message data itself. */
+  if(!upb_strreadfile(MESSAGE_FILE, &str)) {
+    fprintf(stderr, "Error reading " MESSAGE_FILE "\n");
+    return false;
+  }
+  return true;
+}
+
+static void cleanup()
+{
+  upb_strfree(str);
+  upb_context_free(&c);
+}
+
+static size_t run()
+{
+  size_t read;
+  upb_msg_parse_reset(&s, data, m, false, BYREF);
+  upb_status_t status = upb_msg_parse(&s, str.ptr, str.byte_len, &read);
+  if(status != UPB_STATUS_OK && read != str.byte_len) {
+    fprintf(stderr, "Error. :(  error=%d, read=%lu\n", status, read);
+    return 0;
+  }
+  return read;
+}