Beginnings of a more extensive benchmarking suite.

7 files changed, 274 insertions, 115 deletions

diff --git a/benchmark/benchmark.cc b/benchmark/benchmark.cc
deleted file mode 100644
index c3f2f6e..0000000
--- a/benchmark/benchmark.cc
+++ /dev/null
@@ -1,115 +0,0 @@
-
-#include <time.h>
-#include "google_messages.pb.h"
-#include <google/protobuf/dynamic_message.h>
-#include "test_util.h"
-#include "upb_context.h"
-#include "upb_msg.h"
-
-int main ()
-{
-  /* Initialize upb state, parse descriptor. */
-  struct upb_context c;
-  upb_context_init(&c);
-  struct upb_string fds;
-  if(!upb_strreadfile("benchmark/google_messages.proto.pb", &fds)) {
-    fprintf(stderr, "Couldn't read google_speed.proto.bin.\n");
-    return 1;
-  }
-  if(!upb_context_parsefds(&c, &fds)) {
-    fprintf(stderr, "Error parsing or resolving proto.\n");
-    return 1;
-  }
-  upb_strfree(fds);
-  char class_name[] = "benchmarks.SpeedMessage2";
-  struct upb_string proto_name;
-  proto_name.ptr = class_name;
-  proto_name.byte_len = sizeof(class_name)-1;
-  struct upb_symtab_entry *e = upb_context_lookup(&c, &proto_name);
-  if(!e || e->type != UPB_SYM_MESSAGE) {
-    fprintf(stderr, "Error finding symbol '" UPB_STRFMT "'.\n",
-            UPB_STRARG(proto_name));
-    return 1;
-  }
-
-  /* upb speed test, copying string. */
-  struct upb_msg *m = e->ref.msg;
-  struct upb_msg_parse_state s;
-  void *data = upb_msgdata_new(m);
-  upb_msg_parse_init(&s, data, m, false, true);
-  size_t read;
-  struct upb_string str;
-  if(!upb_strreadfile("benchmark/google_message2.dat", &str)) {
-    fprintf(stderr, "Error reading google_message2.dat\n");
-    return 1;
-  }
-  size_t total = 0;
-  clock_t before = clock();
-  for(int i = 0; i < 2000; i++) {
-    upb_msg_parse_reset(&s, data, m, false, false);
-    upb_status_t status = upb_msg_parse(&s, str.ptr, str.byte_len, &read);
-    if(status != UPB_STATUS_OK && read != str.byte_len) {
-      fprintf(stderr, "Error. :(  error=%d, read=%lu\n", status, read);
-      return 1;
-    }
-    total += str.byte_len;
-  }
-  double elapsed = ((double)clock() - before) / CLOCKS_PER_SEC;
-  fprintf(stderr, "upb parsed %sB, ", eng(total, 3, false));
-  fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false));
-
-  /* upb speed test, referencing strings. */
-  total = 0;
-  before = clock();
-  for(int i = 0; i < 2000; i++) {
-    upb_msg_parse_reset(&s, data, m, false, true);
-    upb_status_t status = upb_msg_parse(&s, str.ptr, str.byte_len, &read);
-    if(status != UPB_STATUS_OK && read != str.byte_len) {
-      fprintf(stderr, "Error. :(  error=%d, read=%lu\n", status, read);
-      return 1;
-    }
-    total += str.byte_len;
-  }
-  elapsed = ((double)clock() - before) / CLOCKS_PER_SEC;
-  fprintf(stderr, "upb(byref) parsed %sB, ", eng(total, 3, false));
-  fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false));
-  upb_msg_parse_free(&s);
-  upb_msgdata_free(data, m, true);
-  upb_context_free(&c);
-
-  /* proto2 speed test, dynamic type. */
-  std::string stlstr(str.ptr, str.byte_len);
-  upb_strfree(str);
-
-  google::protobuf::DynamicMessageFactory factory;
-  const google::protobuf::Message *dynamic_msg_prototype = factory.GetPrototype(benchmarks::SpeedMessage2::descriptor());
-  google::protobuf::Message *dynamic_msg = dynamic_msg_prototype->New();
-  total = 0;
-  before = clock();
-  for(int i = 0; i < 2000; i++) {
-    if(!dynamic_msg->ParseFromString(stlstr)) {
-      fprintf(stderr, "Error parsing with proto2.\n");
-      return 1;
-    }
-    total += str.byte_len;
-  }
-  delete dynamic_msg;
-  elapsed = ((double)clock() - before) / CLOCKS_PER_SEC;
-  fprintf(stderr, "proto2(dynamic) parsed %sB, ", eng(total, 3, false));
-  fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false));
-
-  /* proto2 speed test, compiled-in type. */
-  benchmarks::SpeedMessage2 msg;
-  total = 0;
-  before = clock();
-  for(int i = 0; i < 2000; i++) {
-    if(!msg.ParseFromString(stlstr)) {
-      fprintf(stderr, "Error parsing with proto2.\n");
-      return 1;
-    }
-    total += str.byte_len;
-  }
-  elapsed = ((double)clock() - before) / CLOCKS_PER_SEC;
-  fprintf(stderr, "proto2 parsed %sB, ", eng(total, 3, false));
-  fprintf(stderr, "%sB/s\n", eng(total/elapsed, 3, false));
-}
diff --git a/benchmark/gen.sh b/benchmark/gen.sh
new file mode 100755
index 0000000..288071e
--- /dev/null
+++ b/benchmark/gen.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+cd `dirname $0`
+CXXFLAGS="-O3 -msse3 -I../src -I../descriptor -Wall"
+CFLAGS="-std=c99 $CXXFLAGS"
+set -e
+set -v
+
+gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \
+    -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \
+    -DMESSAGE_FILE=\"google_message1.dat\" \
+    -DBYREF=false \
+    $CFLAGS \
+    parsetostruct_upb_table.c -o b_parsetostruct_googlemessage1_upb_table_byval ../src/libupb.a
+
+gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage1\" \
+    -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \
+    -DMESSAGE_FILE=\"google_message1.dat\" \
+    -DBYREF=true \
+    $CFLAGS \
+    parsetostruct_upb_table.c -o b_parsetostruct_googlemessage1_upb_table_byref ../src/libupb.a
+
+gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \
+    -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \
+    -DMESSAGE_FILE=\"google_message2.dat\" \
+    -DBYREF=false \
+    $CFLAGS \
+    parsetostruct_upb_table.c -o b_parsetostruct_googlemessage2_upb_table_byval ../src/libupb.a
+
+gcc -DMESSAGE_NAME=\"benchmarks.SpeedMessage2\" \
+    -DMESSAGE_DESCRIPTOR_FILE=\"google_messages.proto.pb\" \
+    -DMESSAGE_FILE=\"google_message2.dat\" \
+    -DBYREF=true \
+    $CFLAGS \
+    parsetostruct_upb_table.c -o b_parsetostruct_googlemessage2_upb_table_byref ../src/libupb.a
+
+g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage2" \
+    -DMESSAGE_FILE=\"google_message2.dat\" \
+    -DMESSAGE_HFILE=\"google_messages.pb.h\" \
+    $CXXFLAGS \
+    parsetostruct_proto2_table.cc -o b_parsetostruct_googlemessage2_proto2_table -lprotobuf -lpthread google_messages.pb.o
+
+g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage2" \
+    -DMESSAGE_FILE=\"google_message2.dat\" \
+    -DMESSAGE_HFILE=\"google_messages.pb.h\" \
+    $CXXFLAGS \
+    parsetostruct_proto2_compiled.cc -o b_parsetostruct_googlemessage2_proto2_compiled -lprotobuf -lpthread google_messages.pb.o
+
+g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage1" \
+    -DMESSAGE_FILE=\"google_message1.dat\" \
+    -DMESSAGE_HFILE=\"google_messages.pb.h\" \
+    $CXXFLAGS \
+    parsetostruct_proto2_table.cc -o b_parsetostruct_googlemessage1_proto2_table -lprotobuf -lpthread google_messages.pb.o
+
+g++ -DMESSAGE_CIDENT="benchmarks::SpeedMessage1" \
+    -DMESSAGE_FILE=\"google_message1.dat\" \
+    -DMESSAGE_HFILE=\"google_messages.pb.h\" \
+    $CXXFLAGS \
+    parsetostruct_proto2_compiled.cc -o b_parsetostruct_googlemessage1_proto2_compiled -lprotobuf -lpthread google_messages.pb.o
diff --git a/benchmark/getgraphs.py b/benchmark/getgraphs.py
new file mode 100644
index 0000000..392b6ef
--- /dev/null
+++ b/benchmark/getgraphs.py
@@ -0,0 +1,35 @@
+
+import sys
+
+benchmarks = {}
+color_map = {'proto2_compiled': 'FF0000',
+             'proto2_table': 'FF00FF',
+             'upb_table_byref': '0000FF',
+             'upb_table_byval': '00FF00'}
+for line in sys.stdin:
+  name, val = line.split(': ')
+  components = name.split('_')
+  benchmark = '_'.join(components[1:3])
+  variant = '_'.join(components[3:])
+  if benchmark not in benchmarks:
+    benchmarks[benchmark] = []
+  benchmarks[benchmark].append((variant, int(val)))
+
+def encode(x):
+  digits = (range(ord("A"), ord("Z")+1) + range(ord("a"), ord("z")+1) +
+            range(ord("0"), ord("9")+1) + [ord("."), ord("-")])
+  return chr(digits[x / 64]) + chr(digits[x % 64])
+
+for benchmark, values in benchmarks.items():
+  def cmp(a, b):
+    return b[1] - a[1]
+  values.sort(cmp)
+  variants = [x[0] for x in values]
+  values = [x[1] for x in values]
+  scaling = 300
+  encoded_values = [encode((x * 4096 / scaling) - 1) for x in values]
+  legend = "chdl=%s" % ("|".join(variants))
+  colors = "chco=%s" % ("|".join([color_map[x] for x in variants]))
+  data = "chd=e:%s" % ("".join(encoded_values))
+  url = "http://chart.apis.google.com/chart?cht=bhs&chs=500x200&chtt=%s+(MB/s)&chxt=x&chxr=0,0,%d&%s" % (benchmark, scaling, "&".join([legend, data, colors]))
+  print url
diff --git a/benchmark/main.c b/benchmark/main.c
new file mode 100644
index 0000000..c038287
--- /dev/null
+++ b/benchmark/main.c
@@ -0,0 +1,32 @@
+
+#include <stdbool.h>
+#include <time.h>
+#include <stdio.h>
+
+static bool initialize();
+static void cleanup();
+static size_t run();
+
+int main (int argc, char *argv[])
+{
+  if(!initialize()) {
+    fprintf(stderr, "%s: failed to initialize\n", argv[0]);
+    return 1;
+  }
+
+  size_t total_bytes = 0;
+  clock_t before = clock();
+  for(int i = 0; true; i++) {
+    if((i & 0xFF) == 0 && (clock() - before > CLOCKS_PER_SEC)) break;
+    size_t bytes = run();
+    if(bytes == 0) {
+      fprintf(stderr, "%s: failed.\n", argv[0]);
+      return 2;
+    }
+    total_bytes += bytes;
+  }
+  double elapsed = ((double)clock() - before) / CLOCKS_PER_SEC;
+  printf("%s: %d\n", argv[0], (int)(total_bytes / elapsed / (1 << 20)));
+  cleanup();
+  return 0;
+}
diff --git a/benchmark/parsetostruct_proto2_compiled.cc b/benchmark/parsetostruct_proto2_compiled.cc
new file mode 100644
index 0000000..339e184
--- /dev/null
+++ b/benchmark/parsetostruct_proto2_compiled.cc
@@ -0,0 +1,37 @@
+
+#include "main.c"
+#include MESSAGE_HFILE
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+
+static std::string str;
+MESSAGE_CIDENT msg;
+
+static bool initialize()
+{
+  /* Read the message data itself. */
+  std::ifstream stream(MESSAGE_FILE);
+  if(!stream.is_open()) {
+    fprintf(stderr, "Error opening " MESSAGE_FILE ".\n");
+    return false;
+  }
+  std::stringstream stringstream;
+  stringstream << stream.rdbuf();
+  str = stringstream.str();
+  return true;
+}
+
+static void cleanup()
+{
+}
+
+static size_t run()
+{
+  if(!msg.ParseFromString(str)) {
+    fprintf(stderr, "Error parsing with proto2.\n");
+    return 0;
+  }
+  return str.size();
+}
diff --git a/benchmark/parsetostruct_proto2_table.cc b/benchmark/parsetostruct_proto2_table.cc
new file mode 100644
index 0000000..815834e
--- /dev/null
+++ b/benchmark/parsetostruct_proto2_table.cc
@@ -0,0 +1,44 @@
+
+#include "main.c"
+#include <google/protobuf/dynamic_message.h>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+#include MESSAGE_HFILE
+
+static std::string str;
+static google::protobuf::DynamicMessageFactory factory;
+static google::protobuf::Message *msg;
+
+static bool initialize()
+{
+  /* Read the message data itself. */
+  std::ifstream stream(MESSAGE_FILE);
+  if(!stream.is_open()) {
+    fprintf(stderr, "Error opening " MESSAGE_FILE ".\n");
+    return false;
+  }
+  std::stringstream stringstream;
+  stringstream << stream.rdbuf();
+  str = stringstream.str();
+
+  /* Create the DynamicMessage. */
+  const google::protobuf::Message *dynamic_msg_prototype =
+      factory.GetPrototype(MESSAGE_CIDENT::descriptor());
+  msg = dynamic_msg_prototype->New();
+  return true;
+}
+
+static void cleanup()
+{
+  delete msg;
+}
+
+static size_t run()
+{
+  if(!msg->ParseFromString(str)) {
+    fprintf(stderr, "Error parsing with proto2.\n");
+    return 0;
+  }
+  return str.size();
+}
diff --git a/benchmark/parsetostruct_upb_table.c b/benchmark/parsetostruct_upb_table.c
new file mode 100644
index 0000000..f93a6a7
--- /dev/null
+++ b/benchmark/parsetostruct_upb_table.c
@@ -0,0 +1,67 @@
+
+#include "main.c"
+
+#include "upb_context.h"
+#include "upb_msg.h"
+
+static struct upb_context c;
+static struct upb_string str;
+static struct upb_msg_parse_state s;
+static struct upb_msg *m;
+static void *data;
+
+static bool initialize()
+{
+  /* Initialize upb state, parse descriptor. */
+  upb_context_init(&c);
+  struct upb_string fds;
+  if(!upb_strreadfile(MESSAGE_DESCRIPTOR_FILE, &fds)) {
+    fprintf(stderr, "Couldn't read " MESSAGE_DESCRIPTOR_FILE ".\n");
+    return false;
+  }
+  if(!upb_context_parsefds(&c, &fds)) {
+    fprintf(stderr, "Error importing " MESSAGE_DESCRIPTOR_FILE ".\n");
+    return false;
+  }
+  upb_strfree(fds);
+
+  char class_name[] = MESSAGE_NAME;
+  struct upb_string proto_name;
+  proto_name.ptr = class_name;
+  proto_name.byte_len = sizeof(class_name)-1;
+  struct upb_symtab_entry *e = upb_context_lookup(&c, &proto_name);
+  if(!e || e->type != UPB_SYM_MESSAGE) {
+    fprintf(stderr, "Error finding symbol '" UPB_STRFMT "'.\n",
+            UPB_STRARG(proto_name));
+    return false;
+  }
+
+  m = e->ref.msg;
+  data = upb_msgdata_new(m);
+  upb_msg_parse_init(&s, data, m, false, true);
+
+  /* Read the message data itself. */
+  if(!upb_strreadfile(MESSAGE_FILE, &str)) {
+    fprintf(stderr, "Error reading " MESSAGE_FILE "\n");
+    return false;
+  }
+  return true;
+}
+
+static void cleanup()
+{
+  upb_strfree(str);
+  upb_context_free(&c);
+}
+
+static size_t run()
+{
+  size_t read;
+  upb_msg_parse_reset(&s, data, m, false, BYREF);
+  upb_status_t status = upb_msg_parse(&s, str.ptr, str.byte_len, &read);
+  if(status != UPB_STATUS_OK && read != str.byte_len) {
+    fprintf(stderr, "Error. :(  error=%d, read=%lu\n", status, read);
+    return 0;
+  }
+  return read;
+}