Merge branch 'master' into apitweaks

2 files changed, 141 insertions, 18 deletions

diff --git a/upbc/generator.cc b/upbc/generator.cc
index 6c0179a..3b581f4 100644
--- a/upbc/generator.cc
+++ b/upbc/generator.cc
@@ -8,6 +8,7 @@
 #include "absl/strings/substitute.h"
 #include "google/protobuf/compiler/code_generator.h"
 #include "google/protobuf/descriptor.h"
+#include "google/protobuf/descriptor.pb.h"
 #include "google/protobuf/io/zero_copy_stream.h"
 
 #include "upbc/generator.h"
@@ -32,6 +33,14 @@ static std::string SourceFilename(std::string proto_filename) {
   return StripExtension(proto_filename) + ".upb.c";
 }
 
+static std::string DefHeaderFilename(std::string proto_filename) {
+  return StripExtension(proto_filename) + ".upbdefs.h";
+}
+
+static std::string DefSourceFilename(std::string proto_filename) {
+  return StripExtension(proto_filename) + ".upbdefs.c";
+}
+
 class Output {
  public:
   Output(protobuf::io::ZeroCopyOutputStream* stream) : stream_(stream) {}
@@ -112,7 +121,6 @@ std::vector<const protobuf::Descriptor*> SortedMessages(
   for (int i = 0; i < file->message_type_count(); i++) {
     AddMessages(file->message_type(i), &messages);
   }
-  //SortDefs(&messages);
   return messages;
 }
 
@@ -165,6 +173,10 @@ std::string ToCIdent(absl::string_view str) {
   return absl::StrReplaceAll(str, {{".", "_"}, {"/", "_"}});
 }
 
+std::string DefInitSymbol(const protobuf::FileDescriptor *file) {
+  return ToCIdent(file->name()) + "_upbdefinit";
+}
+
 std::string ToPreproc(absl::string_view str) {
   return absl::AsciiStrToUpper(ToCIdent(str));
 }
@@ -195,13 +207,12 @@ std::string CTypeInternal(const protobuf::FieldDescriptor* field,
       return maybe_const + maybe_struct + MessageName(field->message_type()) +
              "*";
     }
-    case protobuf::FieldDescriptor::CPPTYPE_ENUM:
-      return ToCIdent(field->enum_type()->full_name());
     case protobuf::FieldDescriptor::CPPTYPE_BOOL:
       return "bool";
     case protobuf::FieldDescriptor::CPPTYPE_FLOAT:
       return "float";
     case protobuf::FieldDescriptor::CPPTYPE_INT32:
+    case protobuf::FieldDescriptor::CPPTYPE_ENUM:
       return "int32_t";
     case protobuf::FieldDescriptor::CPPTYPE_UINT32:
       return "uint32_t";
@@ -490,53 +501,68 @@ void WriteHeader(const protobuf::FileDescriptor* file, Output& output) {
       "#include \"upb/decode.h\"\n"
       "#include \"upb/encode.h\"\n"
       "#include \"upb/port_def.inc\"\n"
-      "UPB_BEGIN_EXTERN_C\n\n",
+      "#ifdef __cplusplus\n"
+      "extern \"C\" {\n"
+      "#endif\n\n",
       ToPreproc(file->name()));
 
+  std::vector<const protobuf::Descriptor*> this_file_messages =
+      SortedMessages(file);
+
   // Forward-declare types defined in this file.
-  for (auto message : SortedMessages(file)) {
+  for (auto message : this_file_messages) {
     output("struct $0;\n", ToCIdent(message->full_name()));
   }
-  for (auto message : SortedMessages(file)) {
+  for (auto message : this_file_messages) {
     output("typedef struct $0 $0;\n", ToCIdent(message->full_name()));
   }
-  for (auto message : SortedMessages(file)) {
+  for (auto message : this_file_messages) {
     output("extern const upb_msglayout $0;\n", MessageInit(message));
   }
 
   // Forward-declare types not in this file, but used as submessages.
-  std::set<const protobuf::Descriptor*> forward_messages;
+  // Order by full name for consistent ordering.
+  std::map<std::string, const protobuf::Descriptor*> forward_messages;
+
   for (auto message : SortedMessages(file)) {
     for (int i = 0; i < message->field_count(); i++) {
       const protobuf::FieldDescriptor* field = message->field(i);
       if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE &&
           field->file() != field->message_type()->file()) {
-        forward_messages.insert(field->message_type());
+        forward_messages[field->message_type()->full_name()] =
+            field->message_type();
       }
     }
   }
-  for (const auto& descriptor : forward_messages) {
-    output("struct $0;\n", MessageName(descriptor));
+  for (const auto& pair : forward_messages) {
+    output("struct $0;\n", MessageName(pair.second));
   }
-  for (const auto& descriptor : forward_messages) {
-    output("extern const upb_msglayout $0;\n", MessageInit(descriptor));
+  for (const auto& pair : forward_messages) {
+    output("extern const upb_msglayout $0;\n", MessageInit(pair.second));
   }
 
+  std::vector<const protobuf::EnumDescriptor*> this_file_enums =
+      SortedEnums(file);
+
   output(
       "\n"
       "/* Enums */\n\n");
-  for (auto enumdesc : SortedEnums(file)) {
+  for (auto enumdesc : this_file_enums) {
     output("typedef enum {\n");
     DumpEnumValues(enumdesc, output);
     output("} $0;\n\n", ToCIdent(enumdesc->full_name()));
   }
 
-  for (auto message : SortedMessages(file)) {
+  output("\n");
+
+  for (auto message : this_file_messages) {
     GenerateMessageInHeader(message, output);
   }
 
   output(
-      "UPB_END_EXTERN_C\n"
+      "#ifdef __cplusplus\n"
+      "}  /* extern \"C\" */\n"
+      "#endif\n"
       "\n"
       "#include \"upb/port_undef.inc\"\n"
       "\n"
@@ -610,7 +636,7 @@ void WriteSource(const protobuf::FileDescriptor* file, Output& output) {
         }
 
         if (MessageLayout::HasHasbit(field)) {
-          presence = absl::StrCat(layout.GetHasbitIndex(field) + 1);
+          presence = absl::StrCat(layout.GetHasbitIndex(field));
         } else if (field->containing_oneof()) {
           MessageLayout::Size case_offset =
               layout.GetOneofCaseOffset(field->containing_oneof());
@@ -647,6 +673,95 @@ void WriteSource(const protobuf::FileDescriptor* file, Output& output) {
   output("\n");
 }
 
+void GenerateMessageDefAccessor(const protobuf::Descriptor* d, Output& output) {
+  output("UPB_INLINE const upb_msgdef *$0_getmsgdef(upb_symtab *s) {\n",
+         ToCIdent(d->full_name()));
+  output("  _upb_symtab_loaddefinit(s, &$0);\n", DefInitSymbol(d->file()));
+  output("  return upb_symtab_lookupmsg(s, \"$0\");\n", d->full_name());
+  output("}\n");
+  output("\n");
+
+  for (int i = 0; i < d->nested_type_count(); i++) {
+    GenerateMessageDefAccessor(d->nested_type(i), output);
+  }
+}
+
+void WriteDefHeader(const protobuf::FileDescriptor* file, Output& output) {
+  EmitFileWarning(file, output);
+
+  output("#include \"upb/def.h\"\n");
+  output("\n");
+
+  output("extern upb_def_init $0;\n", DefInitSymbol(file));
+  output("\n");
+
+  for (int i = 0; i < file->message_type_count(); i++) {
+    GenerateMessageDefAccessor(file->message_type(i), output);
+  }
+}
+
+// Escape C++ trigraphs by escaping question marks to \?
+std::string EscapeTrigraphs(absl::string_view to_escape) {
+  return absl::StrReplaceAll(to_escape, {{"?", "\\?"}});
+}
+
+void WriteDefSource(const protobuf::FileDescriptor* file, Output& output) {
+  EmitFileWarning(file, output);
+
+  output("#include \"upb/def.h\"\n");
+  output("\n");
+
+  for (int i = 0; i < file->dependency_count(); i++) {
+    output("extern upb_def_init $0;\n", DefInitSymbol(file->dependency(i)));
+  }
+
+  protobuf::FileDescriptorProto file_proto;
+  file->CopyTo(&file_proto);
+  std::string file_data;
+  file_proto.SerializeToString(&file_data);
+
+  output("static const char descriptor[$0] =\n", file_data.size());
+
+  {
+    if (file_data.size() > 65535) {
+      // Workaround for MSVC: "Error C1091: compiler limit: string exceeds
+      // 65535 bytes in length". Declare a static array of chars rather than
+      // use a string literal. Only write 25 bytes per line.
+      static const size_t kBytesPerLine = 25;
+      output("{ ");
+      for (size_t i = 0; i < file_data.size();) {
+        for (size_t j = 0; j < kBytesPerLine && i < file_data.size(); ++i, ++j) {
+          output("'$0', ", absl::CEscape(file_data.substr(i, 1)));
+        }
+        output("\n");
+      }
+      output("'\\0' }");  // null-terminate
+    } else {
+      // Only write 40 bytes per line.
+      static const size_t kBytesPerLine = 40;
+      for (size_t i = 0; i < file_data.size(); i += kBytesPerLine) {
+        output(
+            "\"$0\"\n",
+            EscapeTrigraphs(absl::CEscape(file_data.substr(i, kBytesPerLine))));
+      }
+    }
+    output(";\n");
+  }
+
+  output("static upb_def_init *deps[$0] = {\n", file->dependency_count() + 1);
+  for (int i = 0; i < file->dependency_count(); i++) {
+    output("  &$0,\n", DefInitSymbol(file->dependency(i)));
+  }
+  output("  NULL\n");
+  output("};\n");
+
+  output("upb_def_init $0 = {\n", DefInitSymbol(file));
+  output("  deps,\n");
+  output("  \"$0\",\n", file->name());
+  output("  UPB_STRVIEW_INIT(descriptor, $0)\n", file_data.size());
+  output("};\n");
+}
+
 bool Generator::Generate(const protobuf::FileDescriptor* file,
                          const std::string& parameter,
                          protoc::GeneratorContext* context,
@@ -657,6 +772,12 @@ bool Generator::Generate(const protobuf::FileDescriptor* file,
   Output c_output(context->Open(SourceFilename(file->name())));
   WriteSource(file, c_output);
 
+  Output h_def_output(context->Open(DefHeaderFilename(file->name())));
+  WriteDefHeader(file, h_def_output);
+
+  Output c_def_output(context->Open(DefSourceFilename(file->name())));
+  WriteDefSource(file, c_def_output);
+
   return true;
 }
 
diff --git a/upbc/message_layout.cc b/upbc/message_layout.cc
index 5956424..f0a6872 100644
--- a/upbc/message_layout.cc
+++ b/upbc/message_layout.cc
@@ -130,7 +130,9 @@ void MessageLayout::PlaceNonOneofFields(
   int hasbit_count = 0;
   for (auto field : field_order) {
     if (HasHasbit(field)) {
-      hasbit_indexes_[field] = hasbit_count++;
+      // We don't use hasbit 0, so that 0 can indicate "no presence" in the
+      // table. This wastes one hasbit, but we don't worry about it for now.
+      hasbit_indexes_[field] = ++hasbit_count;
     }
   }