Squashed 'libs/protobuf/' content from commit fcd3b9a85

git-subtree-dir: libs/protobuf
git-subtree-split: fcd3b9a85ef36e46643dc30176cea1a7ad62e02b

conformance/BUILD.bazel

@@ -0,0 +1,370 @@
+# Conformance testing for Protobuf.
+
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "objc_library")
+load("//:protobuf.bzl", "internal_csharp_proto_library", "internal_objc_proto_library", "internal_php_proto_library", "internal_py_proto_library", "internal_ruby_proto_library")
+load("//build_defs:internal_shell.bzl", "inline_sh_binary")
+load(
+    "@rules_pkg//:mappings.bzl",
+    "pkg_attributes",
+    "pkg_filegroup",
+    "pkg_files",
+    "strip_prefix",
+)
+
+exports_files([
+    "bazel_conformance_test_runner.sh",
+    "failure_list_cpp.txt",
+    "failure_list_csharp.txt",
+    "failure_list_java.txt",
+    "failure_list_java_lite.txt",
+    "failure_list_objc.txt",
+    "failure_list_php.txt",
+    "failure_list_php_c.txt",
+    "failure_list_python.txt",
+    "failure_list_python_cpp.txt",
+    "failure_list_ruby.txt",
+    "failure_list_jruby.txt",
+    "text_format_failure_list_cpp.txt",
+    "text_format_failure_list_csharp.txt",
+    "text_format_failure_list_java.txt",
+    "text_format_failure_list_java_lite.txt",
+    "text_format_failure_list_php.txt",
+    "text_format_failure_list_php_c.txt",
+    "text_format_failure_list_python.txt",
+    "text_format_failure_list_python_cpp.txt",
+    "text_format_failure_list_ruby.txt",
+    "text_format_failure_list_jruby.txt",
+])
+
+cc_proto_library(
+    name = "test_messages_proto2_proto_cc",
+    deps = ["//src/google/protobuf:test_messages_proto2_proto"],
+)
+
+cc_proto_library(
+    name = "test_messages_proto3_proto_cc",
+    deps = ["//src/google/protobuf:test_messages_proto3_proto"],
+)
+
+proto_library(
+    name = "conformance_proto",
+    srcs = ["conformance.proto"],
+    visibility = ["//visibility:public"],
+)
+
+cc_proto_library(
+    name = "conformance_cc_proto",
+    deps = [":conformance_proto"],
+)
+
+internal_csharp_proto_library(
+    name = "conformance_csharp_proto",
+    srcs = ["conformance.proto"],
+    visibility = [
+        "//csharp:__subpackages__",
+    ],
+)
+
+java_proto_library(
+    name = "conformance_java_proto",
+    visibility = [
+        "//java:__subpackages__",
+    ],
+    deps = [":conformance_proto"],
+)
+
+java_lite_proto_library(
+    name = "conformance_java_proto_lite",
+    visibility = [
+        "//java:__subpackages__",
+    ],
+    deps = [":conformance_proto"],
+)
+
+internal_objc_proto_library(
+    name = "conformance_objc_proto",
+    srcs = ["conformance.proto"],
+    visibility = [
+        "//conformance:__pkg__",
+        "//objc:__pkg__",
+    ],
+)
+
+internal_py_proto_library(
+    name = "conformance_py_proto",
+    srcs = ["conformance.proto"],
+    srcs_version = "PY2AND3",
+    visibility = [
+        "//python:__subpackages__",
+    ],
+)
+
+internal_php_proto_library(
+    name = "conformance_php_proto",
+    srcs = ["conformance.proto"],
+    outs = [
+        "Conformance/ConformanceRequest.php",
+        "Conformance/ConformanceResponse.php",
+        "Conformance/FailureSet.php",
+        "Conformance/JspbEncodingConfig.php",
+        "Conformance/TestCategory.php",
+        "Conformance/WireFormat.php",
+        "GPBMetadata/Conformance.php",
+    ],
+    visibility = [
+        "//conformance:__pkg__",
+        "//php:__pkg__",
+    ],
+)
+
+internal_ruby_proto_library(
+    name = "conformance_ruby_proto",
+    srcs = ["conformance.proto"],
+    visibility = [
+        "//conformance:__pkg__",
+        "//ruby:__pkg__",
+    ],
+)
+
+cc_library(
+    name = "conformance_test",
+    srcs = [
+        "conformance_test.cc",
+        "conformance_test_runner.cc",
+    ],
+    hdrs = [
+        "conformance_test.h",
+    ],
+    includes = ["."],
+    deps = [
+        ":conformance_cc_proto",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_library(
+    name = "binary_json_conformance_suite",
+    srcs = ["binary_json_conformance_suite.cc"],
+    hdrs = ["binary_json_conformance_suite.h"],
+    deps = [
+        ":conformance_test",
+        "@jsoncpp//:jsoncpp",
+        ":test_messages_proto2_proto_cc",
+        ":test_messages_proto3_proto_cc",
+        "@com_google_absl//absl/status",
+    ],
+)
+
+cc_library(
+    name = "text_format_conformance_suite",
+    srcs = ["text_format_conformance_suite.cc"],
+    hdrs = ["text_format_conformance_suite.h"],
+    deps = [
+        ":conformance_test",
+        ":test_messages_proto2_proto_cc",
+        ":test_messages_proto3_proto_cc",
+    ],
+)
+
+cc_binary(
+    name = "conformance_test_runner",
+    srcs = ["conformance_test_main.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":binary_json_conformance_suite",
+        ":conformance_test",
+        ":text_format_conformance_suite",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+)
+
+cc_binary(
+    name = "conformance_cpp",
+    testonly = 1,
+    srcs = ["conformance_cpp.cc"],
+    visibility = ["//src:__subpackages__"],
+    deps = [
+        ":conformance_cc_proto",
+        "//:protobuf",
+        "//:test_messages_proto2_cc_proto",
+        "//:test_messages_proto3_cc_proto",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+java_binary(
+    name = "conformance_java",
+    testonly = 1,
+    srcs = ["ConformanceJava.java"],
+    main_class = "ConformanceJava",
+    visibility = [
+        "//java:__subpackages__",
+    ],
+    deps = [
+        ":conformance_java_proto",
+        "//:protobuf_java",
+        "//:protobuf_java_util",
+        "//:test_messages_proto2_java_proto",
+        "//:test_messages_proto3_java_proto",
+    ],
+)
+
+java_binary(
+    name = "conformance_java_lite",
+    testonly = 1,
+    srcs = ["ConformanceJavaLite.java"],
+    main_class = "ConformanceJavaLite",
+    visibility = [
+        "//java:__subpackages__",
+    ],
+    deps = [
+        ":conformance_java_proto_lite",
+        "//:protobuf_java_util",
+        "//:protobuf_javalite",
+        "//:test_messages_proto2_java_proto_lite",
+        "//:test_messages_proto3_java_proto_lite",
+    ],
+)
+
+py_binary(
+    name = "conformance_python",
+    testonly = 1,
+    srcs = ["conformance_python.py"],
+    imports = [
+        "..",
+        "../python",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//python:__subpackages__"],
+    deps = [
+        ":conformance_py_proto",
+        "//:protobuf_python",
+        "//python:test_messages_proto2_py_proto",
+        "//python:test_messages_proto3_py_proto",
+    ],
+)
+
+inline_sh_binary(
+    name = "conformance_php",
+    testonly = 1,
+    srcs = [
+        "autoload.php",
+        "conformance_php.php",
+    ],
+    cmd = """
+        php -d include_path=conformance:src/google/protobuf \\
+            -d auto_prepend_file=$(rootpath autoload.php) \\
+            $(rootpath conformance_php.php)
+    """,
+    visibility = ["//php:__subpackages__"],
+    deps = [
+        ":conformance_php_proto",
+        "//:test_messages_proto3_php_proto",
+        "//php:source_files",
+    ],
+)
+
+inline_sh_binary(
+    name = "conformance_php_c",
+    testonly = 1,
+    srcs = [
+        "conformance_php.php",
+        "//php:extension",
+    ],
+    cmd = """
+        php -dextension=$(rootpath //php:extension) \\
+            -d include_path=conformance:src/google/protobuf \\
+            $(rootpath conformance_php.php)
+    """,
+    visibility = ["//php:__subpackages__"],
+    deps = [
+        ":conformance_php_proto",
+        "//:test_messages_proto3_php_proto",
+    ],
+)
+
+inline_sh_binary(
+    name = "conformance_csharp",
+    testonly = 1,
+    srcs = ["//csharp/src/Google.Protobuf.Conformance:conformance_dll"],
+    cmd = "dotnet $(rootpath //csharp/src/Google.Protobuf.Conformance:conformance_dll)",
+    visibility = ["//csharp:__subpackages__"],
+    deps = [
+        "//csharp/src/Google.Protobuf.Conformance:conformance_runfiles",
+    ],
+)
+
+objc_library(
+    name = "conformance_objc_lib",
+    testonly = 1,
+    non_arc_srcs = ["conformance_objc.m"],
+    # See https://github.com/bazelbuild/bazel/issues/12897.
+    tags = ["manual"],
+    deps = [
+        ":conformance_objc_proto",
+        "//:test_messages_proto2_objc_proto",
+        "//:test_messages_proto3_objc_proto",
+    ],
+)
+
+cc_binary(
+    name = "conformance_objc",
+    testonly = 1,
+    # See https://github.com/bazelbuild/bazel/issues/12897.
+    tags = ["manual"],
+    visibility = ["//objectivec:__subpackages__"],
+    deps = [":conformance_objc_lib"],
+)
+
+inline_sh_binary(
+    name = "conformance_ruby",
+    testonly = 1,
+    srcs = ["conformance_ruby.rb"],
+    cmd = "RUBYLIB=ruby/lib:conformance:src $(rootpath conformance_ruby.rb)",
+    visibility = ["//ruby:__subpackages__"],
+    deps = [
+        ":conformance_ruby_proto",
+        "//:test_messages_proto2_ruby_proto",
+        "//:test_messages_proto3_ruby_proto",
+        "//:well_known_ruby_protos",
+        "//ruby:protobuf",
+    ],
+)
+
+################################################################################
+# Distribution files
+################################################################################
+
+filegroup(
+    name = "all_files",
+    srcs = glob(["**/*"]),
+    visibility = ["//src/google/protobuf/compiler/csharp:__pkg__"],
+)
+
+pkg_files(
+    name = "dist_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            # Handled by dist_scripts:
+            "bazel_conformance_test_runner.sh",
+
+            # The following are not in autotools dist:
+            "autoload.php",
+            "failure_list_jruby.txt",
+            "update_failure_list.py",
+        ],
+    ),
+    strip_prefix = strip_prefix.from_root(""),
+    visibility = ["//pkg:__pkg__"],
+)
+
+pkg_filegroup(
+    name = "all_dist_files",
+    srcs = [
+        ":dist_files",
+    ],
+    visibility = ["//pkg:__pkg__"],
+)

conformance/ConformanceJava.java

@@ -0,0 +1,418 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import com.google.protobuf.AbstractMessage;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.ExtensionRegistry;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.Parser;
+import com.google.protobuf.TextFormat;
+import com.google.protobuf.conformance.Conformance;
+import com.google.protobuf.util.JsonFormat;
+import com.google.protobuf.util.JsonFormat.TypeRegistry;
+import com.google.protobuf_test_messages.proto2.TestMessagesProto2;
+import com.google.protobuf_test_messages.proto2.TestMessagesProto2.TestAllTypesProto2;
+import com.google.protobuf_test_messages.proto3.TestMessagesProto3;
+import com.google.protobuf_test_messages.proto3.TestMessagesProto3.TestAllTypesProto3;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+class ConformanceJava {
+  private int testCount = 0;
+  private TypeRegistry typeRegistry;
+
+  private boolean readFromStdin(byte[] buf, int len) throws Exception {
+    int ofs = 0;
+    while (len > 0) {
+      int read = System.in.read(buf, ofs, len);
+      if (read == -1) {
+        return false; // EOF
+      }
+      ofs += read;
+      len -= read;
+    }
+
+    return true;
+  }
+
+  private void writeToStdout(byte[] buf) throws Exception {
+    System.out.write(buf);
+  }
+
+  // Returns -1 on EOF (the actual values will always be positive).
+  private int readLittleEndianIntFromStdin() throws Exception {
+    byte[] buf = new byte[4];
+    if (!readFromStdin(buf, 4)) {
+      return -1;
+    }
+    return (buf[0] & 0xff)
+        | ((buf[1] & 0xff) << 8)
+        | ((buf[2] & 0xff) << 16)
+        | ((buf[3] & 0xff) << 24);
+  }
+
+  private void writeLittleEndianIntToStdout(int val) throws Exception {
+    byte[] buf = new byte[4];
+    buf[0] = (byte) val;
+    buf[1] = (byte) (val >> 8);
+    buf[2] = (byte) (val >> 16);
+    buf[3] = (byte) (val >> 24);
+    writeToStdout(buf);
+  }
+
+  private enum BinaryDecoderType {
+    BTYE_STRING_DECODER,
+    BYTE_ARRAY_DECODER,
+    ARRAY_BYTE_BUFFER_DECODER,
+    READONLY_ARRAY_BYTE_BUFFER_DECODER,
+    DIRECT_BYTE_BUFFER_DECODER,
+    READONLY_DIRECT_BYTE_BUFFER_DECODER,
+    INPUT_STREAM_DECODER;
+  }
+
+  private static class BinaryDecoder<T extends AbstractMessage> {
+    public T decode(
+        ByteString bytes, BinaryDecoderType type, Parser<T> parser, ExtensionRegistry extensions)
+        throws InvalidProtocolBufferException {
+      switch (type) {
+        case BTYE_STRING_DECODER:
+        case BYTE_ARRAY_DECODER:
+          return parser.parseFrom(bytes, extensions);
+        case ARRAY_BYTE_BUFFER_DECODER:
+          {
+            ByteBuffer buffer = ByteBuffer.allocate(bytes.size());
+            bytes.copyTo(buffer);
+            buffer.flip();
+            return parser.parseFrom(CodedInputStream.newInstance(buffer), extensions);
+          }
+        case READONLY_ARRAY_BYTE_BUFFER_DECODER:
+          {
+            return parser.parseFrom(
+                CodedInputStream.newInstance(bytes.asReadOnlyByteBuffer()), extensions);
+          }
+        case DIRECT_BYTE_BUFFER_DECODER:
+          {
+            ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.size());
+            bytes.copyTo(buffer);
+            buffer.flip();
+            return parser.parseFrom(CodedInputStream.newInstance(buffer), extensions);
+          }
+        case READONLY_DIRECT_BYTE_BUFFER_DECODER:
+          {
+            ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.size());
+            bytes.copyTo(buffer);
+            buffer.flip();
+            return parser.parseFrom(
+                CodedInputStream.newInstance(buffer.asReadOnlyBuffer()), extensions);
+          }
+        case INPUT_STREAM_DECODER:
+          {
+            return parser.parseFrom(bytes.newInput(), extensions);
+          }
+      }
+      return null;
+    }
+  }
+
+  private <T extends AbstractMessage> T parseBinary(
+      ByteString bytes, Parser<T> parser, ExtensionRegistry extensions)
+      throws InvalidProtocolBufferException {
+    ArrayList<T> messages = new ArrayList<>();
+    ArrayList<InvalidProtocolBufferException> exceptions = new ArrayList<>();
+
+    for (int i = 0; i < BinaryDecoderType.values().length; i++) {
+      messages.add(null);
+      exceptions.add(null);
+    }
+    if (messages.isEmpty()) {
+      throw new RuntimeException("binary decoder types missing");
+    }
+
+    BinaryDecoder<T> decoder = new BinaryDecoder<>();
+
+    boolean hasMessage = false;
+    boolean hasException = false;
+    for (int i = 0; i < BinaryDecoderType.values().length; ++i) {
+      try {
+        // = BinaryDecoderType.values()[i].parseProto3(bytes);
+        messages.set(i, decoder.decode(bytes, BinaryDecoderType.values()[i], parser, extensions));
+        hasMessage = true;
+      } catch (InvalidProtocolBufferException e) {
+        exceptions.set(i, e);
+        hasException = true;
+      }
+    }
+
+    if (hasMessage && hasException) {
+      StringBuilder sb =
+          new StringBuilder("Binary decoders disagreed on whether the payload was valid.\n");
+      for (int i = 0; i < BinaryDecoderType.values().length; ++i) {
+        sb.append(BinaryDecoderType.values()[i].name());
+        if (messages.get(i) != null) {
+          sb.append(" accepted the payload.\n");
+        } else {
+          sb.append(" rejected the payload.\n");
+        }
+      }
+      throw new RuntimeException(sb.toString());
+    }
+
+    if (hasException) {
+      // We do not check if exceptions are equal. Different implementations may return different
+      // exception messages. Throw an arbitrary one out instead.
+      InvalidProtocolBufferException exception = null;
+      for (InvalidProtocolBufferException e : exceptions) {
+        if (exception != null) {
+          exception.addSuppressed(e);
+        } else {
+          exception = e;
+        }
+      }
+      throw exception;
+    }
+
+    // Fast path comparing all the messages with the first message, assuming equality being
+    // symmetric and transitive.
+    boolean allEqual = true;
+    for (int i = 1; i < messages.size(); ++i) {
+      if (!messages.get(0).equals(messages.get(i))) {
+        allEqual = false;
+        break;
+      }
+    }
+
+    // Slow path: compare and find out all unequal pairs.
+    if (!allEqual) {
+      StringBuilder sb = new StringBuilder();
+      for (int i = 0; i < messages.size() - 1; ++i) {
+        for (int j = i + 1; j < messages.size(); ++j) {
+          if (!messages.get(i).equals(messages.get(j))) {
+            sb.append(BinaryDecoderType.values()[i].name())
+                .append(" and ")
+                .append(BinaryDecoderType.values()[j].name())
+                .append(" parsed the payload differently.\n");
+          }
+        }
+      }
+      throw new RuntimeException(sb.toString());
+    }
+
+    return messages.get(0);
+  }
+
+  private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest request) {
+    AbstractMessage testMessage;
+    String messageType = request.getMessageType();
+    boolean isProto3 =
+        messageType.equals("protobuf_test_messages.proto3.TestAllTypesProto3");
+    boolean isProto2 =
+        messageType.equals("protobuf_test_messages.proto2.TestAllTypesProto2");
+
+    switch (request.getPayloadCase()) {
+      case PROTOBUF_PAYLOAD:
+        {
+          if (isProto3) {
+            try {
+              ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+              TestMessagesProto3.registerAllExtensions(extensions);
+              testMessage =
+                  parseBinary(
+                      request.getProtobufPayload(), TestAllTypesProto3.parser(), extensions);
+            } catch (InvalidProtocolBufferException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+            }
+          } else if (isProto2) {
+            try {
+              ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+              TestMessagesProto2.registerAllExtensions(extensions);
+              testMessage =
+                  parseBinary(
+                      request.getProtobufPayload(), TestAllTypesProto2.parser(), extensions);
+            } catch (InvalidProtocolBufferException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+            }
+          } else {
+            throw new IllegalArgumentException(
+                "Protobuf request has unexpected payload type: " + messageType);
+          }
+          break;
+        }
+      case JSON_PAYLOAD:
+        {
+          try {
+            JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(typeRegistry);
+            if (request.getTestCategory()
+                == Conformance.TestCategory.JSON_IGNORE_UNKNOWN_PARSING_TEST) {
+              parser = parser.ignoringUnknownFields();
+            }
+            if (isProto3) {
+              TestMessagesProto3.TestAllTypesProto3.Builder builder =
+                  TestMessagesProto3.TestAllTypesProto3.newBuilder();
+              parser.merge(request.getJsonPayload(), builder);
+              testMessage = builder.build();
+            } else if (isProto2) {
+              TestMessagesProto2.TestAllTypesProto2.Builder builder =
+                  TestMessagesProto2.TestAllTypesProto2.newBuilder();
+              parser.merge(request.getJsonPayload(), builder);
+              testMessage = builder.build();
+            } else {
+              throw new IllegalArgumentException(
+                  "Protobuf request has unexpected payload type: " + messageType);
+            }
+          } catch (InvalidProtocolBufferException e) {
+            return Conformance.ConformanceResponse.newBuilder()
+                .setParseError(e.getMessage())
+                .build();
+          }
+          break;
+        }
+      case TEXT_PAYLOAD:
+        {
+          if (isProto3) {
+            try {
+              TestMessagesProto3.TestAllTypesProto3.Builder builder =
+                  TestMessagesProto3.TestAllTypesProto3.newBuilder();
+              TextFormat.merge(request.getTextPayload(), builder);
+              testMessage = builder.build();
+            } catch (TextFormat.ParseException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+            }
+          } else if (isProto2) {
+            try {
+              TestMessagesProto2.TestAllTypesProto2.Builder builder =
+                  TestMessagesProto2.TestAllTypesProto2.newBuilder();
+              TextFormat.merge(request.getTextPayload(), builder);
+              testMessage = builder.build();
+            } catch (TextFormat.ParseException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+            }
+          } else {
+            throw new IllegalArgumentException(
+               "Protobuf request has unexpected payload type: " + messageType);
+          }
+          break;
+        }
+      case PAYLOAD_NOT_SET:
+        {
+          throw new IllegalArgumentException("Request didn't have payload.");
+        }
+
+      default:
+        {
+          throw new IllegalArgumentException("Unexpected payload case.");
+        }
+    }
+
+    switch (request.getRequestedOutputFormat()) {
+      case UNSPECIFIED:
+        throw new IllegalArgumentException("Unspecified output format.");
+
+      case PROTOBUF:
+        {
+          ByteString messageString = testMessage.toByteString();
+          return Conformance.ConformanceResponse.newBuilder()
+              .setProtobufPayload(messageString)
+              .build();
+        }
+
+      case JSON:
+        try {
+          return Conformance.ConformanceResponse.newBuilder()
+              .setJsonPayload(
+                  JsonFormat.printer().usingTypeRegistry(typeRegistry).print(testMessage))
+              .build();
+        } catch (InvalidProtocolBufferException | IllegalArgumentException e) {
+          return Conformance.ConformanceResponse.newBuilder()
+              .setSerializeError(e.getMessage())
+              .build();
+        }
+
+      case TEXT_FORMAT:
+        return Conformance.ConformanceResponse.newBuilder()
+            .setTextPayload(TextFormat.printer().printToString(testMessage))
+            .build();
+
+      default:
+        {
+          throw new IllegalArgumentException("Unexpected request output.");
+        }
+    }
+  }
+
+  private boolean doTestIo() throws Exception {
+    int bytes = readLittleEndianIntFromStdin();
+
+    if (bytes == -1) {
+      return false; // EOF
+    }
+
+    byte[] serializedInput = new byte[bytes];
+
+    if (!readFromStdin(serializedInput, bytes)) {
+      throw new RuntimeException("Unexpected EOF from test program.");
+    }
+
+    Conformance.ConformanceRequest request =
+        Conformance.ConformanceRequest.parseFrom(serializedInput);
+    Conformance.ConformanceResponse response = doTest(request);
+    byte[] serializedOutput = response.toByteArray();
+
+    writeLittleEndianIntToStdout(serializedOutput.length);
+    writeToStdout(serializedOutput);
+
+    return true;
+  }
+
+  public void run() throws Exception {
+    typeRegistry =
+        TypeRegistry.newBuilder()
+            .add(TestMessagesProto3.TestAllTypesProto3.getDescriptor())
+            .build();
+    while (doTestIo()) {
+      this.testCount++;
+    }
+
+    System.err.println(
+        "ConformanceJava: received EOF from test runner after " + this.testCount + " tests");
+  }
+
+  public static void main(String[] args) throws Exception {
+    new ConformanceJava().run();
+  }
+}

conformance/ConformanceJavaLite.java

@@ -0,0 +1,351 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.ExtensionRegistryLite;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.MessageLite;
+import com.google.protobuf.Parser;
+import com.google.protobuf.conformance.Conformance;
+import com.google.protobuf_test_messages.proto2.TestMessagesProto2;
+import com.google.protobuf_test_messages.proto2.TestMessagesProto2.TestAllTypesProto2;
+import com.google.protobuf_test_messages.proto3.TestMessagesProto3;
+import com.google.protobuf_test_messages.proto3.TestMessagesProto3.TestAllTypesProto3;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+class ConformanceJavaLite {
+  private int testCount = 0;
+
+  private boolean readFromStdin(byte[] buf, int len) throws Exception {
+    int ofs = 0;
+    while (len > 0) {
+      int read = System.in.read(buf, ofs, len);
+      if (read == -1) {
+        return false; // EOF
+      }
+      ofs += read;
+      len -= read;
+    }
+
+    return true;
+  }
+
+  private void writeToStdout(byte[] buf) throws Exception {
+    System.out.write(buf);
+  }
+
+  // Returns -1 on EOF (the actual values will always be positive).
+  private int readLittleEndianIntFromStdin() throws Exception {
+    byte[] buf = new byte[4];
+    if (!readFromStdin(buf, 4)) {
+      return -1;
+    }
+    return (buf[0] & 0xff)
+        | ((buf[1] & 0xff) << 8)
+        | ((buf[2] & 0xff) << 16)
+        | ((buf[3] & 0xff) << 24);
+  }
+
+  private void writeLittleEndianIntToStdout(int val) throws Exception {
+    byte[] buf = new byte[4];
+    buf[0] = (byte) val;
+    buf[1] = (byte) (val >> 8);
+    buf[2] = (byte) (val >> 16);
+    buf[3] = (byte) (val >> 24);
+    writeToStdout(buf);
+  }
+
+  private enum BinaryDecoderType {
+    BTYE_STRING_DECODER,
+    BYTE_ARRAY_DECODER,
+    ARRAY_BYTE_BUFFER_DECODER,
+    READONLY_ARRAY_BYTE_BUFFER_DECODER,
+    DIRECT_BYTE_BUFFER_DECODER,
+    READONLY_DIRECT_BYTE_BUFFER_DECODER,
+    INPUT_STREAM_DECODER;
+  }
+
+  private static class BinaryDecoder<T extends MessageLite> {
+    public T decode(
+        ByteString bytes,
+        BinaryDecoderType type,
+        Parser<T> parser,
+        ExtensionRegistryLite extensions)
+        throws InvalidProtocolBufferException {
+      switch (type) {
+        case BTYE_STRING_DECODER:
+        case BYTE_ARRAY_DECODER:
+          return parser.parseFrom(bytes, extensions);
+        case ARRAY_BYTE_BUFFER_DECODER:
+          {
+            ByteBuffer buffer = ByteBuffer.allocate(bytes.size());
+            bytes.copyTo(buffer);
+            buffer.flip();
+            return parser.parseFrom(CodedInputStream.newInstance(buffer), extensions);
+          }
+        case READONLY_ARRAY_BYTE_BUFFER_DECODER:
+          {
+            return parser.parseFrom(
+                CodedInputStream.newInstance(bytes.asReadOnlyByteBuffer()), extensions);
+          }
+        case DIRECT_BYTE_BUFFER_DECODER:
+          {
+            ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.size());
+            bytes.copyTo(buffer);
+            buffer.flip();
+            return parser.parseFrom(CodedInputStream.newInstance(buffer), extensions);
+          }
+        case READONLY_DIRECT_BYTE_BUFFER_DECODER:
+          {
+            ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.size());
+            bytes.copyTo(buffer);
+            buffer.flip();
+            return parser.parseFrom(
+                CodedInputStream.newInstance(buffer.asReadOnlyBuffer()), extensions);
+          }
+        case INPUT_STREAM_DECODER:
+          {
+            return parser.parseFrom(bytes.newInput(), extensions);
+          }
+      }
+      return null;
+    }
+  }
+
+  private <T extends MessageLite> T parseBinary(
+      ByteString bytes, Parser<T> parser, ExtensionRegistryLite extensions)
+      throws InvalidProtocolBufferException {
+    ArrayList<T> messages = new ArrayList<>();
+    ArrayList<InvalidProtocolBufferException> exceptions = new ArrayList<>();
+
+    for (int i = 0; i < BinaryDecoderType.values().length; i++) {
+      messages.add(null);
+      exceptions.add(null);
+    }
+    if (messages.isEmpty()) {
+      throw new RuntimeException("binary decoder types missing");
+    }
+
+    BinaryDecoder<T> decoder = new BinaryDecoder<>();
+
+    boolean hasMessage = false;
+    boolean hasException = false;
+    for (int i = 0; i < BinaryDecoderType.values().length; ++i) {
+      try {
+        messages.set(i, decoder.decode(bytes, BinaryDecoderType.values()[i], parser, extensions));
+        hasMessage = true;
+      } catch (InvalidProtocolBufferException e) {
+        exceptions.set(i, e);
+        hasException = true;
+      }
+    }
+
+    if (hasMessage && hasException) {
+      StringBuilder sb =
+          new StringBuilder("Binary decoders disagreed on whether the payload was valid.\n");
+      for (int i = 0; i < BinaryDecoderType.values().length; ++i) {
+        sb.append(BinaryDecoderType.values()[i].name());
+        if (messages.get(i) != null) {
+          sb.append(" accepted the payload.\n");
+        } else {
+          sb.append(" rejected the payload.\n");
+        }
+      }
+      throw new RuntimeException(sb.toString());
+    }
+
+    if (hasException) {
+      // We do not check if exceptions are equal. Different implementations may return different
+      // exception messages. Throw an arbitrary one out instead.
+      InvalidProtocolBufferException exception = null;
+      for (InvalidProtocolBufferException e : exceptions) {
+        if (exception != null) {
+          exception.addSuppressed(e);
+        } else {
+          exception = e;
+        }
+      }
+      throw exception;
+    }
+
+    // Fast path comparing all the messages with the first message, assuming equality being
+    // symmetric and transitive.
+    boolean allEqual = true;
+    for (int i = 1; i < messages.size(); ++i) {
+      if (!messages.get(0).equals(messages.get(i))) {
+        allEqual = false;
+        break;
+      }
+    }
+
+    // Slow path: compare and find out all unequal pairs.
+    if (!allEqual) {
+      StringBuilder sb = new StringBuilder();
+      for (int i = 0; i < messages.size() - 1; ++i) {
+        for (int j = i + 1; j < messages.size(); ++j) {
+          if (!messages.get(i).equals(messages.get(j))) {
+            sb.append(BinaryDecoderType.values()[i].name())
+                .append(" and ")
+                .append(BinaryDecoderType.values()[j].name())
+                .append(" parsed the payload differently.\n");
+          }
+        }
+      }
+      throw new RuntimeException(sb.toString());
+    }
+
+    return messages.get(0);
+  }
+
+  private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest request) {
+    com.google.protobuf.MessageLite testMessage;
+    boolean isProto3 =
+        request.getMessageType().equals("protobuf_test_messages.proto3.TestAllTypesProto3");
+    boolean isProto2 =
+        request.getMessageType().equals("protobuf_test_messages.proto2.TestAllTypesProto2");
+
+    switch (request.getPayloadCase()) {
+      case PROTOBUF_PAYLOAD:
+        {
+          if (isProto3) {
+            try {
+              ExtensionRegistryLite extensions = ExtensionRegistryLite.newInstance();
+              TestMessagesProto3.registerAllExtensions(extensions);
+              testMessage =
+                  parseBinary(
+                      request.getProtobufPayload(), TestAllTypesProto3.parser(), extensions);
+            } catch (InvalidProtocolBufferException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+            }
+          } else if (isProto2) {
+            try {
+              ExtensionRegistryLite extensions = ExtensionRegistryLite.newInstance();
+              TestMessagesProto2.registerAllExtensions(extensions);
+              testMessage =
+                  parseBinary(
+                      request.getProtobufPayload(), TestAllTypesProto2.parser(), extensions);
+            } catch (InvalidProtocolBufferException e) {
+              return Conformance.ConformanceResponse.newBuilder()
+                  .setParseError(e.getMessage())
+                  .build();
+            }
+          } else {
+            throw new RuntimeException("Protobuf request doesn't have specific payload type.");
+          }
+          break;
+        }
+      case JSON_PAYLOAD:
+        {
+          return Conformance.ConformanceResponse.newBuilder()
+              .setSkipped("Lite runtime does not support JSON format.")
+              .build();
+        }
+      case TEXT_PAYLOAD:
+        {
+          return Conformance.ConformanceResponse.newBuilder()
+              .setSkipped("Lite runtime does not support Text format.")
+              .build();
+        }
+      case PAYLOAD_NOT_SET:
+        {
+          throw new RuntimeException("Request didn't have payload.");
+        }
+      default:
+        {
+          throw new RuntimeException("Unexpected payload case.");
+        }
+    }
+
+    switch (request.getRequestedOutputFormat()) {
+      case UNSPECIFIED:
+        throw new RuntimeException("Unspecified output format.");
+
+      case PROTOBUF:
+        return Conformance.ConformanceResponse.newBuilder()
+            .setProtobufPayload(testMessage.toByteString())
+            .build();
+
+      case JSON:
+        return Conformance.ConformanceResponse.newBuilder()
+            .setSkipped("Lite runtime does not support JSON format.")
+            .build();
+
+      case TEXT_FORMAT:
+        return Conformance.ConformanceResponse.newBuilder()
+            .setSkipped("Lite runtime does not support Text format.")
+            .build();
+      default:
+        {
+          throw new RuntimeException("Unexpected request output.");
+        }
+    }
+  }
+
+  private boolean doTestIo() throws Exception {
+    int bytes = readLittleEndianIntFromStdin();
+
+    if (bytes == -1) {
+      return false; // EOF
+    }
+
+    byte[] serializedInput = new byte[bytes];
+
+    if (!readFromStdin(serializedInput, bytes)) {
+      throw new RuntimeException("Unexpected EOF from test program.");
+    }
+
+    Conformance.ConformanceRequest request =
+        Conformance.ConformanceRequest.parseFrom(serializedInput);
+    Conformance.ConformanceResponse response = doTest(request);
+    byte[] serializedOutput = response.toByteArray();
+
+    writeLittleEndianIntToStdout(serializedOutput.length);
+    writeToStdout(serializedOutput);
+
+    return true;
+  }
+
+  public void run() throws Exception {
+    while (doTestIo()) {
+      this.testCount++;
+    }
+
+    System.err.println(
+        "ConformanceJavaLite: received EOF from test runner after " + this.testCount + " tests");
+  }
+
+  public static void main(String[] args) throws Exception {
+    new ConformanceJavaLite().run();
+  }
+}

conformance/README.md

@@ -0,0 +1,104 @@
+Protocol Buffers - Google's data interchange format
+===================================================
+
+Copyright 2008 Google Inc.
+
+This directory contains conformance tests for testing completeness and
+correctness of Protocol Buffers implementations.  These tests are designed
+to be easy to run against any Protocol Buffers implementation.
+
+This directory contains the tester process `conformance-test`, which
+contains all of the tests themselves.  Then separate programs written
+in whatever language you want to test communicate with the tester
+program over a pipe.
+
+If you're not using Bazel to run these tests, make sure you build the C++
+tester code beforehand, e.g. from the base directory:
+
+    $ cmake . -Dprotobuf_BUILD_CONFORMANCE=ON && cmake --build .
+
+This will produce a `conformance_test_runner` binary that can be used to run
+conformance tests on any executable.  Pass it `--help` for more information.
+
+Running the tests for C++
+-------------------------
+
+To run the tests against the C++ implementation, run:
+
+    $ bazel test //src:conformance_test
+
+Or alternatively with CMake:
+
+    $ ctest -R conformance_cpp_test
+
+Running the tests for other languages
+-------------------------------------
+
+All of the languages in the Protobuf source tree are set up to run conformance
+tests using similar patterns.  You can either use Bazel to run the
+`conformance_test` target defined in the language's root `BUILD.bazel` file,
+or create an executable for a custom test and pass it to
+`conformance_test_runner`.
+
+Note: CMake can be used to build the conformance test runner, but not any of
+the conformance test executables outside C++.  So if you aren't using Bazel
+you'll need to create the executable you pass to `conformance_test_runner` via
+some alternate build system.
+
+While we plan to model all our supported languages more completely in Bazel,
+today some of them are a bit tricky to run.  Below is a list of the commands
+(and prerequisites) to run each language's conformance tests.
+
+Java:
+
+    $ bazel test //java/core:conformance_test //java/lite:conformance_test
+
+Python:
+
+    $ bazel test //python:conformance_test
+
+Python C++:
+
+    $ bazel test //python:conformance_test_cpp --define=use_fast_cpp_protos=true
+
+C#:
+
+    $ `which dotnet || echo "You must have dotnet installed!"
+    $ `bazel test //csharp:conformance_test \
+        --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=~ \
+        --action_env=DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
+
+Objective-c (Mac only):
+
+    $ `bazel test //objectivec:conformance_test --macos_minimum_os=10.9
+
+Ruby:
+
+    $ [[ $(ruby --version) == "ruby"* ]] || echo "Select a C Ruby!"
+    $ bazel test //ruby:conformance_test --define=ruby_platform=c \
+        --action_env=PATH --action_env=GEM_PATH --action_env=GEM_HOME
+
+JRuby:
+
+    $ [[ $(ruby --version) == "jruby"* ]] || echo "Switch to Java Ruby!"
+    $ bazel test //ruby:conformance_test_jruby --define=ruby_platform=java \
+        --action_env=PATH --action_env=GEM_PATH --action_env=GEM_HOME
+
+Testing other Protocol Buffer implementations
+---------------------------------------------
+
+To run these tests against a new Protocol Buffers implementation, write a
+program in your language that uses the protobuf implementation you want
+to test.  This program should implement the testing protocol defined in
+[conformance.proto](https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance.proto).
+This is designed to be as easy as possible: the C++ version is only
+150 lines and is a good example for what this program should look like
+(see [conformance_cpp.cc](https://github.com/protocolbuffers/protobuf/blob/main/conformance/conformance_cpp.cc)).
+The program only needs to be able to read from stdin and write to stdout.
+
+Portability
+-----------
+
+Note that the test runner currently does not work on Windows.  Patches
+to fix this are welcome!  (But please get in touch first to settle on
+a general implementation strategy).

conformance/autoload.php

@@ -0,0 +1,21 @@
+<?php
+
+define("GOOGLE_INTERNAL_NAMESPACE", "Google\\Protobuf\\Internal\\");
+define("GOOGLE_NAMESPACE", "Google\\Protobuf\\");
+define("GOOGLE_GPBMETADATA_NAMESPACE", "GPBMetadata\\Google\\Protobuf\\");
+
+function protobuf_autoloader_impl($class, $prefix) {
+    $length = strlen($prefix);
+    if ((substr($class, 0, $length) === $prefix)) {
+        $path = 'php/src/' . implode('/', array_map('ucwords', explode('\\', $class))) . '.php';
+        include_once $path;
+    }
+}
+
+function protobuf_autoloader($class) {
+    protobuf_autoloader_impl($class, GOOGLE_INTERNAL_NAMESPACE);
+    protobuf_autoloader_impl($class, GOOGLE_NAMESPACE);
+    protobuf_autoloader_impl($class, GOOGLE_GPBMETADATA_NAMESPACE);
+}
+
+spl_autoload_register('protobuf_autoloader');

conformance/bazel_conformance_test_runner.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# This is an internal file that should only be called from Bazel rules.  For
+# custom conformance tests outside of Bazel use CMAKE with
+# -Dprotobuf_BUILD_CONFORMANCE=ON to build the test runner.
+
+set -x
+echo $@
+
+set -euo pipefail
+# --- begin runfiles.bash initialization ---
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+    if [[ -f "$0.runfiles_manifest" ]]; then
+      export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+    elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+      export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+    elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+      export RUNFILES_DIR="$0.runfiles"
+    fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+  exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+TESTEE=unset
+FAILURE_LIST=unset
+TEXT_FORMAT_FAILURE_LIST=unset
+
+while [[ -n "$@" ]]; do
+  arg="$1"; shift
+  val="$1"; shift
+  case "$arg" in
+    "--testee") TESTEE="$val" ;;
+    "--failure_list") FAILURE_LIST="$val" ;;
+    "--text_format_failure_list") TEXT_FORMAT_FAILURE_LIST="$val" ;;
+    *) echo "Flag $arg is not recognized." && exit 1 ;;
+  esac
+done
+
+conformance_test_runner=$(rlocation com_google_protobuf/conformance/conformance_test_runner)
+conformance_testee=$(rlocation $TESTEE)
+args=(--enforce_recommended)
+
+failure_list=$(rlocation $FAILURE_LIST) || unset
+if [ -n "$failure_list" ] ; then
+  args+=(--failure_list $failure_list)
+fi
+
+text_format_failure_list=$(rlocation $TEXT_FORMAT_FAILURE_LIST) || unset
+if [ -n "$text_format_failure_list" ]; then
+  args+=(--text_format_failure_list $text_format_failure_list)
+fi
+
+$conformance_test_runner "${args[@]}" $conformance_testee

conformance/binary_json_conformance_suite.h

@@ -0,0 +1,164 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H
+#define CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H
+
+#include "google/protobuf/descriptor.h"
+#include "json/json.h"
+#include "conformance_test.h"
+
+namespace google {
+namespace protobuf {
+
+class BinaryAndJsonConformanceSuite : public ConformanceTestSuite {
+ public:
+  BinaryAndJsonConformanceSuite() {}
+
+ private:
+  void RunSuiteImpl() override;
+  void RunBinaryPerformanceTests();
+  void RunJsonPerformanceTests();
+  void RunJsonTests();
+  void RunJsonTestsForFieldNameConvention();
+  void RunJsonTestsForNonRepeatedTypes();
+  void RunJsonTestsForRepeatedTypes();
+  void RunJsonTestsForNullTypes();
+  void RunJsonTestsForWrapperTypes();
+  void RunJsonTestsForFieldMask();
+  void RunJsonTestsForStruct();
+  void RunJsonTestsForValue();
+  void RunJsonTestsForAny();
+  void RunValidJsonTest(const std::string& test_name, ConformanceLevel level,
+                        const std::string& input_json,
+                        const std::string& equivalent_text_format);
+  void RunValidJsonTest(const std::string& test_name, ConformanceLevel level,
+                        const std::string& input_json,
+                        const std::string& equivalent_text_format,
+                        bool is_proto3);
+  void RunValidJsonTestWithMessage(const std::string& test_name,
+                                   ConformanceLevel level,
+                                   const std::string& input_json,
+                                   const std::string& equivalent_text_forma,
+                                   const Message& prototype);
+  void RunValidJsonTestWithProtobufInput(
+      const std::string& test_name, ConformanceLevel level,
+      const protobuf_test_messages::proto3::TestAllTypesProto3& input,
+      const std::string& equivalent_text_format);
+  void RunValidJsonIgnoreUnknownTest(const std::string& test_name,
+                                     ConformanceLevel level,
+                                     const std::string& input_json,
+                                     const std::string& equivalent_text_format);
+  void RunValidProtobufTest(const std::string& test_name,
+                            ConformanceLevel level,
+                            const std::string& input_protobuf,
+                            const std::string& equivalent_text_format,
+                            bool is_proto3);
+  void RunValidBinaryProtobufTest(const std::string& test_name,
+                                  ConformanceLevel level,
+                                  const std::string& input_protobuf,
+                                  bool is_proto3);
+  void RunValidBinaryProtobufTest(const std::string& test_name,
+                                  ConformanceLevel level,
+                                  const std::string& input_protobuf,
+                                  const std::string& expected_protobuf,
+                                  bool is_proto3);
+  void RunBinaryPerformanceMergeMessageWithField(const std::string& test_name,
+                                                 const std::string& field_proto,
+                                                 bool is_proto3);
+
+  void RunValidProtobufTestWithMessage(
+      const std::string& test_name, ConformanceLevel level,
+      const Message* input, const std::string& equivalent_text_format,
+      bool is_proto3);
+
+  bool ParseJsonResponse(
+      const conformance::ConformanceResponse& response,
+      Message* test_message);
+  bool ParseResponse(
+      const conformance::ConformanceResponse& response,
+      const ConformanceRequestSetting& setting,
+      Message* test_message) override;
+
+  typedef std::function<bool(const Json::Value&)> Validator;
+  void RunValidJsonTestWithValidator(const std::string& test_name,
+                                     ConformanceLevel level,
+                                     const std::string& input_json,
+                                     const Validator& validator,
+                                     bool is_proto3);
+  void ExpectParseFailureForJson(const std::string& test_name,
+                                 ConformanceLevel level,
+                                 const std::string& input_json);
+  void ExpectSerializeFailureForJson(const std::string& test_name,
+                                     ConformanceLevel level,
+                                     const std::string& text_format);
+  void ExpectParseFailureForProtoWithProtoVersion(const std::string& proto,
+                                                  const std::string& test_name,
+                                                  ConformanceLevel level,
+                                                  bool is_proto3);
+  void ExpectParseFailureForProto(const std::string& proto,
+                                  const std::string& test_name,
+                                  ConformanceLevel level);
+  void ExpectHardParseFailureForProto(const std::string& proto,
+                                      const std::string& test_name,
+                                      ConformanceLevel level);
+  void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type);
+  void TestIllegalTags();
+  template <class MessageType>
+  void TestOneofMessage (MessageType &message,
+                         bool is_proto3);
+  template <class MessageType>
+  void TestUnknownMessage (MessageType &message,
+                           bool is_proto3);
+  void TestValidDataForType(
+      google::protobuf::FieldDescriptor::Type,
+      std::vector<std::pair<std::string, std::string>> values);
+  void TestValidDataForRepeatedScalarMessage();
+  void TestValidDataForMapType(google::protobuf::FieldDescriptor::Type,
+                               google::protobuf::FieldDescriptor::Type);
+  void TestValidDataForOneofType(google::protobuf::FieldDescriptor::Type);
+  void TestMergeOneofMessage();
+  void TestOverwriteMessageValueMap();
+  void TestBinaryPerformanceForAlternatingUnknownFields();
+  void TestBinaryPerformanceMergeMessageWithRepeatedFieldForType(
+      google::protobuf::FieldDescriptor::Type);
+  void TestBinaryPerformanceMergeMessageWithUnknownFieldForType(
+      google::protobuf::FieldDescriptor::Type);
+  void TestJsonPerformanceMergeMessageWithRepeatedFieldForType(
+      google::protobuf::FieldDescriptor::Type, std::string field_value);
+
+  std::unique_ptr<google::protobuf::util::TypeResolver> type_resolver_;
+  std::string type_url_;
+};
+
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H

conformance/conformance.proto

@@ -0,0 +1,180 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package conformance;
+
+option java_package = "com.google.protobuf.conformance";
+option objc_class_prefix = "Conformance";
+
+// This defines the conformance testing protocol.  This protocol exists between
+// the conformance test suite itself and the code being tested.  For each test,
+// the suite will send a ConformanceRequest message and expect a
+// ConformanceResponse message.
+//
+// You can either run the tests in two different ways:
+//
+//   1. in-process (using the interface in conformance_test.h).
+//
+//   2. as a sub-process communicating over a pipe.  Information about how to
+//      do this is in conformance_test_runner.cc.
+//
+// Pros/cons of the two approaches:
+//
+//   - running as a sub-process is much simpler for languages other than C/C++.
+//
+//   - running as a sub-process may be more tricky in unusual environments like
+//     iOS apps, where fork/stdin/stdout are not available.
+
+enum WireFormat {
+  UNSPECIFIED = 0;
+  PROTOBUF = 1;
+  JSON = 2;
+  JSPB = 3;  // Google internal only. Opensource testees just skip it.
+  TEXT_FORMAT = 4;
+}
+
+enum TestCategory {
+  UNSPECIFIED_TEST = 0;
+  BINARY_TEST = 1;  // Test binary wire format.
+  JSON_TEST = 2;    // Test json wire format.
+  // Similar to JSON_TEST. However, during parsing json, testee should ignore
+  // unknown fields. This feature is optional. Each implementation can decide
+  // whether to support it.  See
+  // https://developers.google.com/protocol-buffers/docs/proto3#json_options
+  // for more detail.
+  JSON_IGNORE_UNKNOWN_PARSING_TEST = 3;
+  // Test jspb wire format. Google internal only. Opensource testees just skip
+  // it.
+  JSPB_TEST = 4;
+  // Test text format. For cpp, java and python, testees can already deal with
+  // this type. Testees of other languages can simply skip it.
+  TEXT_FORMAT_TEST = 5;
+}
+
+// The conformance runner will request a list of failures as the first request.
+// This will be known by message_type == "conformance.FailureSet", a conformance
+// test should return a serialized FailureSet in protobuf_payload.
+message FailureSet {
+  repeated string failure = 1;
+}
+
+// Represents a single test case's input.  The testee should:
+//
+//   1. parse this proto (which should always succeed)
+//   2. parse the protobuf or JSON payload in "payload" (which may fail)
+//   3. if the parse succeeded, serialize the message in the requested format.
+message ConformanceRequest {
+  // The payload (whether protobuf of JSON) is always for a
+  // protobuf_test_messages.proto3.TestAllTypes proto (as defined in
+  // src/google/protobuf/proto3_test_messages.proto).
+  oneof payload {
+    bytes protobuf_payload = 1;
+    string json_payload = 2;
+    // Google internal only.  Opensource testees just skip it.
+    string jspb_payload = 7;
+    string text_payload = 8;
+  }
+
+  // Which format should the testee serialize its message to?
+  WireFormat requested_output_format = 3;
+
+  // The full name for the test message to use; for the moment, either:
+  // protobuf_test_messages.proto3.TestAllTypesProto3 or
+  // protobuf_test_messages.google.protobuf.TestAllTypesProto2.
+  string message_type = 4;
+
+  // Each test is given a specific test category. Some category may need
+  // specific support in testee programs. Refer to the definition of
+  // TestCategory for more information.
+  TestCategory test_category = 5;
+
+  // Specify details for how to encode jspb.
+  JspbEncodingConfig jspb_encoding_options = 6;
+
+  // This can be used in json and text format. If true, testee should print
+  // unknown fields instead of ignore. This feature is optional.
+  bool print_unknown_fields = 9;
+}
+
+// Represents a single test case's output.
+message ConformanceResponse {
+  oneof result {
+    // This string should be set to indicate parsing failed.  The string can
+    // provide more information about the parse error if it is available.
+    //
+    // Setting this string does not necessarily mean the testee failed the
+    // test.  Some of the test cases are intentionally invalid input.
+    string parse_error = 1;
+
+    // If the input was successfully parsed but errors occurred when
+    // serializing it to the requested output format, set the error message in
+    // this field.
+    string serialize_error = 6;
+
+    // This should be set if the test program timed out.  The string should
+    // provide more information about what the child process was doing when it
+    // was killed.
+    string timeout_error = 9;
+
+    // This should be set if some other error occurred.  This will always
+    // indicate that the test failed.  The string can provide more information
+    // about the failure.
+    string runtime_error = 2;
+
+    // If the input was successfully parsed and the requested output was
+    // protobuf, serialize it to protobuf and set it in this field.
+    bytes protobuf_payload = 3;
+
+    // If the input was successfully parsed and the requested output was JSON,
+    // serialize to JSON and set it in this field.
+    string json_payload = 4;
+
+    // For when the testee skipped the test, likely because a certain feature
+    // wasn't supported, like JSON input/output.
+    string skipped = 5;
+
+    // If the input was successfully parsed and the requested output was JSPB,
+    // serialize to JSPB and set it in this field. JSPB is google internal only
+    // format. Opensource testees can just skip it.
+    string jspb_payload = 7;
+
+    // If the input was successfully parsed and the requested output was
+    // TEXT_FORMAT, serialize to TEXT_FORMAT and set it in this field.
+    string text_payload = 8;
+  }
+}
+
+// Encoding options for jspb format.
+message JspbEncodingConfig {
+  // Encode the value field of Any as jspb array if true, otherwise binary.
+  bool use_jspb_array_any_format = 1;
+}

conformance/conformance_cpp.cc

@@ -0,0 +1,274 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <errno.h>
+#include <stdarg.h>
+#include <unistd.h>
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "google/protobuf/stubs/logging.h"
+#include "google/protobuf/stubs/common.h"
+#include "google/protobuf/message.h"
+#include "google/protobuf/text_format.h"
+#include "google/protobuf/util/json_util.h"
+#include "google/protobuf/util/type_resolver_util.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "conformance/conformance.pb.h"
+#include "conformance/conformance.pb.h"
+#include "google/protobuf/test_messages_proto2.pb.h"
+#include "google/protobuf/test_messages_proto3.pb.h"
+#include "google/protobuf/test_messages_proto3.pb.h"
+#include "google/protobuf/util/type_resolver.h"
+#include "google/protobuf/stubs/status_macros.h"
+
+// Must be included last.
+#include "google/protobuf/port_def.inc"
+
+namespace google {
+namespace protobuf {
+namespace {
+using ::conformance::ConformanceRequest;
+using ::conformance::ConformanceResponse;
+using ::google::protobuf::util::BinaryToJsonString;
+using ::google::protobuf::util::JsonParseOptions;
+using ::google::protobuf::util::JsonToBinaryString;
+using ::google::protobuf::util::NewTypeResolverForDescriptorPool;
+using ::google::protobuf::util::TypeResolver;
+using ::protobuf_test_messages::proto2::TestAllTypesProto2;
+using ::protobuf_test_messages::proto3::TestAllTypesProto3;
+
+absl::Status ReadFd(int fd, char* buf, size_t len) {
+  while (len > 0) {
+    ssize_t bytes_read = read(fd, buf, len);
+
+    if (bytes_read == 0) {
+      return absl::DataLossError("unexpected EOF");
+    }
+
+    if (bytes_read < 0) {
+      return absl::ErrnoToStatus(errno, "error reading from test runner");
+    }
+
+    len -= bytes_read;
+    buf += bytes_read;
+  }
+  return absl::OkStatus();
+}
+
+absl::Status WriteFd(int fd, const void* buf, size_t len) {
+  if (static_cast<size_t>(write(fd, buf, len)) != len) {
+    return absl::ErrnoToStatus(errno, "error reading to test runner");
+  }
+  return absl::OkStatus();
+}
+
+class Harness {
+ public:
+  Harness() {
+    google::protobuf::LinkMessageReflection<TestAllTypesProto2>();
+    google::protobuf::LinkMessageReflection<TestAllTypesProto3>();
+
+    resolver_.reset(NewTypeResolverForDescriptorPool(
+        "type.googleapis.com", DescriptorPool::generated_pool()));
+    type_url_ = absl::StrCat("type.googleapis.com/",
+                             TestAllTypesProto3::GetDescriptor()->full_name());
+  }
+
+  absl::StatusOr<ConformanceResponse> RunTest(
+      const ConformanceRequest& request);
+
+  // Returns Ok(true) if we're done processing requests.
+  absl::StatusOr<bool> ServeConformanceRequest();
+
+ private:
+  bool verbose_ = false;
+  std::unique_ptr<TypeResolver> resolver_;
+  std::string type_url_;
+};
+
+absl::StatusOr<ConformanceResponse> Harness::RunTest(
+    const ConformanceRequest& request) {
+  const Descriptor* descriptor =
+      DescriptorPool::generated_pool()->FindMessageTypeByName(
+          request.message_type());
+  if (descriptor == nullptr) {
+    return absl::NotFoundError(
+        absl::StrCat("No such message type: ", request.message_type()));
+  }
+
+  std::unique_ptr<Message> test_message(
+      MessageFactory::generated_factory()->GetPrototype(descriptor)->New());
+  ConformanceResponse response;
+
+  switch (request.payload_case()) {
+    case ConformanceRequest::kProtobufPayload: {
+      if (!test_message->ParseFromString(request.protobuf_payload())) {
+        response.set_parse_error("parse error (no more details available)");
+        return response;
+      }
+      break;
+    }
+
+    case ConformanceRequest::kJsonPayload: {
+      JsonParseOptions options;
+      options.ignore_unknown_fields =
+          (request.test_category() ==
+           conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST);
+
+      std::string proto_binary;
+      absl::Status status =
+          JsonToBinaryString(resolver_.get(), type_url_, request.json_payload(),
+                             &proto_binary, options);
+      if (!status.ok()) {
+        response.set_parse_error(
+            absl::StrCat("parse error: ", status.message()));
+        return response;
+      }
+
+      if (!test_message->ParseFromString(proto_binary)) {
+        response.set_runtime_error(
+            "parsing JSON generated invalid proto output");
+        return response;
+      }
+
+      break;
+    }
+
+    case ConformanceRequest::kTextPayload: {
+      if (!TextFormat::ParseFromString(request.text_payload(),
+                                       test_message.get())) {
+        response.set_parse_error("parse error (no more details available)");
+        return response;
+      }
+      break;
+    }
+
+    case ConformanceRequest::PAYLOAD_NOT_SET:
+      return absl::InvalidArgumentError("request didn't have payload");
+
+    default:
+      return absl::InvalidArgumentError(
+          absl::StrCat("unknown payload type", request.payload_case()));
+  }
+
+  switch (request.requested_output_format()) {
+    case conformance::UNSPECIFIED:
+      return absl::InvalidArgumentError("unspecified output format");
+
+    case conformance::PROTOBUF: {
+      GOOGLE_CHECK(
+          test_message->SerializeToString(response.mutable_protobuf_payload()));
+      break;
+    }
+
+    case conformance::JSON: {
+      std::string proto_binary;
+      GOOGLE_CHECK(test_message->SerializeToString(&proto_binary));
+      absl::Status status =
+          BinaryToJsonString(resolver_.get(), type_url_, proto_binary,
+                             response.mutable_json_payload());
+      if (!status.ok()) {
+        response.set_serialize_error(absl::StrCat(
+            "failed to serialize JSON output: ", status.message()));
+      }
+      break;
+    }
+
+    case conformance::TEXT_FORMAT: {
+      TextFormat::Printer printer;
+      printer.SetHideUnknownFields(!request.print_unknown_fields());
+      GOOGLE_CHECK(printer.PrintToString(*test_message,
+                                  response.mutable_text_payload()));
+      break;
+    }
+
+    default:
+      return absl::InvalidArgumentError(absl::StrCat(
+          "unknown output format", request.requested_output_format()));
+  }
+
+  return response;
+}
+
+absl::StatusOr<bool> Harness::ServeConformanceRequest() {
+  uint32_t in_len;
+  if (!ReadFd(STDIN_FILENO, reinterpret_cast<char*>(&in_len), sizeof(in_len))
+           .ok()) {
+    // EOF means we're done.
+    return true;
+  }
+
+  std::string serialized_input;
+  serialized_input.resize(in_len);
+  RETURN_IF_ERROR(ReadFd(STDIN_FILENO, &serialized_input[0], in_len));
+
+  ConformanceRequest request;
+  GOOGLE_CHECK(request.ParseFromString(serialized_input));
+
+  absl::StatusOr<ConformanceResponse> response = RunTest(request);
+  RETURN_IF_ERROR(response.status());
+
+  std::string serialized_output;
+  response->SerializeToString(&serialized_output);
+
+  uint32_t out_len = static_cast<uint32_t>(serialized_output.size());
+  RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, &out_len, sizeof(out_len)));
+  RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, serialized_output.data(), out_len));
+
+  if (verbose_) {
+    GOOGLE_LOG(INFO) << "conformance-cpp: request=" << request.ShortDebugString()
+              << ", response=" << response->ShortDebugString();
+  }
+  return false;
+}
+}  // namespace
+}  // namespace protobuf
+}  // namespace google
+
+int main() {
+  google::protobuf::Harness harness;
+  int total_runs = 0;
+  while (true) {
+    auto is_done = harness.ServeConformanceRequest();
+    if (!is_done.ok()) {
+      GOOGLE_LOG(FATAL) << is_done.status();
+    }
+    if (*is_done) {
+      break;
+    }
+    total_runs++;
+  }
+  GOOGLE_LOG(INFO) << "conformance-cpp: received EOF from test runner after "
+            << total_runs << " tests";
+}

conformance/conformance_objc.m

@@ -0,0 +1,212 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2015 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#import <Foundation/Foundation.h>
+
+#import "Conformance.pbobjc.h"
+#import "google/protobuf/TestMessagesProto2.pbobjc.h"
+#import "google/protobuf/TestMessagesProto3.pbobjc.h"
+
+static void Die(NSString *format, ...) __dead2;
+
+static BOOL verbose = NO;
+static int32_t testCount = 0;
+
+static void Die(NSString *format, ...) {
+  va_list args;
+  va_start(args, format);
+  NSString *msg = [[NSString alloc] initWithFormat:format arguments:args];
+  NSLog(@"%@", msg);
+  va_end(args);
+  [msg release];
+  exit(66);
+}
+
+static NSData *CheckedReadDataOfLength(NSFileHandle *handle, NSUInteger numBytes) {
+  NSData *data = [handle readDataOfLength:numBytes];
+  NSUInteger dataLen = data.length;
+  if (dataLen == 0) {
+    return nil;  // EOF.
+  }
+  if (dataLen != numBytes) {
+    Die(@"Failed to read the request length (%d), only got: %@", numBytes, data);
+  }
+  return data;
+}
+
+static ConformanceResponse *DoTest(ConformanceRequest *request) {
+  ConformanceResponse *response = [ConformanceResponse message];
+  GPBMessage *testMessage = nil;
+
+  switch (request.payloadOneOfCase) {
+    case ConformanceRequest_Payload_OneOfCase_GPBUnsetOneOfCase:
+      response.runtimeError =
+          [NSString stringWithFormat:@"Request didn't have a payload: %@", request];
+      break;
+
+    case ConformanceRequest_Payload_OneOfCase_ProtobufPayload: {
+      Class msgClass = nil;
+      if ([request.messageType isEqual:@"protobuf_test_messages.proto3.TestAllTypesProto3"]) {
+        msgClass = [Proto3TestAllTypesProto3 class];
+      } else if ([request.messageType
+                     isEqual:@"protobuf_test_messages.proto2.TestAllTypesProto2"]) {
+        msgClass = [Proto2TestAllTypesProto2 class];
+      } else {
+        response.runtimeError =
+            [NSString stringWithFormat:@"Protobuf request had an unknown message_type: %@",
+                                       request.messageType];
+        break;
+      }
+      NSError *error = nil;
+      testMessage = [msgClass parseFromData:request.protobufPayload error:&error];
+      if (!testMessage) {
+        response.parseError = [NSString stringWithFormat:@"Parse error: %@", error];
+      }
+      break;
+    }
+
+    case ConformanceRequest_Payload_OneOfCase_JsonPayload:
+      response.skipped = @"ObjC doesn't support parsing JSON";
+      break;
+
+    case ConformanceRequest_Payload_OneOfCase_JspbPayload:
+      response.skipped = @"ConformanceRequest had a jspb_payload ConformanceRequest.payload;"
+                          " those aren't supposed to happen with opensource.";
+      break;
+
+    case ConformanceRequest_Payload_OneOfCase_TextPayload:
+      response.skipped = @"ObjC doesn't support parsing TextFormat";
+      break;
+  }
+
+  if (testMessage) {
+    switch (request.requestedOutputFormat) {
+      case ConformanceWireFormat_GPBUnrecognizedEnumeratorValue:
+      case ConformanceWireFormat_Unspecified:
+        response.runtimeError =
+            [NSString stringWithFormat:@"Unrecognized/unspecified output format: %@", request];
+        break;
+
+      case ConformanceWireFormat_Protobuf:
+        response.protobufPayload = testMessage.data;
+        if (!response.protobufPayload) {
+          response.serializeError =
+              [NSString stringWithFormat:@"Failed to make data from: %@", testMessage];
+        }
+        break;
+
+      case ConformanceWireFormat_Json:
+        response.skipped = @"ObjC doesn't support generating JSON";
+        break;
+
+      case ConformanceWireFormat_Jspb:
+        response.skipped =
+            @"ConformanceRequest had a requested_output_format of JSPB WireFormat; that"
+             " isn't supposed to happen with opensource.";
+        break;
+
+      case ConformanceWireFormat_TextFormat:
+        // ObjC only has partial objc generation, so don't attempt any tests that need
+        // support.
+        response.skipped = @"ObjC doesn't support generating TextFormat";
+        break;
+    }
+  }
+
+  return response;
+}
+
+static uint32_t UInt32FromLittleEndianData(NSData *data) {
+  if (data.length != sizeof(uint32_t)) {
+    Die(@"Data not the right size for uint32_t: %@", data);
+  }
+  uint32_t value;
+  memcpy(&value, data.bytes, sizeof(uint32_t));
+  return CFSwapInt32LittleToHost(value);
+}
+
+static NSData *UInt32ToLittleEndianData(uint32_t num) {
+  uint32_t value = CFSwapInt32HostToLittle(num);
+  return [NSData dataWithBytes:&value length:sizeof(uint32_t)];
+}
+
+static BOOL DoTestIo(NSFileHandle *input, NSFileHandle *output) {
+  // See conformance_test_runner.cc for the wire format.
+  NSData *data = CheckedReadDataOfLength(input, sizeof(uint32_t));
+  if (!data) {
+    // EOF.
+    return NO;
+  }
+  uint32_t numBytes = UInt32FromLittleEndianData(data);
+  data = CheckedReadDataOfLength(input, numBytes);
+  if (!data) {
+    Die(@"Failed to read request");
+  }
+
+  NSError *error = nil;
+  ConformanceRequest *request = [ConformanceRequest parseFromData:data error:&error];
+  if (!request) {
+    Die(@"Failed to parse the message data: %@", error);
+  }
+
+  ConformanceResponse *response = DoTest(request);
+  if (!response) {
+    Die(@"Failed to make a reply from %@", request);
+  }
+
+  data = response.data;
+  [output writeData:UInt32ToLittleEndianData((int32_t)data.length)];
+  [output writeData:data];
+
+  if (verbose) {
+    NSLog(@"Request: %@", request);
+    NSLog(@"Response: %@", response);
+  }
+
+  ++testCount;
+  return YES;
+}
+
+int main(int argc, const char *argv[]) {
+  @autoreleasepool {
+    NSFileHandle *input = [[NSFileHandle fileHandleWithStandardInput] retain];
+    NSFileHandle *output = [[NSFileHandle fileHandleWithStandardOutput] retain];
+
+    BOOL notDone = YES;
+    while (notDone) {
+      @autoreleasepool {
+        notDone = DoTestIo(input, output);
+      }
+    }
+
+    NSLog(@"Received EOF from test runner after %d tests, exiting.", testCount);
+  }
+  return 0;
+}

conformance/conformance_php.php

@@ -0,0 +1,120 @@
+<?php
+
+require_once("Conformance/WireFormat.php");
+require_once("Conformance/ConformanceResponse.php");
+require_once("Conformance/ConformanceRequest.php");
+require_once("Conformance/FailureSet.php");
+require_once("Conformance/JspbEncodingConfig.php");
+require_once("Conformance/TestCategory.php");
+require_once("Protobuf_test_messages/Proto3/ForeignMessage.php");
+require_once("Protobuf_test_messages/Proto3/ForeignEnum.php");
+require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3.php");
+require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/AliasedEnum.php");
+require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedMessage.php");
+require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3/NestedEnum.php");
+
+require_once("GPBMetadata/Conformance.php");
+require_once("GPBMetadata/TestMessagesProto3.php");
+
+use  \Conformance\TestCategory;
+use  \Conformance\WireFormat;
+
+if (!ini_get("date.timezone")) {
+  ini_set("date.timezone", "UTC");
+}
+
+$test_count = 0;
+
+function doTest($request)
+{
+    $test_message = new \Protobuf_test_messages\Proto3\TestAllTypesProto3();
+    $response = new \Conformance\ConformanceResponse();
+    if ($request->getPayload() == "protobuf_payload") {
+      if ($request->getMessageType() == "conformance.FailureSet") {
+        $response->setProtobufPayload("");
+        return $response;
+      } elseif ($request->getMessageType() == "protobuf_test_messages.proto3.TestAllTypesProto3") {
+        try {
+          $test_message->mergeFromString($request->getProtobufPayload());
+        } catch (Exception $e) {
+          $response->setParseError($e->getMessage());
+          return $response;
+        }
+      } elseif ($request->getMessageType() == "protobuf_test_messages.proto2.TestAllTypesProto2") {
+        $response->setSkipped("PHP doesn't support proto2");
+        return $response;
+      } else {
+        trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR);
+      }
+    } elseif ($request->getPayload() == "json_payload") {
+      $ignore_json_unknown =
+          ($request->getTestCategory() ==
+              TestCategory::JSON_IGNORE_UNKNOWN_PARSING_TEST);
+      try {
+          $test_message->mergeFromJsonString($request->getJsonPayload(),
+                                             $ignore_json_unknown);
+      } catch (Exception $e) {
+          $response->setParseError($e->getMessage());
+          return $response;
+      }
+	} elseif ($request->getPayload() == "text_payload") {
+		$response->setSkipped("PHP doesn't support text format yet");
+        return $response;
+	} else {
+      trigger_error("Request didn't have payload.", E_USER_ERROR);
+    }
+
+    if ($request->getRequestedOutputFormat() == WireFormat::UNSPECIFIED) {
+      trigger_error("Unspecified output format.", E_USER_ERROR);
+    } elseif ($request->getRequestedOutputFormat() == WireFormat::PROTOBUF) {
+      $response->setProtobufPayload($test_message->serializeToString());
+    } elseif ($request->getRequestedOutputFormat() == WireFormat::JSON) {
+      try {
+          $response->setJsonPayload($test_message->serializeToJsonString());
+      } catch (Exception $e) {
+          $response->setSerializeError($e->getMessage());
+          return $response;
+      }
+    }
+
+    return $response;
+}
+
+function doTestIO()
+{
+    $length_bytes = fread(STDIN, 4);
+    if (strlen($length_bytes) == 0) {
+      return false;   # EOF
+    } elseif (strlen($length_bytes) != 4) {
+      fwrite(STDERR, "I/O error\n");
+      return false;
+    }
+
+    $length = unpack("V", $length_bytes)[1];
+    $serialized_request = fread(STDIN, $length);
+    if (strlen($serialized_request) != $length) {
+      trigger_error("I/O error", E_USER_ERROR);
+    }
+
+    $request = new \Conformance\ConformanceRequest();
+    $request->mergeFromString($serialized_request);
+
+    $response = doTest($request);
+
+    $serialized_response = $response->serializeToString();
+    fwrite(STDOUT, pack("V", strlen($serialized_response)));
+    fwrite(STDOUT, $serialized_response);
+
+    $GLOBALS['test_count'] += 1;
+
+    return true;
+}
+
+while(true){
+  if (!doTestIO()) {
+      fprintf(STDERR,
+             "conformance_php: received EOF from test runner " .
+             "after %d tests, exiting\n", $test_count);
+      exit;
+  }
+}

conformance/conformance_python.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A conformance test implementation for the Python protobuf library.
+
+See conformance.proto for more information.
+"""
+
+import struct
+import sys
+import os
+from google.protobuf import json_format
+from google.protobuf import message
+from google.protobuf import test_messages_proto3_pb2
+from google.protobuf import test_messages_proto2_pb2
+from google.protobuf import text_format
+from conformance import conformance_pb2
+
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'wb', 0)
+sys.stdin = os.fdopen(sys.stdin.fileno(), 'rb', 0)
+
+test_count = 0
+verbose = False
+
+class ProtocolError(Exception):
+  pass
+
+def do_test(request):
+  response = conformance_pb2.ConformanceResponse()
+
+  if request.message_type == "conformance.FailureSet":
+    failure_set = conformance_pb2.FailureSet()
+    failures = []
+    # TODO(gerbens): Remove, this is a hack to detect if the old vs new
+    # parser is used by the cpp code. Relying on a bug in the old parser.
+    hack_proto = test_messages_proto2_pb2.TestAllTypesProto2()
+    old_parser = True
+    try:
+      hack_proto.ParseFromString(b"\322\002\001")
+    except message.DecodeError as e:
+      old_parser = False
+    if old_parser:
+      # the string above is one of the failing conformance test strings of the
+      # old parser. If we succeed the c++ implementation is using the old
+      # parser so we add the list of failing conformance tests.
+      failures = [
+          "Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE",
+          "Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.BOOL",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.DOUBLE",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.ENUM",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.FIXED32",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.FIXED64",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.FLOAT",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.INT32",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.INT64",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SFIXED32",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SFIXED64",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SINT32",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.SINT64",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.UINT32",
+          "Required.Proto3.ProtobufInput.PrematureEofInPackedField.UINT64",
+          "Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE",
+          "Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.BOOL",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.DOUBLE",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.ENUM",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.FIXED32",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.FIXED64",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.FLOAT",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.INT32",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.INT64",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SFIXED32",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SFIXED64",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32",
+          "Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64",
+      ]
+    for x in failures:
+      failure_set.failure.append(x)
+    response.protobuf_payload = failure_set.SerializeToString()
+    return response
+
+  isProto3 = (request.message_type == "protobuf_test_messages.proto3.TestAllTypesProto3")
+  isJson = (request.WhichOneof('payload') == 'json_payload')
+  isProto2 = (request.message_type == "protobuf_test_messages.proto2.TestAllTypesProto2")
+
+  if (not isProto3) and (not isJson) and (not isProto2):
+    raise ProtocolError("Protobuf request doesn't have specific payload type")
+
+  test_message = test_messages_proto2_pb2.TestAllTypesProto2() if isProto2 else \
+    test_messages_proto3_pb2.TestAllTypesProto3()
+
+  try:
+    if request.WhichOneof('payload') == 'protobuf_payload':
+      try:
+        test_message.ParseFromString(request.protobuf_payload)
+      except message.DecodeError as e:
+        response.parse_error = str(e)
+        return response
+
+    elif request.WhichOneof('payload') == 'json_payload':
+      try:
+        ignore_unknown_fields = \
+            request.test_category == \
+                conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST
+        json_format.Parse(request.json_payload, test_message,
+                          ignore_unknown_fields)
+      except Exception as e:
+        response.parse_error = str(e)
+        return response
+
+    elif request.WhichOneof('payload') == 'text_payload':
+      try:
+        text_format.Parse(request.text_payload, test_message)
+      except Exception as e:
+        response.parse_error = str(e)
+        return response
+
+    else:
+      raise ProtocolError("Request didn't have payload.")
+
+    if request.requested_output_format == conformance_pb2.UNSPECIFIED:
+      raise ProtocolError("Unspecified output format")
+
+    elif request.requested_output_format == conformance_pb2.PROTOBUF:
+      response.protobuf_payload = test_message.SerializeToString()
+
+    elif request.requested_output_format == conformance_pb2.JSON:
+      try:
+        response.json_payload = json_format.MessageToJson(test_message)
+      except Exception as e:
+        response.serialize_error = str(e)
+        return response
+
+    elif request.requested_output_format == conformance_pb2.TEXT_FORMAT:
+      response.text_payload = text_format.MessageToString(
+          test_message, print_unknown_fields=request.print_unknown_fields)
+
+  except Exception as e:
+    response.runtime_error = str(e)
+
+  return response
+
+def do_test_io():
+  length_bytes = sys.stdin.read(4)
+  if len(length_bytes) == 0:
+    return False   # EOF
+  elif len(length_bytes) != 4:
+    raise IOError("I/O error")
+
+  # "I" is "unsigned int", so this depends on running on a platform with
+  # 32-bit "unsigned int" type.  The Python struct module unfortunately
+  # has no format specifier for uint32_t.
+  length = struct.unpack("<I", length_bytes)[0]
+  serialized_request = sys.stdin.read(length)
+  if len(serialized_request) != length:
+    raise IOError("I/O error")
+
+  request = conformance_pb2.ConformanceRequest()
+  request.ParseFromString(serialized_request)
+
+  response = do_test(request)
+
+  serialized_response = response.SerializeToString()
+  sys.stdout.write(struct.pack("<I", len(serialized_response)))
+  sys.stdout.write(serialized_response)
+  sys.stdout.flush()
+
+  if verbose:
+    sys.stderr.write("conformance_python: request=%s, response=%s\n" % (
+                       request.ShortDebugString().c_str(),
+                       response.ShortDebugString().c_str()))
+
+  global test_count
+  test_count += 1
+
+  return True
+
+while True:
+  if not do_test_io():
+    sys.stderr.write("conformance_python: received EOF from test runner " +
+                     "after %s tests, exiting\n" % (test_count))
+    sys.exit(0)

conformance/conformance_ruby.rb

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require 'conformance_pb'
+require 'google/protobuf/test_messages_proto3_pb'
+require 'google/protobuf/test_messages_proto2_pb'
+
+$test_count = 0
+$verbose = false
+
+def do_test(request)
+  test_message = ProtobufTestMessages::Proto3::TestAllTypesProto3.new
+  response = Conformance::ConformanceResponse.new
+  descriptor = Google::Protobuf::DescriptorPool.generated_pool.lookup(request.message_type)
+
+  unless descriptor
+    response.skipped = "Unknown message type: " + request.message_type
+  end
+
+  begin
+    case request.payload
+    when :protobuf_payload
+      begin
+        test_message = descriptor.msgclass.decode(request.protobuf_payload)
+      rescue Google::Protobuf::ParseError => err
+        response.parse_error = err.message.encode('utf-8')
+        return response
+      end
+
+    when :json_payload
+      begin
+        options = {}
+        if request.test_category == :JSON_IGNORE_UNKNOWN_PARSING_TEST
+          options[:ignore_unknown_fields] = true
+        end
+        test_message = descriptor.msgclass.decode_json(request.json_payload, options)
+      rescue Google::Protobuf::ParseError => err
+        response.parse_error = err.message.encode('utf-8')
+        return response
+      end
+
+    when :text_payload
+      begin
+        response.skipped = "Ruby doesn't support text format"
+        return response
+      end
+
+    when nil
+      fail "Request didn't have payload"
+    end
+
+    case request.requested_output_format
+    when :UNSPECIFIED
+      fail 'Unspecified output format'
+
+    when :PROTOBUF
+      begin
+        response.protobuf_payload = test_message.to_proto
+      rescue Google::Protobuf::ParseError => err
+        response.serialize_error = err.message.encode('utf-8')
+      end
+
+    when :JSON
+      begin
+        response.json_payload = test_message.to_json
+      rescue Google::Protobuf::ParseError => err
+        response.serialize_error = err.message.encode('utf-8')
+      end
+
+    when nil
+      fail "Request didn't have requested output format"
+    end
+  rescue StandardError => err
+    response.runtime_error = err.message.encode('utf-8')
+  end
+
+  response
+end
+
+# Returns true if the test ran successfully, false on legitimate EOF.
+# If EOF is encountered in an unexpected place, raises IOError.
+def do_test_io
+  length_bytes = STDIN.read(4)
+  return false if length_bytes.nil?
+
+  length = length_bytes.unpack('V').first
+  serialized_request = STDIN.read(length)
+  if serialized_request.nil? || serialized_request.length != length
+    fail IOError
+  end
+
+  request = Conformance::ConformanceRequest.decode(serialized_request)
+
+  response = do_test(request)
+
+  serialized_response = Conformance::ConformanceResponse.encode(response)
+  STDOUT.write([serialized_response.length].pack('V'))
+  STDOUT.write(serialized_response)
+  STDOUT.flush
+
+  if $verbose
+    STDERR.puts("conformance_ruby: request=#{request.to_json}, " \
+                                 "response=#{response.to_json}\n")
+  end
+
+  $test_count += 1
+
+  true
+end
+
+loop do
+  unless do_test_io
+    STDERR.puts('conformance_ruby: received EOF from test runner ' \
+                "after #{$test_count} tests, exiting")
+    break
+  end
+end

conformance/conformance_test.cc

@@ -0,0 +1,519 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "conformance_test.h"
+
+#include <stdarg.h>
+
+#include <fstream>
+#include <set>
+#include <string>
+
+#include "google/protobuf/message.h"
+#include "google/protobuf/text_format.h"
+#include "google/protobuf/util/field_comparator.h"
+#include "google/protobuf/util/json_util.h"
+#include "google/protobuf/util/message_differencer.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
+#include "conformance/conformance.pb.h"
+#include "conformance/conformance.pb.h"
+
+using conformance::ConformanceRequest;
+using conformance::ConformanceResponse;
+using conformance::WireFormat;
+using google::protobuf::TextFormat;
+using google::protobuf::util::DefaultFieldComparator;
+using google::protobuf::util::MessageDifferencer;
+using std::string;
+
+namespace {
+
+static string ToOctString(const string& binary_string) {
+  string oct_string;
+  for (size_t i = 0; i < binary_string.size(); i++) {
+    uint8_t c = binary_string.at(i);
+    uint8_t high = c / 64;
+    uint8_t mid = (c % 64) / 8;
+    uint8_t low = c % 8;
+    oct_string.push_back('\\');
+    oct_string.push_back('0' + high);
+    oct_string.push_back('0' + mid);
+    oct_string.push_back('0' + low);
+  }
+  return oct_string;
+}
+
+}  // namespace
+
+namespace google {
+namespace protobuf {
+
+ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
+    ConformanceLevel level,
+    conformance::WireFormat input_format,
+    conformance::WireFormat output_format,
+    conformance::TestCategory test_category,
+    const Message& prototype_message,
+    const string& test_name, const string& input)
+    : level_(level),
+      input_format_(input_format),
+      output_format_(output_format),
+      prototype_message_(prototype_message),
+      prototype_message_for_compare_(prototype_message.New()),
+      test_name_(test_name) {
+  switch (input_format) {
+    case conformance::PROTOBUF: {
+      request_.set_protobuf_payload(input);
+      break;
+    }
+
+    case conformance::JSON: {
+      request_.set_json_payload(input);
+      break;
+    }
+
+    case conformance::JSPB: {
+      request_.set_jspb_payload(input);
+      break;
+    }
+
+    case conformance::TEXT_FORMAT: {
+      request_.set_text_payload(input);
+      break;
+    }
+
+    default:
+      GOOGLE_LOG(FATAL) << "Unspecified input format";
+  }
+
+  request_.set_test_category(test_category);
+
+  request_.set_message_type(prototype_message.GetDescriptor()->full_name());
+  request_.set_requested_output_format(output_format);
+}
+
+std::unique_ptr<Message>
+ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const {
+  return std::unique_ptr<Message>(prototype_message_for_compare_->New());
+}
+
+string ConformanceTestSuite::ConformanceRequestSetting::
+    GetTestName() const {
+  string rname =
+      prototype_message_.GetDescriptor()->file()->syntax() ==
+        FileDescriptor::SYNTAX_PROTO3 ? "Proto3" : "Proto2";
+
+  return absl::StrCat(ConformanceLevelToString(level_), ".", rname, ".",
+                      InputFormatString(input_format_), ".", test_name_, ".",
+                      OutputFormatString(output_format_));
+}
+
+string ConformanceTestSuite::ConformanceRequestSetting::
+    ConformanceLevelToString(
+        ConformanceLevel level) const {
+  switch (level) {
+    case REQUIRED: return "Required";
+    case RECOMMENDED: return "Recommended";
+  }
+  GOOGLE_LOG(FATAL) << "Unknown value: " << level;
+  return "";
+}
+
+string ConformanceTestSuite::ConformanceRequestSetting::
+    InputFormatString(conformance::WireFormat format) const {
+  switch (format) {
+    case conformance::PROTOBUF:
+      return "ProtobufInput";
+    case conformance::JSON:
+      return "JsonInput";
+    case conformance::TEXT_FORMAT:
+      return "TextFormatInput";
+    default:
+      GOOGLE_LOG(FATAL) << "Unspecified output format";
+  }
+  return "";
+}
+
+string ConformanceTestSuite::ConformanceRequestSetting::
+    OutputFormatString(conformance::WireFormat format) const {
+  switch (format) {
+    case conformance::PROTOBUF:
+      return "ProtobufOutput";
+    case conformance::JSON:
+      return "JsonOutput";
+    case conformance::TEXT_FORMAT:
+      return "TextFormatOutput";
+    default:
+      GOOGLE_LOG(FATAL) << "Unspecified output format";
+  }
+  return "";
+}
+
+void ConformanceTestSuite::TruncateDebugPayload(string* payload) {
+  if (payload != nullptr && payload->size() > 200) {
+    payload->resize(200);
+    payload->append("...(truncated)");
+  }
+}
+
+const ConformanceRequest ConformanceTestSuite::TruncateRequest(
+    const ConformanceRequest& request) {
+  ConformanceRequest debug_request(request);
+  switch (debug_request.payload_case()) {
+    case ConformanceRequest::kProtobufPayload:
+      TruncateDebugPayload(debug_request.mutable_protobuf_payload());
+      break;
+    case ConformanceRequest::kJsonPayload:
+      TruncateDebugPayload(debug_request.mutable_json_payload());
+      break;
+    case ConformanceRequest::kTextPayload:
+      TruncateDebugPayload(debug_request.mutable_text_payload());
+      break;
+    case ConformanceRequest::kJspbPayload:
+      TruncateDebugPayload(debug_request.mutable_jspb_payload());
+      break;
+    default:
+      // Do nothing.
+      break;
+  }
+  return debug_request;
+}
+
+const ConformanceResponse ConformanceTestSuite::TruncateResponse(
+    const ConformanceResponse& response) {
+  ConformanceResponse debug_response(response);
+  switch (debug_response.result_case()) {
+    case ConformanceResponse::kProtobufPayload:
+      TruncateDebugPayload(debug_response.mutable_protobuf_payload());
+      break;
+    case ConformanceResponse::kJsonPayload:
+      TruncateDebugPayload(debug_response.mutable_json_payload());
+      break;
+    case ConformanceResponse::kTextPayload:
+      TruncateDebugPayload(debug_response.mutable_text_payload());
+      break;
+    case ConformanceResponse::kJspbPayload:
+      TruncateDebugPayload(debug_response.mutable_jspb_payload());
+      break;
+    default:
+      // Do nothing.
+      break;
+  }
+  return debug_response;
+}
+
+void ConformanceTestSuite::ReportSuccess(const string& test_name) {
+  if (expected_to_fail_.erase(test_name) != 0) {
+    absl::StrAppendFormat(
+        &output_,
+        "ERROR: test %s is in the failure list, but test succeeded.  "
+        "Remove it from the failure list.\n",
+        test_name);
+    unexpected_succeeding_tests_.insert(test_name);
+  }
+  successes_++;
+}
+
+void ConformanceTestSuite::ReportFailure(const string& test_name,
+                                         ConformanceLevel level,
+                                         const ConformanceRequest& request,
+                                         const ConformanceResponse& response,
+                                         absl::string_view message) {
+  if (expected_to_fail_.erase(test_name) == 1) {
+    expected_failures_++;
+    if (!verbose_)
+      return;
+  } else if (level == RECOMMENDED && !enforce_recommended_) {
+    absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test_name);
+  } else {
+    absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test_name);
+    unexpected_failing_tests_.insert(test_name);
+  }
+
+  absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", message,
+                        TruncateRequest(request).ShortDebugString(),
+                        TruncateResponse(response).ShortDebugString());
+}
+
+void ConformanceTestSuite::ReportSkip(const string& test_name,
+                                      const ConformanceRequest& request,
+                                      const ConformanceResponse& response) {
+  if (verbose_) {
+    absl::StrAppendFormat(
+        &output_, "SKIPPED, test=%s request=%s, response=%s\n", test_name,
+        request.ShortDebugString(), response.ShortDebugString());
+  }
+  skipped_.insert(test_name);
+}
+
+void ConformanceTestSuite::RunValidInputTest(
+    const ConformanceRequestSetting& setting,
+    const string& equivalent_text_format) {
+  std::unique_ptr<Message> reference_message(setting.NewTestMessage());
+  GOOGLE_CHECK(TextFormat::ParseFromString(equivalent_text_format,
+                                    reference_message.get()))
+      << "Failed to parse data for test case: " << setting.GetTestName()
+      << ", data: " << equivalent_text_format;
+  const string equivalent_wire_format = reference_message->SerializeAsString();
+  RunValidBinaryInputTest(setting, equivalent_wire_format);
+}
+
+void ConformanceTestSuite::RunValidBinaryInputTest(
+    const ConformanceRequestSetting& setting,
+    const string& equivalent_wire_format, bool require_same_wire_format) {
+  const ConformanceRequest& request = setting.GetRequest();
+  ConformanceResponse response;
+  RunTest(setting.GetTestName(), request, &response);
+  VerifyResponse(setting, equivalent_wire_format, response, true,
+                 require_same_wire_format);
+}
+
+void ConformanceTestSuite::VerifyResponse(
+    const ConformanceRequestSetting& setting,
+    const string& equivalent_wire_format, const ConformanceResponse& response,
+    bool need_report_success, bool require_same_wire_format) {
+  std::unique_ptr<Message> test_message(setting.NewTestMessage());
+  const ConformanceRequest& request = setting.GetRequest();
+  const string& test_name = setting.GetTestName();
+  ConformanceLevel level = setting.GetLevel();
+  std::unique_ptr<Message> reference_message = setting.NewTestMessage();
+
+  GOOGLE_CHECK(reference_message->ParseFromString(equivalent_wire_format))
+      << "Failed to parse wire data for test case: " << test_name;
+
+  switch (response.result_case()) {
+    case ConformanceResponse::RESULT_NOT_SET:
+      ReportFailure(test_name, level, request, response,
+                    "Response didn't have any field in the Response.");
+      return;
+
+    case ConformanceResponse::kParseError:
+    case ConformanceResponse::kTimeoutError:
+    case ConformanceResponse::kRuntimeError:
+    case ConformanceResponse::kSerializeError:
+      ReportFailure(test_name, level, request, response,
+                    "Failed to parse input or produce output.");
+      return;
+
+    case ConformanceResponse::kSkipped:
+      ReportSkip(test_name, request, response);
+      return;
+
+    default:
+      if (!ParseResponse(response, setting, test_message.get())) return;
+  }
+
+  MessageDifferencer differencer;
+  DefaultFieldComparator field_comparator;
+  field_comparator.set_treat_nan_as_equal(true);
+  differencer.set_field_comparator(&field_comparator);
+  string differences;
+  differencer.ReportDifferencesToString(&differences);
+
+  bool check = false;
+
+  if (require_same_wire_format) {
+    GOOGLE_DCHECK_EQ(response.result_case(), ConformanceResponse::kProtobufPayload);
+    const string& protobuf_payload = response.protobuf_payload();
+    check = equivalent_wire_format == protobuf_payload;
+    differences = absl::StrCat("Expect: ", ToOctString(equivalent_wire_format),
+                               ", but got: ", ToOctString(protobuf_payload));
+  } else {
+    check = differencer.Compare(*reference_message, *test_message);
+  }
+
+  if (check) {
+    if (need_report_success) {
+      ReportSuccess(test_name);
+    }
+  } else {
+    ReportFailure(
+        test_name, level, request, response,
+        absl::StrCat("Output was not equivalent to reference message: ",
+                     differences));
+  }
+}
+
+void ConformanceTestSuite::RunTest(const string& test_name,
+                                   const ConformanceRequest& request,
+                                   ConformanceResponse* response) {
+  if (test_names_.insert(test_name).second == false) {
+    GOOGLE_LOG(FATAL) << "Duplicated test name: " << test_name;
+  }
+
+  string serialized_request;
+  string serialized_response;
+  request.SerializeToString(&serialized_request);
+
+  runner_->RunTest(test_name, serialized_request, &serialized_response);
+
+  if (!response->ParseFromString(serialized_response)) {
+    response->Clear();
+    response->set_runtime_error("response proto could not be parsed.");
+  }
+
+  if (verbose_) {
+    absl::StrAppendFormat(
+        &output_, "conformance test: name=%s, request=%s, response=%s\n",
+        test_name, TruncateRequest(request).ShortDebugString(),
+        TruncateResponse(*response).ShortDebugString());
+  }
+}
+
+bool ConformanceTestSuite::CheckSetEmpty(
+    const std::set<string>& set_to_check,
+    const std::string& write_to_file,
+    const std::string& msg) {
+  if (set_to_check.empty()) {
+    return true;
+  } else {
+    absl::StrAppendFormat(&output_, "\n");
+    absl::StrAppendFormat(&output_, "%s\n\n", msg);
+    for (absl::string_view v : set_to_check) {
+      absl::StrAppendFormat(&output_, "  %s\n", v);
+    }
+    absl::StrAppendFormat(&output_, "\n");
+
+    if (!write_to_file.empty()) {
+      std::string full_filename;
+      const std::string* filename = &write_to_file;
+      if (!output_dir_.empty()) {
+        full_filename = output_dir_;
+        if (*output_dir_.rbegin() != '/') {
+          full_filename.push_back('/');
+        }
+        full_filename += write_to_file;
+        filename = &full_filename;
+      }
+      std::ofstream os(*filename);
+      if (os) {
+        for (absl::string_view v : set_to_check) {
+          os << v << "\n";
+        }
+      } else {
+        absl::StrAppendFormat(&output_, "Failed to open file: %s\n", *filename);
+      }
+    }
+
+    return false;
+  }
+}
+
+string ConformanceTestSuite::WireFormatToString(
+    WireFormat wire_format) {
+  switch (wire_format) {
+    case conformance::PROTOBUF:
+      return "PROTOBUF";
+    case conformance::JSON:
+      return "JSON";
+    case conformance::JSPB:
+      return "JSPB";
+    case conformance::TEXT_FORMAT:
+      return "TEXT_FORMAT";
+    case conformance::UNSPECIFIED:
+      return "UNSPECIFIED";
+    default:
+      GOOGLE_LOG(FATAL) << "unknown wire type: " << wire_format;
+  }
+  return "";
+}
+
+void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) {
+  expected_to_fail_.insert(test_name);
+}
+
+bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
+                                    std::string* output, const string& filename,
+                                    conformance::FailureSet* failure_list) {
+  runner_ = runner;
+  successes_ = 0;
+  expected_failures_ = 0;
+  skipped_.clear();
+  test_names_.clear();
+  unexpected_failing_tests_.clear();
+  unexpected_succeeding_tests_.clear();
+
+  output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n";
+
+  failure_list_filename_ = filename;
+  expected_to_fail_.clear();
+  for (const string& failure : failure_list->failure()) {
+    AddExpectedFailedTest(failure);
+  }
+  RunSuiteImpl();
+
+  bool ok = true;
+  if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt",
+                     "These tests were listed in the failure list, but they "
+                     "don't exist.  Remove them from the failure list by "
+                     "running:\n"
+                     "  ./update_failure_list.py " + failure_list_filename_ +
+                     " --remove nonexistent_tests.txt")) {
+    ok = false;
+  }
+  if (!CheckSetEmpty(unexpected_failing_tests_, "failing_tests.txt",
+                     "These tests failed.  If they can't be fixed right now, "
+                     "you can add them to the failure list so the overall "
+                     "suite can succeed.  Add them to the failure list by "
+                     "running:\n"
+                     "  ./update_failure_list.py " + failure_list_filename_ +
+                     " --add failing_tests.txt")) {
+    ok = false;
+  }
+  if (!CheckSetEmpty(unexpected_succeeding_tests_, "succeeding_tests.txt",
+                     "These tests succeeded, even though they were listed in "
+                     "the failure list.  Remove them from the failure list "
+                     "by running:\n"
+                     "  ./update_failure_list.py " + failure_list_filename_ +
+                     " --remove succeeding_tests.txt")) {
+    ok = false;
+  }
+
+  if (verbose_) {
+    CheckSetEmpty(skipped_, "",
+                  "These tests were skipped (probably because support for some "
+                  "features is not implemented)");
+  }
+
+  absl::StrAppendFormat(&output_,
+                        "CONFORMANCE SUITE %s: %d successes, %zu skipped, "
+                        "%d expected failures, %zu unexpected failures.\n",
+                        ok ? "PASSED" : "FAILED", successes_, skipped_.size(),
+                        expected_failures_, unexpected_failing_tests_.size());
+  absl::StrAppendFormat(&output_, "\n");
+
+  output->assign(output_);
+
+  return ok;
+}
+
+}  // namespace protobuf
+}  // namespace google

conformance/conformance_test.h

@@ -0,0 +1,341 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+// This file defines a protocol for running the conformance test suite
+// in-process.  In other words, the suite itself will run in the same process as
+// the code under test.
+//
+// For pros and cons of this approach, please see conformance.proto.
+
+#ifndef CONFORMANCE_CONFORMANCE_TEST_H
+#define CONFORMANCE_CONFORMANCE_TEST_H
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/wire_format_lite.h"
+#include "google/protobuf/util/type_resolver.h"
+#include "conformance/conformance.pb.h"
+
+namespace conformance {
+class ConformanceRequest;
+class ConformanceResponse;
+}  // namespace conformance
+
+namespace protobuf_test_messages {
+namespace proto3 {
+class TestAllTypesProto3;
+}  // namespace proto3
+}  // namespace protobuf_test_messages
+
+namespace google {
+namespace protobuf {
+
+class ConformanceTestSuite;
+
+class ConformanceTestRunner {
+ public:
+  virtual ~ConformanceTestRunner() {}
+
+  // Call to run a single conformance test.
+  //
+  // "input" is a serialized conformance.ConformanceRequest.
+  // "output" should be set to a serialized conformance.ConformanceResponse.
+  //
+  // If there is any error in running the test itself, set "runtime_error" in
+  // the response.
+  virtual void RunTest(const std::string& test_name,
+                       const std::string& input,
+                       std::string* output) = 0;
+};
+
+// Test runner that spawns the process being tested and communicates with it
+// over a pipe.
+class ForkPipeRunner : public ConformanceTestRunner {
+ public:
+  // Note: Run() doesn't take ownership of the pointers inside suites.
+  static int Run(int argc, char *argv[],
+                 const std::vector<ConformanceTestSuite*>& suites);
+
+  ForkPipeRunner(const std::string& executable,
+                 const std::vector<std::string>& executable_args,
+                 bool performance)
+      : child_pid_(-1),
+        executable_(executable),
+        executable_args_(executable_args),
+        performance_(performance) {}
+
+  explicit ForkPipeRunner(const std::string& executable)
+      : child_pid_(-1), executable_(executable) {}
+
+  virtual ~ForkPipeRunner() {}
+
+  void RunTest(const std::string& test_name,
+               const std::string& request,
+               std::string* response);
+
+ private:
+  void SpawnTestProgram();
+
+  void CheckedWrite(int fd, const void *buf, size_t len);
+  bool TryRead(int fd, void *buf, size_t len);
+  void CheckedRead(int fd, void *buf, size_t len);
+
+  int write_fd_;
+  int read_fd_;
+  pid_t child_pid_;
+  std::string executable_;
+  const std::vector<std::string> executable_args_;
+  bool performance_;
+  std::string current_test_name_;
+};
+
+// Class representing the test suite itself.  To run it, implement your own
+// class derived from ConformanceTestRunner, class derived from
+// ConformanceTestSuite and then write code like:
+//
+//    class MyConformanceTestSuite : public ConformanceTestSuite {
+//     public:
+//      void RunSuiteImpl() {
+//        // INSERT ACTUAL TESTS.
+//      }
+//    };
+//
+//    class MyConformanceTestRunner : public ConformanceTestRunner {
+//     public:
+//      static int Run(int argc, char *argv[],
+//                 ConformanceTestSuite* suite);
+//
+//     private:
+//      virtual void RunTest(...) {
+//        // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE.
+//      }
+//    };
+//
+//    int main() {
+//      MyConformanceTestSuite suite;
+//      MyConformanceTestRunner::Run(argc, argv, &suite);
+//    }
+//
+class ConformanceTestSuite {
+ public:
+  ConformanceTestSuite()
+      : verbose_(false),
+        performance_(false),
+        enforce_recommended_(false),
+        failure_list_flag_name_("--failure_list") {}
+  virtual ~ConformanceTestSuite() {}
+
+  void SetPerformance(bool performance) { performance_ = performance; }
+  void SetVerbose(bool verbose) { verbose_ = verbose; }
+
+  // Whether to require the testee to pass RECOMMENDED tests. By default failing
+  // a RECOMMENDED test case will not fail the entire suite but will only
+  // generated a warning. If this flag is set to true, RECOMMENDED tests will
+  // be treated the same way as REQUIRED tests and failing a RECOMMENDED test
+  // case will cause the entire test suite to fail as well. An implementation
+  // can enable this if it wants to be strictly conforming to protobuf spec.
+  // See the comments about ConformanceLevel below to learn more about the
+  // difference between REQUIRED and RECOMMENDED test cases.
+  void SetEnforceRecommended(bool value) {
+    enforce_recommended_ = value;
+  }
+
+  // Gets the flag name to the failure list file.
+  // By default, this would return --failure_list
+  std::string GetFailureListFlagName() { return failure_list_flag_name_; }
+
+  void SetFailureListFlagName(const std::string& failure_list_flag_name) {
+    failure_list_flag_name_ = failure_list_flag_name;
+  }
+
+  // Sets the path of the output directory.
+  void SetOutputDir(const char* output_dir) {
+    output_dir_ = output_dir;
+  }
+
+  // Run all the conformance tests against the given test runner.
+  // Test output will be stored in "output".
+  //
+  // Returns true if the set of failing tests was exactly the same as the
+  // failure list.
+  // The filename here is *only* used to create/format useful error messages for
+  // how to update the failure list.  We do NOT read this file at all.
+  bool RunSuite(ConformanceTestRunner* runner, std::string* output,
+                const std::string& filename,
+                conformance::FailureSet* failure_list);
+
+ protected:
+  // Test cases are classified into a few categories:
+  //   REQUIRED: the test case must be passed for an implementation to be
+  //             interoperable with other implementations. For example, a
+  //             parser implementation must accept both packed and unpacked
+  //             form of repeated primitive fields.
+  //   RECOMMENDED: the test case is not required for the implementation to
+  //                be interoperable with other implementations, but is
+  //                recommended for best performance and compatibility. For
+  //                example, a proto3 serializer should serialize repeated
+  //                primitive fields in packed form, but an implementation
+  //                failing to do so will still be able to communicate with
+  //                other implementations.
+  enum ConformanceLevel {
+    REQUIRED = 0,
+    RECOMMENDED = 1,
+  };
+
+  class ConformanceRequestSetting {
+   public:
+    ConformanceRequestSetting(ConformanceLevel level,
+                              conformance::WireFormat input_format,
+                              conformance::WireFormat output_format,
+                              conformance::TestCategory test_category,
+                              const Message& prototype_message,
+                              const std::string& test_name,
+                              const std::string& input);
+    virtual ~ConformanceRequestSetting() {}
+
+    std::unique_ptr<Message> NewTestMessage() const;
+
+    std::string GetTestName() const;
+
+    const conformance::ConformanceRequest& GetRequest() const {
+      return request_;
+    }
+
+    const ConformanceLevel GetLevel() const {
+      return level_;
+    }
+
+    std::string ConformanceLevelToString(ConformanceLevel level) const;
+
+    void SetPrintUnknownFields(bool print_unknown_fields) {
+      request_.set_print_unknown_fields(true);
+    }
+
+    void SetPrototypeMessageForCompare(const Message& message) {
+      prototype_message_for_compare_.reset(message.New());
+    }
+
+   protected:
+    virtual std::string InputFormatString(conformance::WireFormat format) const;
+    virtual std::string OutputFormatString(
+        conformance::WireFormat format) const;
+    conformance::ConformanceRequest request_;
+
+   private:
+    ConformanceLevel level_;
+    ::conformance::WireFormat input_format_;
+    ::conformance::WireFormat output_format_;
+    const Message& prototype_message_;
+    std::unique_ptr<Message> prototype_message_for_compare_;
+    std::string test_name_;
+  };
+
+  bool CheckSetEmpty(const std::set<std::string>& set_to_check,
+                     const std::string& write_to_file, const std::string& msg);
+  std::string WireFormatToString(conformance::WireFormat wire_format);
+
+  // Parse payload in the response to the given message. Returns true on
+  // success.
+  virtual bool ParseResponse(
+      const conformance::ConformanceResponse& response,
+      const ConformanceRequestSetting& setting,
+      Message* test_message) = 0;
+
+  void VerifyResponse(const ConformanceRequestSetting& setting,
+                      const std::string& equivalent_wire_format,
+                      const conformance::ConformanceResponse& response,
+                      bool need_report_success, bool require_same_wire_format);
+
+  void TruncateDebugPayload(std::string* payload);
+  const conformance::ConformanceRequest TruncateRequest(
+      const conformance::ConformanceRequest& request);
+  const conformance::ConformanceResponse TruncateResponse(
+      const conformance::ConformanceResponse& response);
+
+  void ReportSuccess(const std::string& test_name);
+  void ReportFailure(const std::string& test_name, ConformanceLevel level,
+                     const conformance::ConformanceRequest& request,
+                     const conformance::ConformanceResponse& response,
+                     absl::string_view message);
+  void ReportSkip(const std::string& test_name,
+                  const conformance::ConformanceRequest& request,
+                  const conformance::ConformanceResponse& response);
+
+  void RunValidInputTest(const ConformanceRequestSetting& setting,
+                         const std::string& equivalent_text_format);
+  void RunValidBinaryInputTest(const ConformanceRequestSetting& setting,
+                               const std::string& equivalent_wire_format,
+                               bool require_same_wire_format = false);
+
+  void RunTest(const std::string& test_name,
+               const conformance::ConformanceRequest& request,
+               conformance::ConformanceResponse* response);
+
+  void AddExpectedFailedTest(const std::string& test_name);
+
+  virtual void RunSuiteImpl() = 0;
+
+  ConformanceTestRunner* runner_;
+  int successes_;
+  int expected_failures_;
+  bool verbose_;
+  bool performance_;
+  bool enforce_recommended_;
+  std::string output_;
+  std::string output_dir_;
+  std::string failure_list_flag_name_;
+  std::string failure_list_filename_;
+
+  // The set of test names that are expected to fail in this run, but haven't
+  // failed yet.
+  std::set<std::string> expected_to_fail_;
+
+  // The set of test names that have been run.  Used to ensure that there are no
+  // duplicate names in the suite.
+  std::set<std::string> test_names_;
+
+  // The set of tests that failed, but weren't expected to.
+  std::set<std::string> unexpected_failing_tests_;
+
+  // The set of tests that succeeded, but weren't expected to.
+  std::set<std::string> unexpected_succeeding_tests_;
+
+  // The set of tests that the testee opted out of;
+  std::set<std::string> skipped_;
+};
+
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // CONFORMANCE_CONFORMANCE_TEST_H

conformance/conformance_test_main.cc

@@ -0,0 +1,40 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "binary_json_conformance_suite.h"
+#include "conformance_test.h"
+#include "text_format_conformance_suite.h"
+
+int main(int argc, char *argv[]) {
+  google::protobuf::BinaryAndJsonConformanceSuite binary_and_json_suite;
+  google::protobuf::TextFormatConformanceTestSuite text_format_suite;
+  return google::protobuf::ForkPipeRunner::Run(
+      argc, argv, {&binary_and_json_suite, &text_format_suite});
+}

conformance/conformance_test_runner.cc

@@ -0,0 +1,389 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// This file contains a program for running the test suite in a separate
+// process.  The other alternative is to run the suite in-process.  See
+// conformance.proto for pros/cons of these two options.
+//
+// This program will fork the process under test and communicate with it over
+// its stdin/stdout:
+//
+//     +--------+   pipe   +----------+
+//     | tester | <------> | testee   |
+//     |        |          |          |
+//     |  C++   |          | any lang |
+//     +--------+          +----------+
+//
+// The tester contains all of the test cases and their expected output.
+// The testee is a simple program written in the target language that reads
+// each test case and attempts to produce acceptable output for it.
+//
+// Every test consists of a ConformanceRequest/ConformanceResponse
+// request/reply pair.  The protocol on the pipe is simply:
+//
+//   1. tester sends 4-byte length N (little endian)
+//   2. tester sends N bytes representing a ConformanceRequest proto
+//   3. testee sends 4-byte length M (little endian)
+//   4. testee sends M bytes representing a ConformanceResponse proto
+
+#include <errno.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <cstdlib>
+#include <fstream>
+#include <future>
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "conformance/conformance.pb.h"
+#include "conformance_test.h"
+
+using conformance::ConformanceResponse;
+using google::protobuf::ConformanceTestSuite;
+using std::string;
+using std::vector;
+
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+#define GOOGLE_CHECK_SYSCALL(call)                            \
+  if (call < 0) {                                      \
+    perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
+    exit(1);                                           \
+  }
+
+namespace google {
+namespace protobuf {
+
+void ParseFailureList(const char *filename,
+                      conformance::FailureSet *failure_list) {
+  std::ifstream infile(filename);
+
+  if (!infile.is_open()) {
+    fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
+    exit(1);
+  }
+
+  for (string line; getline(infile, line);) {
+    // Remove whitespace.
+    line.erase(std::remove_if(line.begin(), line.end(), ::isspace), line.end());
+
+    // Remove comments.
+    line = line.substr(0, line.find("#"));
+
+    if (!line.empty()) {
+      failure_list->add_failure(line);
+    }
+  }
+}
+
+void UsageError() {
+  fprintf(stderr, "Usage: conformance-test-runner [options] <test-program>\n");
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Options:\n");
+  fprintf(stderr,
+          "  --failure_list <filename>   Use to specify list of tests\n");
+  fprintf(stderr,
+          "                              that are expected to fail.  File\n");
+  fprintf(stderr,
+          "                              should contain one test name per\n");
+  fprintf(stderr,
+          "                              line.  Use '#' for comments.\n");
+  fprintf(stderr,
+          "  --text_format_failure_list <filename>   Use to specify list \n");
+  fprintf(stderr,
+          "                              of tests that are expected to \n");
+  fprintf(stderr, "                              fail in the \n");
+  fprintf(stderr,
+          "                              text_format_conformance_suite.  \n");
+  fprintf(stderr,
+          "                              File should contain one test name \n");
+  fprintf(stderr,
+          "                              per line.  Use '#' for comments.\n");
+
+  fprintf(stderr,
+          "  --enforce_recommended       Enforce that recommended test\n");
+  fprintf(stderr,
+          "                              cases are also passing. Specify\n");
+  fprintf(stderr,
+          "                              this flag if you want to be\n");
+  fprintf(stderr,
+          "                              strictly conforming to protobuf\n");
+  fprintf(stderr, "                              spec.\n");
+  fprintf(stderr,
+          "  --output_dir                <dirname> Directory to write\n"
+          "                              output files.\n");
+  exit(1);
+}
+
+void ForkPipeRunner::RunTest(const std::string &test_name,
+                             const std::string &request,
+                             std::string *response) {
+  if (child_pid_ < 0) {
+    SpawnTestProgram();
+  }
+  current_test_name_ = test_name;
+
+  uint32_t len = request.size();
+  CheckedWrite(write_fd_, &len, sizeof(uint32_t));
+  CheckedWrite(write_fd_, request.c_str(), request.size());
+
+  if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
+    // We failed to read from the child, assume a crash and try to reap.
+    GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;
+
+    int status = 0;
+    waitpid(child_pid_, &status, WEXITED);
+
+    string error_msg;
+    conformance::ConformanceResponse response_obj;
+    if (WIFEXITED(status)) {
+      if (WEXITSTATUS(status) == 0) {
+        absl::StrAppendFormat(&error_msg,
+                              "child timed out, killed by signal %d",
+                              WTERMSIG(status));
+        response_obj.set_timeout_error(error_msg);
+      } else {
+        absl::StrAppendFormat(&error_msg, "child exited, status=%d",
+                              WEXITSTATUS(status));
+        response_obj.set_runtime_error(error_msg);
+      }
+    } else if (WIFSIGNALED(status)) {
+      absl::StrAppendFormat(&error_msg, "child killed by signal %d",
+                            WTERMSIG(status));
+    }
+    GOOGLE_LOG(INFO) << error_msg;
+    child_pid_ = -1;
+
+    response_obj.SerializeToString(response);
+    return;
+  }
+
+  response->resize(len);
+  CheckedRead(read_fd_, (void *)response->c_str(), len);
+}
+
+int ForkPipeRunner::Run(int argc, char *argv[],
+                        const std::vector<ConformanceTestSuite *> &suites) {
+  if (suites.empty()) {
+    fprintf(stderr, "No test suites found.\n");
+    return EXIT_FAILURE;
+  }
+  bool all_ok = true;
+  for (ConformanceTestSuite *suite : suites) {
+    string program;
+    std::vector<string> program_args;
+    string failure_list_filename;
+    conformance::FailureSet failure_list;
+
+    bool performance = false;
+    for (int arg = 1; arg < argc; ++arg) {
+      if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
+        if (++arg == argc) UsageError();
+        failure_list_filename = argv[arg];
+        ParseFailureList(argv[arg], &failure_list);
+      } else if (strcmp(argv[arg], "--performance") == 0) {
+        performance = true;
+        suite->SetPerformance(true);
+      } else if (strcmp(argv[arg], "--verbose") == 0) {
+        suite->SetVerbose(true);
+      } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
+        suite->SetEnforceRecommended(true);
+      } else if (strcmp(argv[arg], "--output_dir") == 0) {
+        if (++arg == argc) UsageError();
+        suite->SetOutputDir(argv[arg]);
+      } else if (argv[arg][0] == '-') {
+        bool recognized_flag = false;
+        for (ConformanceTestSuite *suite : suites) {
+          if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
+            if (++arg == argc) UsageError();
+            recognized_flag = true;
+          }
+        }
+        if (!recognized_flag) {
+          fprintf(stderr, "Unknown option: %s\n", argv[arg]);
+          UsageError();
+        }
+      } else {
+        program += argv[arg++];
+        while (arg < argc) {
+          program_args.push_back(argv[arg]);
+          arg++;
+        }
+      }
+    }
+
+    ForkPipeRunner runner(program, program_args, performance);
+
+    std::string output;
+    all_ok = all_ok && suite->RunSuite(&runner, &output, failure_list_filename,
+                                       &failure_list);
+
+    fwrite(output.c_str(), 1, output.size(), stderr);
+  }
+  return all_ok ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+// TODO(haberman): make this work on Windows, instead of using these
+// UNIX-specific APIs.
+//
+// There is a platform-agnostic API in
+//    src/google/protobuf/compiler/subprocess.h
+//
+// However that API only supports sending a single message to the subprocess.
+// We really want to be able to send messages and receive responses one at a
+// time:
+//
+// 1. Spawning a new process for each test would take way too long for thousands
+//    of tests and subprocesses like java that can take 100ms or more to start
+//    up.
+//
+// 2. Sending all the tests in one big message and receiving all results in one
+//    big message would take away our visibility about which test(s) caused a
+//    crash or other fatal error.  It would also give us only a single failure
+//    instead of all of them.
+void ForkPipeRunner::SpawnTestProgram() {
+  int toproc_pipe_fd[2];
+  int fromproc_pipe_fd[2];
+  if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
+    perror("pipe");
+    exit(1);
+  }
+
+  pid_t pid = fork();
+  if (pid < 0) {
+    perror("fork");
+    exit(1);
+  }
+
+  if (pid) {
+    // Parent.
+    GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
+    GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
+    write_fd_ = toproc_pipe_fd[1];
+    read_fd_ = fromproc_pipe_fd[0];
+    child_pid_ = pid;
+  } else {
+    // Child.
+    GOOGLE_CHECK_SYSCALL(close(STDIN_FILENO));
+    GOOGLE_CHECK_SYSCALL(close(STDOUT_FILENO));
+    GOOGLE_CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
+    GOOGLE_CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
+
+    GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
+    GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
+    GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[1]));
+    GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
+
+    std::unique_ptr<char[]> executable(new char[executable_.size() + 1]);
+    memcpy(executable.get(), executable_.c_str(), executable_.size());
+    executable[executable_.size()] = '\0';
+
+    std::vector<const char *> argv;
+    argv.push_back(executable.get());
+    GOOGLE_LOG(INFO) << argv[0];
+    for (size_t i = 0; i < executable_args_.size(); ++i) {
+      argv.push_back(executable_args_[i].c_str());
+      GOOGLE_LOG(INFO) << executable_args_[i];
+    }
+    argv.push_back(nullptr);
+    // Never returns.
+    GOOGLE_CHECK_SYSCALL(execv(executable.get(), const_cast<char **>(argv.data())));
+  }
+}
+
+void ForkPipeRunner::CheckedWrite(int fd, const void *buf, size_t len) {
+  if (static_cast<size_t>(write(fd, buf, len)) != len) {
+    GOOGLE_LOG(FATAL) << current_test_name_
+               << ": error writing to test program: " << strerror(errno);
+  }
+}
+
+bool ForkPipeRunner::TryRead(int fd, void *buf, size_t len) {
+  size_t ofs = 0;
+  while (len > 0) {
+    std::future<ssize_t> future = std::async(
+        std::launch::async,
+        [](int fd, void *buf, size_t ofs, size_t len) {
+          return read(fd, (char *)buf + ofs, len);
+        },
+        fd, buf, ofs, len);
+    std::future_status status;
+    if (performance_) {
+      status = future.wait_for(std::chrono::seconds(5));
+      if (status == std::future_status::timeout) {
+        GOOGLE_LOG(ERROR) << current_test_name_ << ": timeout from test program";
+        kill(child_pid_, SIGQUIT);
+        // TODO(sandyzhang): Only log in flag-guarded mode, since reading output
+        // from SIGQUIT is slow and verbose.
+        std::vector<char> err;
+        err.resize(5000);
+        ssize_t err_bytes_read;
+        size_t err_ofs = 0;
+        do {
+          err_bytes_read =
+              read(fd, (void *)&err[err_ofs], err.size() - err_ofs);
+          err_ofs += err_bytes_read;
+        } while (err_bytes_read > 0 && err_ofs < err.size());
+        GOOGLE_LOG(ERROR) << "child_pid_=" << child_pid_ << " SIGQUIT: \n" << &err[0];
+        return false;
+      }
+    } else {
+      future.wait();
+    }
+    ssize_t bytes_read = future.get();
+    if (bytes_read == 0) {
+      GOOGLE_LOG(ERROR) << current_test_name_ << ": unexpected EOF from test program";
+      return false;
+    } else if (bytes_read < 0) {
+      GOOGLE_LOG(ERROR) << current_test_name_
+                 << ": error reading from test program: " << strerror(errno);
+      return false;
+    }
+
+    len -= bytes_read;
+    ofs += bytes_read;
+  }
+
+  return true;
+}
+
+void ForkPipeRunner::CheckedRead(int fd, void *buf, size_t len) {
+  if (!TryRead(fd, buf, len)) {
+    GOOGLE_LOG(FATAL) << current_test_name_
+               << ": error reading from test program: " << strerror(errno);
+  }
+}
+
+}  // namespace protobuf
+}  // namespace google

conformance/defs.bzl

@@ -0,0 +1,48 @@
+"""Starlark definitions for Protobuf conformance tests.
+
+PLEASE DO NOT DEPEND ON THE CONTENTS OF THIS FILE, IT IS UNSTABLE.
+"""
+
+def conformance_test(
+        name,
+        testee,
+        failure_list = None,
+        text_format_failure_list = None,
+        **kwargs):
+    """Conformance test runner.
+
+    Args:
+      name: the name for the test.
+      testee: a conformance test client binary.
+      failure_list: a text file with known failures, one per line.
+      text_format_failure_list: a text file with known failures (one per line)
+          for the text format conformance suite.
+      **kwargs: common arguments to pass to sh_test.
+    """
+    args = ["--testee %s" % _strip_bazel(testee)]
+    failure_lists = []
+    if failure_list:
+        args = args + ["--failure_list %s" % _strip_bazel(failure_list)]
+        failure_lists = failure_lists + [failure_list]
+    if text_format_failure_list:
+        args = args + ["--text_format_failure_list %s" % _strip_bazel(text_format_failure_list)]
+        failure_lists = failure_lists + [text_format_failure_list]
+
+    native.sh_test(
+        name = name,
+        srcs = ["//conformance:bazel_conformance_test_runner.sh"],
+        data = [testee] + failure_lists + [
+            "//conformance:conformance_test_runner",
+        ],
+        args = args,
+        deps = [
+            "@bazel_tools//tools/bash/runfiles",
+        ],
+        tags = ["conformance"],
+        **kwargs
+    )
+
+def _strip_bazel(testee):
+    if testee.startswith("//"):
+        testee = testee.replace("//", "com_google_protobuf/")
+    return testee.replace(":", "/")

conformance/failure_list_cpp.txt

@@ -0,0 +1,35 @@
+# This is the list of conformance tests that are known to fail for the C++
+# implementation right now.  These should be fixed.
+#
+# By listing them here we can keep tabs on which ones are failing and be sure
+# that we don't introduce regressions in other tests.
+#
+# TODO(haberman): insert links to corresponding bugs tracking the issue.
+# Should we use GitHub issues or the Google-internal bug tracker?
+
+Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
+Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
+Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse
+Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue
+Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter
+Recommended.Proto3.JsonInput.FieldNameDuplicate
+Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1
+Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2
+Recommended.Proto3.JsonInput.FieldNameNotQuoted
+Recommended.Proto3.JsonInput.MapFieldValueIsNull
+Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull
+Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull
+Recommended.Proto3.JsonInput.RepeatedFieldTrailingComma
+Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines
+Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace
+Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue
+Recommended.Proto3.JsonInput.StringFieldUppercaseEscapeLetter
+Recommended.Proto3.JsonInput.TrailingCommaInAnObject
+Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines
+Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace
+Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator

conformance/failure_list_csharp.txt

@@ -0,0 +1,7 @@
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
+Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
+Required.Proto3.JsonInput.OneofFieldNullFirst.JsonOutput
+Required.Proto3.JsonInput.OneofFieldNullFirst.ProtobufOutput
+Required.Proto3.JsonInput.OneofFieldNullSecond.JsonOutput
+Required.Proto3.JsonInput.OneofFieldNullSecond.ProtobufOutput

conformance/failure_list_java.txt

@@ -0,0 +1,44 @@
+# This is the list of conformance tests that are known to fail for the Java
+# implementation right now.  These should be fixed.
+#
+# By listing them here we can keep tabs on which ones are failing and be sure
+# that we don't introduce regressions in other tests.
+
+Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
+Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
+Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto3.JsonInput.BoolFieldAllCapitalFalse
+Recommended.Proto3.JsonInput.BoolFieldAllCapitalTrue
+Recommended.Proto3.JsonInput.BoolFieldCamelCaseFalse
+Recommended.Proto3.JsonInput.BoolFieldCamelCaseTrue
+Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse
+Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue
+Recommended.Proto3.JsonInput.BoolMapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted
+Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted
+Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted
+Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter
+Recommended.Proto3.JsonInput.FieldNameDuplicate
+Recommended.Proto3.JsonInput.FieldNameNotQuoted
+Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
+Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
+Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
+Recommended.Proto3.JsonInput.Int32MapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.Int64MapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.JsonWithComments
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue
+Recommended.Proto3.JsonInput.StringFieldSurrogateInWrongOrder
+Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate
+Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate
+Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+Required.Proto3.JsonInput.EnumFieldNotQuoted
+Required.Proto3.JsonInput.Int32FieldLeadingZero
+Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero
+Required.Proto3.JsonInput.Int32FieldPlusSign
+Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
+Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
+Required.Proto3.JsonInput.StringFieldNotAString

conformance/failure_list_java_lite.txt

@@ -0,0 +1,10 @@
+# This is the list of conformance tests that are known to fail for the Java
+# implementation right now. These should be fixed.
+#
+# By listing them here we can keep tabs on which ones are failing and be sure
+# that we don't introduce regressions in other tests.
+
+Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
+Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
+Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
+Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE

conformance/failure_list_jruby.txt

@@ -0,0 +1,95 @@
+Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
+Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
+Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.BOOL.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.BOOL.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.DOUBLE.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.DOUBLE.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FLOAT.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FLOAT.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto3.JsonInput.BoolFieldAllCapitalFalse
+Recommended.Proto3.JsonInput.BoolFieldAllCapitalTrue
+Recommended.Proto3.JsonInput.BoolFieldCamelCaseFalse
+Recommended.Proto3.JsonInput.BoolFieldCamelCaseTrue
+Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse
+Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue
+Recommended.Proto3.JsonInput.BoolMapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted
+Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted
+Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted
+Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter
+Recommended.Proto3.JsonInput.FieldNameDuplicate
+Recommended.Proto3.JsonInput.FieldNameNotQuoted
+Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted
+Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted
+Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted
+Recommended.Proto3.JsonInput.Int32MapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.Int64MapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.JsonWithComments
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey
+Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue
+Recommended.Proto3.JsonInput.StringFieldSurrogateInWrongOrder
+Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate
+Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate
+Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted
+Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted
+Recommended.Proto3.ProtobufInput.ValidDataOneofBinary.MESSAGE.Merge.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.BOOL.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.BOOL.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Required.Proto3.JsonInput.EnumFieldNotQuoted
+Required.Proto3.JsonInput.Int32FieldLeadingZero
+Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero
+Required.Proto3.JsonInput.Int32FieldPlusSign
+Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
+Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
+Required.Proto3.JsonInput.StringFieldNotAString

conformance/failure_list_objc.txt

@@ -0,0 +1,2 @@
+# JSON input or output tests are skipped (in conformance_objc.m) as mobile
+# platforms don't support JSON wire format to avoid code bloat.

conformance/failure_list_php.txt

@@ -0,0 +1,32 @@
+Recommended.FieldMaskNumbersDontRoundTrip.JsonOutput
+Recommended.FieldMaskPathsDontRoundTrip.JsonOutput
+Recommended.FieldMaskTooManyUnderscore.JsonOutput
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
+Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
+Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter
+Recommended.Proto3.ProtobufInput.ValidDataOneofBinary.MESSAGE.Merge.ProtobufOutput
+Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator
+Required.Proto3.JsonInput.DoubleFieldTooSmall
+Required.Proto3.JsonInput.DurationNegativeNanos.JsonOutput
+Required.Proto3.JsonInput.DurationNegativeNanos.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldTooLarge
+Required.Proto3.JsonInput.FloatFieldTooSmall
+Required.Proto3.JsonInput.Int32FieldNotInteger
+Required.Proto3.JsonInput.Int64FieldNotInteger
+Required.Proto3.JsonInput.OneofFieldDuplicate
+Required.Proto3.JsonInput.OneofFieldNullSecond.JsonOutput
+Required.Proto3.JsonInput.OneofFieldNullSecond.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
+Required.Proto3.JsonInput.RepeatedListValue.JsonOutput
+Required.Proto3.JsonInput.RepeatedListValue.ProtobufOutput
+Required.Proto3.JsonInput.StringFieldNotAString
+Required.Proto3.JsonInput.Uint32FieldNotInteger
+Required.Proto3.JsonInput.Uint64FieldNotInteger
+Required.Proto3.ProtobufInput.RepeatedScalarMessageMerge.JsonOutput
+Required.Proto3.ProtobufInput.RepeatedScalarMessageMerge.ProtobufOutput
+Required.Proto3.ProtobufInput.ValidDataOneof.MESSAGE.Merge.JsonOutput
+Required.Proto3.ProtobufInput.ValidDataOneof.MESSAGE.Merge.ProtobufOutput
+Required.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.PackedInput.JsonOutput
+Required.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.UnpackedInput.JsonOutput
+Required.Proto3.ProtobufInput.ValidDataScalar.FLOAT[2].JsonOutput

conformance/failure_list_php_c.txt

@@ -0,0 +1,2 @@
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator

conformance/failure_list_python-post26.txt

@@ -0,0 +1,2 @@
+JsonInput.StringFieldSurrogateInWrongOrder
+JsonInput.StringFieldUnpairedHighSurrogate

conformance/failure_list_python_cpp.txt

@@ -0,0 +1,8 @@
+# This is the list of conformance tests that are known to fail for the
+# Python/C++ implementation right now.  These should be fixed.
+#
+# By listing them here we can keep tabs on which ones are failing and be sure
+# that we don't introduce regressions in other tests.
+#
+# TODO(haberman): insert links to corresponding bugs tracking the issue.
+# Should we use GitHub issues or the Google-internal bug tracker?

conformance/failure_list_ruby.txt

@@ -0,0 +1,58 @@
+Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.BOOL.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.BOOL.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.DOUBLE.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.DOUBLE.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FLOAT.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.FLOAT.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SFIXED64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.PackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataOneofBinary.MESSAGE.Merge.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.BOOL.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.BOOL.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.DOUBLE.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.FLOAT.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SFIXED64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.UnpackedOutput.ProtobufOutput
+Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.UnpackedOutput.ProtobufOutput

conformance/text_format_conformance_suite.cc

@@ -0,0 +1,558 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "text_format_conformance_suite.h"
+
+#include "google/protobuf/any.pb.h"
+#include "google/protobuf/text_format.h"
+#include "conformance_test.h"
+#include "google/protobuf/test_messages_proto2.pb.h"
+#include "google/protobuf/test_messages_proto3.pb.h"
+
+namespace proto2_messages = protobuf_test_messages::proto2;
+
+using conformance::ConformanceRequest;
+using conformance::ConformanceResponse;
+using conformance::WireFormat;
+using google::protobuf::Message;
+using google::protobuf::TextFormat;
+using proto2_messages::TestAllTypesProto2;
+using proto2_messages::UnknownToTestAllTypes;
+using protobuf_test_messages::proto3::TestAllTypesProto3;
+using std::string;
+
+namespace google {
+namespace protobuf {
+
+// The number of repetitions to use for performance tests.
+// Corresponds approx to 500KB wireformat bytes.
+static const size_t kPerformanceRepeatCount = 50000;
+
+TextFormatConformanceTestSuite::TextFormatConformanceTestSuite() {
+  SetFailureListFlagName("--text_format_failure_list");
+}
+
+bool TextFormatConformanceTestSuite::ParseTextFormatResponse(
+    const ConformanceResponse& response,
+    const ConformanceRequestSetting& setting, Message* test_message) {
+  TextFormat::Parser parser;
+  const ConformanceRequest& request = setting.GetRequest();
+  if (request.print_unknown_fields()) {
+    parser.AllowFieldNumber(true);
+  }
+  if (!parser.ParseFromString(response.text_payload(), test_message)) {
+    GOOGLE_LOG(ERROR) << "INTERNAL ERROR: internal text->protobuf transcode "
+               << "yielded unparseable proto. Text payload: "
+               << response.text_payload();
+    return false;
+  }
+
+  return true;
+}
+
+bool TextFormatConformanceTestSuite::ParseResponse(
+    const ConformanceResponse& response,
+    const ConformanceRequestSetting& setting, Message* test_message) {
+  const ConformanceRequest& request = setting.GetRequest();
+  WireFormat requested_output = request.requested_output_format();
+  const string& test_name = setting.GetTestName();
+  ConformanceLevel level = setting.GetLevel();
+
+  switch (response.result_case()) {
+    case ConformanceResponse::kProtobufPayload: {
+      if (requested_output != conformance::PROTOBUF) {
+        ReportFailure(test_name, level, request, response,
+                      absl::StrCat("Test was asked for ",
+                                   WireFormatToString(requested_output),
+                                   " output but provided PROTOBUF instead."));
+        return false;
+      }
+
+      if (!test_message->ParseFromString(response.protobuf_payload())) {
+        ReportFailure(test_name, level, request, response,
+                      "Protobuf output we received from test was unparseable.");
+        return false;
+      }
+
+      break;
+    }
+
+    case ConformanceResponse::kTextPayload: {
+      if (requested_output != conformance::TEXT_FORMAT) {
+        ReportFailure(
+            test_name, level, request, response,
+            absl::StrCat("Test was asked for ",
+                         WireFormatToString(requested_output),
+                         " output but provided TEXT_FORMAT instead."));
+        return false;
+      }
+
+      if (!ParseTextFormatResponse(response, setting, test_message)) {
+        ReportFailure(
+            test_name, level, request, response,
+            "TEXT_FORMAT output we received from test was unparseable.");
+        return false;
+      }
+
+      break;
+    }
+
+    default:
+      GOOGLE_LOG(FATAL) << test_name
+                 << ": unknown payload type: " << response.result_case();
+  }
+
+  return true;
+}
+
+void TextFormatConformanceTestSuite::ExpectParseFailure(const string& test_name,
+                                                        ConformanceLevel level,
+                                                        const string& input) {
+  TestAllTypesProto3 prototype;
+  // We don't expect output, but if the program erroneously accepts the protobuf
+  // we let it send its response as this.  We must not leave it unspecified.
+  ConformanceRequestSetting setting(
+      level, conformance::TEXT_FORMAT, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input);
+  const ConformanceRequest& request = setting.GetRequest();
+  ConformanceResponse response;
+  string effective_test_name =
+      absl::StrCat(setting.ConformanceLevelToString(level),
+                   ".Proto3.TextFormatInput.", test_name);
+
+  RunTest(effective_test_name, request, &response);
+  if (response.result_case() == ConformanceResponse::kParseError) {
+    ReportSuccess(effective_test_name);
+  } else if (response.result_case() == ConformanceResponse::kSkipped) {
+    ReportSkip(effective_test_name, request, response);
+  } else {
+    ReportFailure(effective_test_name, level, request, response,
+                  "Should have failed to parse, but didn't.");
+  }
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTest(
+    const string& test_name, ConformanceLevel level, const string& input_text) {
+  TestAllTypesProto3 prototype;
+  RunValidTextFormatTestWithMessage(test_name, level, input_text, prototype);
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTestProto2(
+    const string& test_name, ConformanceLevel level, const string& input_text) {
+  TestAllTypesProto2 prototype;
+  RunValidTextFormatTestWithMessage(test_name, level, input_text, prototype);
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTestWithExpected(
+    const string& test_name, ConformanceLevel level, const string& input_text,
+    const string& expected_text) {
+  TestAllTypesProto3 prototype;
+  RunValidTextFormatTestWithMessage(test_name, level, input_text, expected_text,
+                                    prototype);
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTestProto2WithExpected(
+    const string& test_name, ConformanceLevel level, const string& input_text,
+    const string& expected_text) {
+  TestAllTypesProto2 prototype;
+  RunValidTextFormatTestWithMessage(test_name, level, input_text, expected_text,
+                                    prototype);
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTestWithMessage(
+    const string& test_name, ConformanceLevel level, const string& input_text,
+    const Message& prototype) {
+  ConformanceRequestSetting setting1(
+      level, conformance::TEXT_FORMAT, conformance::PROTOBUF,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text);
+  RunValidInputTest(setting1, input_text);
+  ConformanceRequestSetting setting2(
+      level, conformance::TEXT_FORMAT, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text);
+  RunValidInputTest(setting2, input_text);
+}
+
+void TextFormatConformanceTestSuite::RunValidTextFormatTestWithMessage(
+    const string& test_name, ConformanceLevel level, const string& input_text,
+    const string& expected_text, const Message& prototype) {
+  ConformanceRequestSetting setting1(
+      level, conformance::TEXT_FORMAT, conformance::PROTOBUF,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text);
+  RunValidInputTest(setting1, expected_text);
+  ConformanceRequestSetting setting2(
+      level, conformance::TEXT_FORMAT, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text);
+  RunValidInputTest(setting2, expected_text);
+}
+
+void TextFormatConformanceTestSuite::RunValidUnknownTextFormatTest(
+    const string& test_name, const Message& message) {
+  string serialized_input;
+  message.SerializeToString(&serialized_input);
+  TestAllTypesProto3 prototype;
+  ConformanceRequestSetting setting1(
+      RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Drop",
+      serialized_input);
+  setting1.SetPrototypeMessageForCompare(message);
+  RunValidBinaryInputTest(setting1, "");
+
+  ConformanceRequestSetting setting2(
+      RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Print",
+      serialized_input);
+  setting2.SetPrototypeMessageForCompare(message);
+  setting2.SetPrintUnknownFields(true);
+  RunValidBinaryInputTest(setting2, serialized_input);
+}
+
+void TextFormatConformanceTestSuite::RunSuiteImpl() {
+  if (!performance_) {
+    RunValidTextFormatTest("HelloWorld", REQUIRED,
+                           "optional_string: 'Hello, World!'");
+    // Integer fields.
+    RunValidTextFormatTest("Int32FieldMaxValue", REQUIRED,
+                           "optional_int32: 2147483647");
+    RunValidTextFormatTest("Int32FieldMinValue", REQUIRED,
+                           "optional_int32: -2147483648");
+    RunValidTextFormatTest("Uint32FieldMaxValue", REQUIRED,
+                           "optional_uint32: 4294967295");
+    RunValidTextFormatTest("Int64FieldMaxValue", REQUIRED,
+                           "optional_int64: 9223372036854775807");
+    RunValidTextFormatTest("Int64FieldMinValue", REQUIRED,
+                           "optional_int64: -9223372036854775808");
+    RunValidTextFormatTest("Uint64FieldMaxValue", REQUIRED,
+                           "optional_uint64: 18446744073709551615");
+
+    // Parsers reject out-of-bound integer values.
+    ExpectParseFailure("Int32FieldTooLarge", REQUIRED,
+                       "optional_int32: 2147483648");
+    ExpectParseFailure("Int32FieldTooSmall", REQUIRED,
+                       "optional_int32: -2147483649");
+    ExpectParseFailure("Uint32FieldTooLarge", REQUIRED,
+                       "optional_uint32: 4294967296");
+    ExpectParseFailure("Int64FieldTooLarge", REQUIRED,
+                       "optional_int64: 9223372036854775808");
+    ExpectParseFailure("Int64FieldTooSmall", REQUIRED,
+                       "optional_int64: -9223372036854775809");
+    ExpectParseFailure("Uint64FieldTooLarge", REQUIRED,
+                       "optional_uint64: 18446744073709551616");
+
+    // Floating point fields
+    RunValidTextFormatTest("FloatField", REQUIRED, "optional_float: 3.192837");
+    RunValidTextFormatTest("FloatFieldWithVeryPreciseNumber", REQUIRED,
+                           "optional_float: 3.123456789123456789");
+    RunValidTextFormatTest("FloatFieldMaxValue", REQUIRED,
+                           "optional_float: 3.4028235e+38");
+    RunValidTextFormatTest("FloatFieldMinValue", REQUIRED,
+                           "optional_float: 1.17549e-38");
+    RunValidTextFormatTest("FloatFieldNaNValue", REQUIRED,
+                           "optional_float: NaN");
+    RunValidTextFormatTest("FloatFieldPosInfValue", REQUIRED,
+                           "optional_float: inf");
+    RunValidTextFormatTest("FloatFieldNegInfValue", REQUIRED,
+                           "optional_float: -inf");
+    RunValidTextFormatTest("FloatFieldWithInt32Max", REQUIRED,
+                           "optional_float: 4294967296");
+    RunValidTextFormatTest("FloatFieldLargerThanInt64", REQUIRED,
+                           "optional_float: 9223372036854775808");
+    RunValidTextFormatTest("FloatFieldTooLarge", REQUIRED,
+                           "optional_float: 3.4028235e+39");
+    RunValidTextFormatTest("FloatFieldTooSmall", REQUIRED,
+                           "optional_float: 1.17549e-39");
+    RunValidTextFormatTest("FloatFieldLargerThanUint64", REQUIRED,
+                           "optional_float: 18446744073709551616");
+
+    // String literals x {Strings, Bytes}
+    for (const auto& field_type : std::vector<std::string>{"String", "Bytes"}) {
+      const std::string field_name =
+          field_type == "String" ? "optional_string" : "optional_bytes";
+      RunValidTextFormatTest(
+          absl::StrCat("StringLiteralConcat", field_type), REQUIRED,
+          absl::StrCat(field_name, ": 'first' \"second\"\n'third'"));
+      RunValidTextFormatTest(
+          absl::StrCat("StringLiteralBasicEscapes", field_type), REQUIRED,
+          absl::StrCat(field_name, ": '\\a\\b\\f\\n\\r\\t\\v\\?\\\\\\'\\\"'"));
+      RunValidTextFormatTest(
+          absl::StrCat("StringLiteralOctalEscapes", field_type), REQUIRED,
+          absl::StrCat(field_name, ": '\\341\\210\\264'"));
+      RunValidTextFormatTest(
+          absl::StrCat("StringLiteralHexEscapes", field_type), REQUIRED,
+          absl::StrCat(field_name, ": '\\xe1\\x88\\xb4'"));
+      RunValidTextFormatTest(
+          absl::StrCat("StringLiteralShortUnicodeEscape", field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\u1234'"));
+      RunValidTextFormatTest(
+          absl::StrCat("StringLiteralLongUnicodeEscapes", field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\U00001234\\U00010437'"));
+      // String literals don't include line feeds.
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralIncludesLF", field_type), REQUIRED,
+          absl::StrCat(field_name, ": 'first line\nsecond line'"));
+      // Unicode escapes don't include code points that lie beyond the planes
+      // (> 0x10ffff).
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralLongUnicodeEscapeTooLarge", field_type),
+          REQUIRED, absl::StrCat(field_name, ": '\\U00110000'"));
+      // Unicode escapes don't include surrogates.
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralShortUnicodeEscapeSurrogatePair",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\ud801\\udc37'"));
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralShortUnicodeEscapeSurrogateFirstOnly",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\ud800'"));
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralShortUnicodeEscapeSurrogateSecondOnly",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\udc00'"));
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralLongUnicodeEscapeSurrogateFirstOnly",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\U0000d800'"));
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralLongUnicodeEscapeSurrogateSecondOnly",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\U0000dc00'"));
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralLongUnicodeEscapeSurrogatePair",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\U0000d801\\U00000dc37'"));
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralUnicodeEscapeSurrogatePairLongShort",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\U0000d801\\udc37'"));
+      ExpectParseFailure(
+          absl::StrCat("StringLiteralUnicodeEscapeSurrogatePairShortLong",
+                       field_type),
+          RECOMMENDED, absl::StrCat(field_name, ": '\\ud801\\U0000dc37'"));
+
+      // The following method depend on the type of field, as strings have extra
+      // validation.
+      const auto test_method =
+          field_type == "String"
+              ? &TextFormatConformanceTestSuite::ExpectParseFailure
+              : &TextFormatConformanceTestSuite::RunValidTextFormatTest;
+
+      // String fields reject invalid UTF-8 byte sequences; bytes fields don't.
+      (this->*test_method)(absl::StrCat(field_type, "FieldBadUTF8Octal"),
+                           REQUIRED, absl::StrCat(field_name, ": '\\300'"));
+      (this->*test_method)(absl::StrCat(field_type, "FieldBadUTF8Hex"),
+                           REQUIRED, absl::StrCat(field_name, ": '\\xc0'"));
+    }
+
+    // Group fields
+    RunValidTextFormatTestProto2("GroupFieldNoColon", REQUIRED,
+                                 "Data { group_int32: 1 }");
+    RunValidTextFormatTestProto2("GroupFieldWithColon", REQUIRED,
+                                 "Data: { group_int32: 1 }");
+    RunValidTextFormatTestProto2("GroupFieldEmpty", REQUIRED, "Data {}");
+
+    // Unknown Fields
+    UnknownToTestAllTypes message;
+    // Unable to print unknown Fixed32/Fixed64 fields as if they are known.
+    // Fixed32/Fixed64 fields are not added in the tests.
+    message.set_optional_int32(123);
+    message.set_optional_string("hello");
+    message.set_optional_bool(true);
+    RunValidUnknownTextFormatTest("ScalarUnknownFields", message);
+
+    message.Clear();
+    message.mutable_nested_message()->set_c(111);
+    RunValidUnknownTextFormatTest("MessageUnknownFields", message);
+
+    message.Clear();
+    message.mutable_optionalgroup()->set_a(321);
+    RunValidUnknownTextFormatTest("GroupUnknownFields", message);
+
+    message.add_repeated_int32(1);
+    message.add_repeated_int32(2);
+    message.add_repeated_int32(3);
+    RunValidUnknownTextFormatTest("RepeatedUnknownFields", message);
+
+    // Any fields
+    RunValidTextFormatTest("AnyField", REQUIRED,
+                           R"(
+        optional_any: {
+          [type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3] {
+            optional_int32: 12345
+          }
+        }
+        )");
+    RunValidTextFormatTest("AnyFieldWithRawBytes", REQUIRED,
+                           R"(
+        optional_any: {
+          type_url: "type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3"
+          value: "\b\271`"
+        }
+        )");
+    ExpectParseFailure("AnyFieldWithInvalidType", REQUIRED,
+                       R"(
+        optional_any: {
+          [type.googleapis.com/unknown] {
+            optional_int32: 12345
+          }
+        }
+        )");
+
+    // Map fields
+    TestAllTypesProto3 prototype;
+    (*prototype.mutable_map_string_string())["c"] = "value";
+    (*prototype.mutable_map_string_string())["b"] = "value";
+    (*prototype.mutable_map_string_string())["a"] = "value";
+    RunValidTextFormatTestWithMessage("AlphabeticallySortedMapStringKeys",
+                                      REQUIRED,
+                                      R"(
+        map_string_string {
+          key: "a"
+          value: "value"
+        }
+        map_string_string {
+          key: "b"
+          value: "value"
+        }
+        map_string_string {
+          key: "c"
+          value: "value"
+        }
+        )",
+                                      prototype);
+
+    prototype.Clear();
+    (*prototype.mutable_map_int32_int32())[3] = 0;
+    (*prototype.mutable_map_int32_int32())[2] = 0;
+    (*prototype.mutable_map_int32_int32())[1] = 0;
+    RunValidTextFormatTestWithMessage("AlphabeticallySortedMapIntKeys",
+                                      REQUIRED,
+                                      R"(
+        map_int32_int32 {
+          key: 1
+          value: 0
+        }
+        map_int32_int32 {
+          key: 2
+          value: 0
+        }
+        map_int32_int32 {
+          key: 3
+          value: 0
+        }
+        )",
+                                      prototype);
+
+    prototype.Clear();
+    (*prototype.mutable_map_bool_bool())[true] = false;
+    (*prototype.mutable_map_bool_bool())[false] = false;
+    RunValidTextFormatTestWithMessage("AlphabeticallySortedMapBoolKeys",
+                                      REQUIRED,
+                                      R"(
+        map_bool_bool {
+          key: false
+          value: false
+        }
+        map_bool_bool {
+          key: true
+          value: false
+        }
+        )",
+                                      prototype);
+
+    prototype.Clear();
+    ConformanceRequestSetting setting_map(
+        REQUIRED, conformance::TEXT_FORMAT, conformance::PROTOBUF,
+        conformance::TEXT_FORMAT_TEST, prototype, "DuplicateMapKey", R"(
+        map_string_nested_message {
+          key: "duplicate"
+          value: { a: 123 }
+        }
+        map_string_nested_message {
+          key: "duplicate"
+          value: { corecursive: {} }
+        }
+        )");
+    // The last-specified value will be retained in a parsed map
+    RunValidInputTest(setting_map, R"(
+        map_string_nested_message {
+          key: "duplicate"
+          value: { corecursive: {} }
+        }
+        )");
+  }
+  // Flag control performance tests to keep them internal and opt-in only
+  if (performance_) {
+    RunTextFormatPerformanceTests();
+  }
+}
+
+void TextFormatConformanceTestSuite::RunTextFormatPerformanceTests() {
+  TestTextFormatPerformanceMergeMessageWithRepeatedField("Bool",
+                                                         "repeated_bool: true");
+  TestTextFormatPerformanceMergeMessageWithRepeatedField(
+      "Double", "repeated_double: 123");
+  TestTextFormatPerformanceMergeMessageWithRepeatedField(
+      "Int32", "repeated_uint32: 123");
+  TestTextFormatPerformanceMergeMessageWithRepeatedField(
+      "Int64", "repeated_uint64: 123");
+  TestTextFormatPerformanceMergeMessageWithRepeatedField(
+      "String", R"(repeated_string: "foo")");
+  TestTextFormatPerformanceMergeMessageWithRepeatedField(
+      "Bytes", R"(repeated_bytes: "foo")");
+}
+
+// This is currently considered valid input by some languages but not others
+void TextFormatConformanceTestSuite::
+    TestTextFormatPerformanceMergeMessageWithRepeatedField(
+        const string& test_type_name, const string& message_field) {
+  string recursive_message = "recursive_message { " + message_field + " }";
+
+  string input;
+  for (size_t i = 0; i < kPerformanceRepeatCount; i++) {
+    input.append(recursive_message);
+  }
+
+  string expected = "recursive_message { ";
+  for (size_t i = 0; i < kPerformanceRepeatCount; i++) {
+    expected.append(message_field + " ");
+  }
+  expected.append("}");
+
+  RunValidTextFormatTestProto2WithExpected(
+      "TestTextFormatPerformanceMergeMessageWithRepeatedField" +
+          test_type_name + "Proto2",
+      RECOMMENDED, input, expected);
+  RunValidTextFormatTestWithExpected(
+      "TestTextFormatPerformanceMergeMessageWithRepeatedField" +
+          test_type_name + "Proto3",
+      RECOMMENDED, input, expected);
+}
+
+}  // namespace protobuf
+}  // namespace google

conformance/text_format_conformance_suite.h

@@ -0,0 +1,85 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef TEXT_FORMAT_CONFORMANCE_SUITE_H_
+#define TEXT_FORMAT_CONFORMANCE_SUITE_H_
+
+#include "conformance_test.h"
+
+namespace google {
+namespace protobuf {
+
+class TextFormatConformanceTestSuite : public ConformanceTestSuite {
+ public:
+  TextFormatConformanceTestSuite();
+
+ private:
+  void RunSuiteImpl() override;
+  void RunTextFormatPerformanceTests();
+  void RunValidTextFormatTest(const std::string& test_name,
+                              ConformanceLevel level, const std::string& input);
+  void RunValidTextFormatTestProto2(const std::string& test_name,
+                                    ConformanceLevel level,
+                                    const std::string& input);
+  void RunValidTextFormatTestWithExpected(const std::string& test_name,
+                                          ConformanceLevel level,
+                                          const std::string& input,
+                                          const std::string& expected);
+  void RunValidTextFormatTestProto2WithExpected(const std::string& test_name,
+                                                ConformanceLevel level,
+                                                const std::string& input,
+                                                const std::string& expected);
+  void RunValidTextFormatTestWithMessage(const std::string& test_name,
+                                         ConformanceLevel level,
+                                         const std::string& input_text,
+                                         const Message& prototype);
+  void RunValidTextFormatTestWithMessage(const std::string& test_name,
+                                         ConformanceLevel level,
+                                         const std::string& input_text,
+                                         const std::string& expected_text,
+                                         const Message& prototype);
+  void RunValidUnknownTextFormatTest(const std::string& test_name,
+                                     const Message& message);
+  void ExpectParseFailure(const std::string& test_name, ConformanceLevel level,
+                          const std::string& input);
+  bool ParseTextFormatResponse(const conformance::ConformanceResponse& response,
+                               const ConformanceRequestSetting& setting,
+                               Message* test_message);
+  bool ParseResponse(const conformance::ConformanceResponse& response,
+                     const ConformanceRequestSetting& setting,
+                     Message* test_message) override;
+  void TestTextFormatPerformanceMergeMessageWithRepeatedField(
+      const std::string& test_type_name, const std::string& message_field);
+};
+
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // TEXT_FORMAT_CONFORMANCE_SUITE_H_

conformance/text_format_failure_list_cpp.txt

@@ -0,0 +1,20 @@
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

conformance/text_format_failure_list_csharp.txt

@@ -0,0 +1,8 @@
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput

conformance/text_format_failure_list_java.txt

@@ -0,0 +1,9 @@
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
+Required.Proto3.TextFormatInput.AnyField.ProtobufOutput
+Required.Proto3.TextFormatInput.AnyField.TextFormatOutput
+
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal

conformance/text_format_failure_list_java_lite.txt

@@ -0,0 +1,5 @@
+# This is the list of conformance tests that are known to fail for the Java
+# Lite TextFormat implementation right now. These should be fixed.
+#
+# By listing them here we can keep tabs on which ones are failing and be sure
+# that we don't introduce regressions in other tests.

conformance/text_format_failure_list_jruby.txt

@@ -0,0 +1,8 @@
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput

conformance/text_format_failure_list_php.txt

@@ -0,0 +1,8 @@
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput

conformance/text_format_failure_list_python.txt

@@ -0,0 +1,34 @@
+# This is the list of text format conformance tests that are known to fail right
+# now.
+# TODO: These should be fixed.
+Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
+Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
+
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

conformance/text_format_failure_list_python_cpp.txt

@@ -0,0 +1,28 @@
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

conformance/text_format_failure_list_ruby.txt

@@ -0,0 +1,8 @@
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput

conformance/update_failure_list.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Script to update a failure list file to add/remove failures.
+
+This is sort of like comm(1), except it recognizes comments and ignores them.
+"""
+
+import argparse
+
+parser = argparse.ArgumentParser(
+    description='Adds/removes failures from the failure list.')
+parser.add_argument('filename', type=str, help='failure list file to update')
+parser.add_argument('--add', dest='add_list', action='append')
+parser.add_argument('--remove', dest='remove_list', action='append')
+
+args = parser.parse_args()
+
+add_set = set()
+remove_set = set()
+
+for add_file in (args.add_list or []):
+  with open(add_file) as f:
+    for line in f:
+      add_set.add(line)
+
+for remove_file in (args.remove_list or []):
+  with open(remove_file) as f:
+    for line in f:
+      if line in add_set:
+        raise Exception("Asked to both add and remove test: " + line)
+      remove_set.add(line.strip())
+
+add_list = sorted(add_set, reverse=True)
+
+with open(args.filename) as in_file:
+    existing_list = in_file.read()
+
+with open(args.filename, "w") as f:
+  for line in existing_list.splitlines(True):
+    test = line.split("#")[0].strip()
+    while len(add_list) > 0 and test > add_list[-1]:
+      f.write(add_list.pop())
+    if test not in remove_set:
+      f.write(line)