Squashed 'libs/protobuf/' content from commit fcd3b9a85

git-subtree-dir: libs/protobuf
git-subtree-split: fcd3b9a85ef36e46643dc30176cea1a7ad62e02b

python/google/protobuf/pyext/README

@@ -0,0 +1,6 @@
+This is the 'v2' C++ implementation for python proto2.
+
+It is active when:
+
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2

python/google/protobuf/pyext/cpp_message.py

@@ -0,0 +1,72 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Protocol message implementation hooks for C++ implementation.
+
+Contains helper functions used to create protocol message classes from
+Descriptor objects at runtime backed by the protocol buffer C++ API.
+"""
+
+__author__ = 'tibell@google.com (Johan Tibell)'
+
+from google.protobuf.internal import api_implementation
+
+
+# pylint: disable=protected-access
+_message = api_implementation._c_module
+# TODO(jieluo): Remove this import after fix api_implementation
+if _message is None:
+  from google.protobuf.pyext import _message
+
+
+class GeneratedProtocolMessageType(_message.MessageMeta):
+
+  """Metaclass for protocol message classes created at runtime from Descriptors.
+
+  The protocol compiler currently uses this metaclass to create protocol
+  message classes at runtime.  Clients can also manually create their own
+  classes at runtime, as in this example:
+
+  mydescriptor = Descriptor(.....)
+  factory = symbol_database.Default()
+  factory.pool.AddDescriptor(mydescriptor)
+  MyProtoClass = factory.GetPrototype(mydescriptor)
+  myproto_instance = MyProtoClass()
+  myproto.foo_field = 23
+  ...
+
+  The above example will not work for nested types. If you wish to include them,
+  use reflection.MakeClass() instead of manually instantiating the class in
+  order to create the appropriate class structure.
+  """
+
+  # Must be consistent with the protocol-compiler code in
+  # proto2/compiler/internal/generator.*.
+  _DESCRIPTOR_KEY = 'DESCRIPTOR'

python/google/protobuf/pyext/descriptor.h

@@ -0,0 +1,105 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: petar@google.com (Petar Petrov)
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "google/protobuf/descriptor.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+extern PyTypeObject PyMessageDescriptor_Type;
+extern PyTypeObject PyFieldDescriptor_Type;
+extern PyTypeObject PyEnumDescriptor_Type;
+extern PyTypeObject PyEnumValueDescriptor_Type;
+extern PyTypeObject PyFileDescriptor_Type;
+extern PyTypeObject PyOneofDescriptor_Type;
+extern PyTypeObject PyServiceDescriptor_Type;
+extern PyTypeObject PyMethodDescriptor_Type;
+
+// Wraps a Descriptor in a Python object.
+// The C++ pointer is usually borrowed from the global DescriptorPool.
+// In any case, it must stay alive as long as the Python object.
+// Returns a new reference.
+PyObject* PyMessageDescriptor_FromDescriptor(const Descriptor* descriptor);
+PyObject* PyFieldDescriptor_FromDescriptor(const FieldDescriptor* descriptor);
+PyObject* PyEnumDescriptor_FromDescriptor(const EnumDescriptor* descriptor);
+PyObject* PyEnumValueDescriptor_FromDescriptor(
+    const EnumValueDescriptor* descriptor);
+PyObject* PyOneofDescriptor_FromDescriptor(const OneofDescriptor* descriptor);
+PyObject* PyFileDescriptor_FromDescriptor(
+    const FileDescriptor* file_descriptor);
+PyObject* PyServiceDescriptor_FromDescriptor(
+    const ServiceDescriptor* descriptor);
+PyObject* PyMethodDescriptor_FromDescriptor(
+    const MethodDescriptor* descriptor);
+
+// Alternate constructor of PyFileDescriptor, used when we already have a
+// serialized FileDescriptorProto that can be cached.
+// Returns a new reference.
+PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb(
+    const FileDescriptor* file_descriptor, PyObject* serialized_pb);
+
+// Return the C++ descriptor pointer.
+// This function checks the parameter type; on error, return NULL with a Python
+// exception set.
+const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj);
+const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj);
+const EnumDescriptor* PyEnumDescriptor_AsDescriptor(PyObject* obj);
+const FileDescriptor* PyFileDescriptor_AsDescriptor(PyObject* obj);
+const ServiceDescriptor* PyServiceDescriptor_AsDescriptor(PyObject* obj);
+
+// Returns the raw C++ pointer.
+const void* PyDescriptor_AsVoidPtr(PyObject* obj);
+
+// Check that the calling Python code is the global scope of a _pb2.py module.
+// This function is used to support the current code generated by the proto
+// compiler, which insists on modifying descriptors after they have been
+// created.
+//
+// stacklevel indicates which Python frame should be the _pb2.py module.
+//
+// Don't use this function outside descriptor classes.
+bool _CalledFromGeneratedFile(int stacklevel);
+
+bool InitDescriptor();
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__

python/google/protobuf/pyext/descriptor_containers.h

@@ -0,0 +1,110 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__
+
+// Mappings and Sequences of descriptors.
+// They implement containers like fields_by_name, EnumDescriptor.values...
+// See descriptor_containers.cc for more description.
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+namespace google {
+namespace protobuf {
+
+class Descriptor;
+class FileDescriptor;
+class EnumDescriptor;
+class OneofDescriptor;
+class ServiceDescriptor;
+
+namespace python {
+
+// Initialize the various types and objects.
+bool InitDescriptorMappingTypes();
+
+// Each function below returns a Mapping, or a Sequence of descriptors.
+// They all return a new reference.
+
+namespace message_descriptor {
+PyObject* NewMessageFieldsByName(const Descriptor* descriptor);
+PyObject* NewMessageFieldsByCamelcaseName(const Descriptor* descriptor);
+PyObject* NewMessageFieldsByNumber(const Descriptor* descriptor);
+PyObject* NewMessageFieldsSeq(const Descriptor* descriptor);
+
+PyObject* NewMessageNestedTypesSeq(const Descriptor* descriptor);
+PyObject* NewMessageNestedTypesByName(const Descriptor* descriptor);
+
+PyObject* NewMessageEnumsByName(const Descriptor* descriptor);
+PyObject* NewMessageEnumsSeq(const Descriptor* descriptor);
+PyObject* NewMessageEnumValuesByName(const Descriptor* descriptor);
+
+PyObject* NewMessageExtensionsByName(const Descriptor* descriptor);
+PyObject* NewMessageExtensionsSeq(const Descriptor* descriptor);
+
+PyObject* NewMessageOneofsByName(const Descriptor* descriptor);
+PyObject* NewMessageOneofsSeq(const Descriptor* descriptor);
+}  // namespace message_descriptor
+
+namespace enum_descriptor {
+PyObject* NewEnumValuesByName(const EnumDescriptor* descriptor);
+PyObject* NewEnumValuesByNumber(const EnumDescriptor* descriptor);
+PyObject* NewEnumValuesSeq(const EnumDescriptor* descriptor);
+}  // namespace enum_descriptor
+
+namespace oneof_descriptor {
+PyObject* NewOneofFieldsSeq(const OneofDescriptor* descriptor);
+}  // namespace oneof_descriptor
+
+namespace file_descriptor {
+PyObject* NewFileMessageTypesByName(const FileDescriptor* descriptor);
+
+PyObject* NewFileEnumTypesByName(const FileDescriptor* descriptor);
+
+PyObject* NewFileExtensionsByName(const FileDescriptor* descriptor);
+
+PyObject* NewFileServicesByName(const FileDescriptor* descriptor);
+
+PyObject* NewFileDependencies(const FileDescriptor* descriptor);
+PyObject* NewFilePublicDependencies(const FileDescriptor* descriptor);
+}  // namespace file_descriptor
+
+namespace service_descriptor {
+PyObject* NewServiceMethodsSeq(const ServiceDescriptor* descriptor);
+PyObject* NewServiceMethodsByName(const ServiceDescriptor* descriptor);
+}  // namespace service_descriptor
+
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_CONTAINERS_H__

python/google/protobuf/pyext/descriptor_database.cc

@@ -0,0 +1,189 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// This file defines a C++ DescriptorDatabase, which wraps a Python Database
+// and delegate all its operations to Python methods.
+
+#include "google/protobuf/pyext/descriptor_database.h"
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "google/protobuf/stubs/logging.h"
+#include "google/protobuf/stubs/common.h"
+#include "google/protobuf/descriptor.pb.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+PyDescriptorDatabase::PyDescriptorDatabase(PyObject* py_database)
+    : py_database_(py_database) {
+  Py_INCREF(py_database_);
+}
+
+PyDescriptorDatabase::~PyDescriptorDatabase() { Py_DECREF(py_database_); }
+
+// Convert a Python object to a FileDescriptorProto pointer.
+// Handles all kinds of Python errors, which are simply logged.
+static bool GetFileDescriptorProto(PyObject* py_descriptor,
+                                   FileDescriptorProto* output) {
+  if (py_descriptor == nullptr) {
+    if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+      // Expected error: item was simply not found.
+      PyErr_Clear();
+    } else {
+      GOOGLE_LOG(ERROR) << "DescriptorDatabase method raised an error";
+      PyErr_Print();
+    }
+    return false;
+  }
+  if (py_descriptor == Py_None) {
+    return false;
+  }
+  const Descriptor* filedescriptor_descriptor =
+      FileDescriptorProto::default_instance().GetDescriptor();
+  CMessage* message = reinterpret_cast<CMessage*>(py_descriptor);
+  if (PyObject_TypeCheck(py_descriptor, CMessage_Type) &&
+      message->message->GetDescriptor() == filedescriptor_descriptor) {
+    // Fast path: Just use the pointer.
+    FileDescriptorProto* file_proto =
+        static_cast<FileDescriptorProto*>(message->message);
+    *output = *file_proto;
+    return true;
+  } else {
+    // Slow path: serialize the message. This allows to use databases which
+    // use a different implementation of FileDescriptorProto.
+    ScopedPyObjectPtr serialized_pb(
+        PyObject_CallMethod(py_descriptor, "SerializeToString", nullptr));
+    if (serialized_pb == nullptr) {
+      GOOGLE_LOG(ERROR)
+          << "DescriptorDatabase method did not return a FileDescriptorProto";
+      PyErr_Print();
+      return false;
+    }
+    char* str;
+    Py_ssize_t len;
+    if (PyBytes_AsStringAndSize(serialized_pb.get(), &str, &len) < 0) {
+      GOOGLE_LOG(ERROR)
+          << "DescriptorDatabase method did not return a FileDescriptorProto";
+      PyErr_Print();
+      return false;
+    }
+    FileDescriptorProto file_proto;
+    if (!file_proto.ParseFromArray(str, len)) {
+      GOOGLE_LOG(ERROR)
+          << "DescriptorDatabase method did not return a FileDescriptorProto";
+      return false;
+    }
+    *output = file_proto;
+    return true;
+  }
+}
+
+// Find a file by file name.
+bool PyDescriptorDatabase::FindFileByName(const std::string& filename,
+                                          FileDescriptorProto* output) {
+  ScopedPyObjectPtr py_descriptor(PyObject_CallMethod(
+      py_database_, "FindFileByName", "s#", filename.c_str(), filename.size()));
+  return GetFileDescriptorProto(py_descriptor.get(), output);
+}
+
+// Find the file that declares the given fully-qualified symbol name.
+bool PyDescriptorDatabase::FindFileContainingSymbol(
+    const std::string& symbol_name, FileDescriptorProto* output) {
+  ScopedPyObjectPtr py_descriptor(
+      PyObject_CallMethod(py_database_, "FindFileContainingSymbol", "s#",
+                          symbol_name.c_str(), symbol_name.size()));
+  return GetFileDescriptorProto(py_descriptor.get(), output);
+}
+
+// Find the file which defines an extension extending the given message type
+// with the given field number.
+// Python DescriptorDatabases are not required to implement this method.
+bool PyDescriptorDatabase::FindFileContainingExtension(
+    const std::string& containing_type, int field_number,
+    FileDescriptorProto* output) {
+  ScopedPyObjectPtr py_method(
+      PyObject_GetAttrString(py_database_, "FindFileContainingExtension"));
+  if (py_method == nullptr) {
+    // This method is not implemented, returns without error.
+    PyErr_Clear();
+    return false;
+  }
+  ScopedPyObjectPtr py_descriptor(
+      PyObject_CallFunction(py_method.get(), "s#i", containing_type.c_str(),
+                            containing_type.size(), field_number));
+  return GetFileDescriptorProto(py_descriptor.get(), output);
+}
+
+// Finds the tag numbers used by all known extensions of
+// containing_type, and appends them to output in an undefined
+// order.
+// Python DescriptorDatabases are not required to implement this method.
+bool PyDescriptorDatabase::FindAllExtensionNumbers(
+    const std::string& containing_type, std::vector<int>* output) {
+  ScopedPyObjectPtr py_method(
+      PyObject_GetAttrString(py_database_, "FindAllExtensionNumbers"));
+  if (py_method == nullptr) {
+    // This method is not implemented, returns without error.
+    PyErr_Clear();
+    return false;
+  }
+  ScopedPyObjectPtr py_list(
+      PyObject_CallFunction(py_method.get(), "s#", containing_type.c_str(),
+                            containing_type.size()));
+  if (py_list == nullptr) {
+    PyErr_Print();
+    return false;
+  }
+  Py_ssize_t size = PyList_Size(py_list.get());
+  int64_t item_value;
+  for (Py_ssize_t i = 0 ; i < size; ++i) {
+    ScopedPyObjectPtr item(PySequence_GetItem(py_list.get(), i));
+    item_value = PyLong_AsLong(item.get());
+    if (item_value < 0) {
+      GOOGLE_LOG(ERROR)
+          << "FindAllExtensionNumbers method did not return "
+          << "valid extension numbers.";
+      PyErr_Print();
+      return false;
+    }
+    output->push_back(item_value);
+  }
+  return true;
+}
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/descriptor_database.h

@@ -0,0 +1,86 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <string>
+#include <vector>
+
+#include "google/protobuf/descriptor_database.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+class PyDescriptorDatabase : public DescriptorDatabase {
+ public:
+  explicit PyDescriptorDatabase(PyObject* py_database);
+  ~PyDescriptorDatabase() override;
+
+  // Implement the abstract interface. All these functions fill the output
+  // with a copy of FileDescriptorProto.
+
+  // Find a file by file name.
+  bool FindFileByName(const std::string& filename,
+                      FileDescriptorProto* output) override;
+
+  // Find the file that declares the given fully-qualified symbol name.
+  bool FindFileContainingSymbol(const std::string& symbol_name,
+                                FileDescriptorProto* output) override;
+
+  // Find the file which defines an extension extending the given message type
+  // with the given field number.
+  // Containing_type must be a fully-qualified type name.
+  // Python objects are not required to implement this method.
+  bool FindFileContainingExtension(const std::string& containing_type,
+                                   int field_number,
+                                   FileDescriptorProto* output) override;
+
+  // Finds the tag numbers used by all known extensions of
+  // containing_type, and appends them to output in an undefined
+  // order.
+  // Python objects are not required to implement this method.
+  bool FindAllExtensionNumbers(const std::string& containing_type,
+                               std::vector<int>* output) override;
+
+ private:
+  // The python object that implements the database. The reference is owned.
+  PyObject* py_database_;
+};
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__

python/google/protobuf/pyext/descriptor_pool.cc

@@ -0,0 +1,820 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Implements the DescriptorPool, which collects all descriptors.
+
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "google/protobuf/descriptor.pb.h"
+#include "google/protobuf/pyext/descriptor.h"
+#include "google/protobuf/pyext/descriptor_database.h"
+#include "google/protobuf/pyext/descriptor_pool.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/message_factory.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+#include "absl/strings/string_view.h"
+
+#define PyString_AsStringAndSize(ob, charpp, sizep)              \
+  (PyUnicode_Check(ob)                                           \
+       ? ((*(charpp) = const_cast<char*>(                        \
+               PyUnicode_AsUTF8AndSize(ob, (sizep)))) == nullptr \
+              ? -1                                               \
+              : 0)                                               \
+       : PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+// A map to cache Python Pools per C++ pointer.
+// Pointers are not owned here, and belong to the PyDescriptorPool.
+static std::unordered_map<const DescriptorPool*, PyDescriptorPool*>*
+    descriptor_pool_map;
+
+namespace cdescriptor_pool {
+
+// Collects errors that occur during proto file building to allow them to be
+// propagated in the python exception instead of only living in ERROR logs.
+class BuildFileErrorCollector : public DescriptorPool::ErrorCollector {
+ public:
+  BuildFileErrorCollector() : error_message(""), had_errors_(false) {}
+
+  void AddError(const std::string& filename, const std::string& element_name,
+                const Message* descriptor, ErrorLocation location,
+                const std::string& message) override {
+    // Replicates the logging behavior that happens in the C++ implementation
+    // when an error collector is not passed in.
+    if (!had_errors_) {
+      error_message +=
+          ("Invalid proto descriptor for file \"" + filename + "\":\n");
+      had_errors_ = true;
+    }
+    // As this only happens on failure and will result in the program not
+    // running at all, no effort is made to optimize this string manipulation.
+    error_message += ("  " + element_name + ": " + message + "\n");
+  }
+
+  void Clear() {
+    had_errors_ = false;
+    error_message = "";
+  }
+
+  std::string error_message;
+
+ private:
+  bool had_errors_;
+};
+
+// Create a Python DescriptorPool object, but does not fill the "pool"
+// attribute.
+static PyDescriptorPool* _CreateDescriptorPool() {
+  PyDescriptorPool* cpool = PyObject_GC_New(
+      PyDescriptorPool, &PyDescriptorPool_Type);
+  if (cpool == nullptr) {
+    return nullptr;
+  }
+
+  cpool->error_collector = nullptr;
+  cpool->underlay = nullptr;
+  cpool->database = nullptr;
+  cpool->is_owned = false;
+  cpool->is_mutable = false;
+
+  cpool->descriptor_options = new std::unordered_map<const void*, PyObject*>();
+
+  cpool->py_message_factory = message_factory::NewMessageFactory(
+      &PyMessageFactory_Type, cpool);
+  if (cpool->py_message_factory == nullptr) {
+    Py_DECREF(cpool);
+    return nullptr;
+  }
+
+  PyObject_GC_Track(cpool);
+
+  return cpool;
+}
+
+// Create a Python DescriptorPool, using the given pool as an underlay:
+// new messages will be added to a custom pool, not to the underlay.
+//
+// Ownership of the underlay is not transferred, its pointer should
+// stay alive.
+static PyDescriptorPool* PyDescriptorPool_NewWithUnderlay(
+    const DescriptorPool* underlay) {
+  PyDescriptorPool* cpool = _CreateDescriptorPool();
+  if (cpool == nullptr) {
+    return nullptr;
+  }
+  cpool->pool = new DescriptorPool(underlay);
+  cpool->is_owned = true;
+  cpool->is_mutable = true;
+  cpool->underlay = underlay;
+
+  if (!descriptor_pool_map->insert(
+      std::make_pair(cpool->pool, cpool)).second) {
+    // Should never happen -- would indicate an internal error / bug.
+    PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered");
+    return nullptr;
+  }
+
+  return cpool;
+}
+
+static PyDescriptorPool* PyDescriptorPool_NewWithDatabase(
+    DescriptorDatabase* database) {
+  PyDescriptorPool* cpool = _CreateDescriptorPool();
+  if (cpool == nullptr) {
+    return nullptr;
+  }
+  if (database != nullptr) {
+    cpool->error_collector = new BuildFileErrorCollector();
+    cpool->pool = new DescriptorPool(database, cpool->error_collector);
+    cpool->is_mutable = false;
+    cpool->database = database;
+  } else {
+    cpool->pool = new DescriptorPool();
+    cpool->is_mutable = true;
+  }
+  cpool->is_owned = true;
+
+  if (!descriptor_pool_map->insert(std::make_pair(cpool->pool, cpool)).second) {
+    // Should never happen -- would indicate an internal error / bug.
+    PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered");
+    return nullptr;
+  }
+
+  return cpool;
+}
+
+// The public DescriptorPool constructor.
+static PyObject* New(PyTypeObject* type,
+                     PyObject* args, PyObject* kwargs) {
+  static const char* kwlist[] = {"descriptor_db", nullptr};
+  PyObject* py_database = nullptr;
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O",
+                                   const_cast<char**>(kwlist), &py_database)) {
+    return nullptr;
+  }
+  DescriptorDatabase* database = nullptr;
+  if (py_database && py_database != Py_None) {
+    database = new PyDescriptorDatabase(py_database);
+  }
+  return reinterpret_cast<PyObject*>(
+      PyDescriptorPool_NewWithDatabase(database));
+}
+
+static void Dealloc(PyObject* pself) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
+  descriptor_pool_map->erase(self->pool);
+  Py_CLEAR(self->py_message_factory);
+  for (std::unordered_map<const void*, PyObject*>::iterator it =
+           self->descriptor_options->begin();
+       it != self->descriptor_options->end(); ++it) {
+    Py_DECREF(it->second);
+  }
+  delete self->descriptor_options;
+  delete self->database;
+  if (self->is_owned) {
+    delete self->pool;
+  }
+  delete self->error_collector;
+  Py_TYPE(self)->tp_free(pself);
+}
+
+static int GcTraverse(PyObject* pself, visitproc visit, void* arg) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
+  Py_VISIT(self->py_message_factory);
+  return 0;
+}
+
+static int GcClear(PyObject* pself) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
+  Py_CLEAR(self->py_message_factory);
+  return 0;
+}
+
+PyObject* SetErrorFromCollector(DescriptorPool::ErrorCollector* self,
+                                const char* name, const char* error_type) {
+  BuildFileErrorCollector* error_collector =
+      reinterpret_cast<BuildFileErrorCollector*>(self);
+  if (error_collector && !error_collector->error_message.empty()) {
+    PyErr_Format(PyExc_KeyError, "Couldn't build file for %s %.200s\n%s",
+                 error_type, name, error_collector->error_message.c_str());
+    error_collector->Clear();
+    return nullptr;
+  }
+  PyErr_Format(PyExc_KeyError, "Couldn't find %s %.200s", error_type, name);
+  return nullptr;
+}
+
+static PyObject* FindMessageByName(PyObject* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const Descriptor* message_descriptor =
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMessageTypeByName(
+          absl::string_view(name, name_size));
+
+  if (message_descriptor == nullptr) {
+    return SetErrorFromCollector(
+        reinterpret_cast<PyDescriptorPool*>(self)->error_collector, name,
+        "message");
+  }
+
+
+  return PyMessageDescriptor_FromDescriptor(message_descriptor);
+}
+
+
+
+
+static PyObject* FindFileByName(PyObject* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  PyDescriptorPool* py_pool = reinterpret_cast<PyDescriptorPool*>(self);
+  const FileDescriptor* file_descriptor =
+      py_pool->pool->FindFileByName(absl::string_view(name, name_size));
+
+  if (file_descriptor == nullptr) {
+    return SetErrorFromCollector(py_pool->error_collector, name, "file");
+  }
+  return PyFileDescriptor_FromDescriptor(file_descriptor);
+}
+
+PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const FieldDescriptor* field_descriptor =
+      self->pool->FindFieldByName(absl::string_view(name, name_size));
+  if (field_descriptor == nullptr) {
+    return SetErrorFromCollector(self->error_collector, name, "field");
+  }
+
+
+  return PyFieldDescriptor_FromDescriptor(field_descriptor);
+}
+
+static PyObject* FindFieldByNameMethod(PyObject* self, PyObject* arg) {
+  return FindFieldByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
+PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const FieldDescriptor* field_descriptor =
+      self->pool->FindExtensionByName(absl::string_view(name, name_size));
+  if (field_descriptor == nullptr) {
+    return SetErrorFromCollector(self->error_collector, name,
+                                 "extension field");
+  }
+
+
+  return PyFieldDescriptor_FromDescriptor(field_descriptor);
+}
+
+static PyObject* FindExtensionByNameMethod(PyObject* self, PyObject* arg) {
+  return FindExtensionByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
+PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const EnumDescriptor* enum_descriptor =
+      self->pool->FindEnumTypeByName(absl::string_view(name, name_size));
+  if (enum_descriptor == nullptr) {
+    return SetErrorFromCollector(self->error_collector, name, "enum");
+  }
+
+
+  return PyEnumDescriptor_FromDescriptor(enum_descriptor);
+}
+
+static PyObject* FindEnumTypeByNameMethod(PyObject* self, PyObject* arg) {
+  return FindEnumTypeByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
+PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const OneofDescriptor* oneof_descriptor =
+      self->pool->FindOneofByName(absl::string_view(name, name_size));
+  if (oneof_descriptor == nullptr) {
+    return SetErrorFromCollector(self->error_collector, name, "oneof");
+  }
+
+
+  return PyOneofDescriptor_FromDescriptor(oneof_descriptor);
+}
+
+static PyObject* FindOneofByNameMethod(PyObject* self, PyObject* arg) {
+  return FindOneofByName(reinterpret_cast<PyDescriptorPool*>(self), arg);
+}
+
+static PyObject* FindServiceByName(PyObject* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const ServiceDescriptor* service_descriptor =
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindServiceByName(
+          absl::string_view(name, name_size));
+  if (service_descriptor == nullptr) {
+    return SetErrorFromCollector(
+        reinterpret_cast<PyDescriptorPool*>(self)->error_collector, name,
+        "service");
+  }
+
+
+  return PyServiceDescriptor_FromDescriptor(service_descriptor);
+}
+
+static PyObject* FindMethodByName(PyObject* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const MethodDescriptor* method_descriptor =
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMethodByName(
+          absl::string_view(name, name_size));
+  if (method_descriptor == nullptr) {
+    return SetErrorFromCollector(
+        reinterpret_cast<PyDescriptorPool*>(self)->error_collector, name,
+        "method");
+  }
+
+
+  return PyMethodDescriptor_FromDescriptor(method_descriptor);
+}
+
+static PyObject* FindFileContainingSymbol(PyObject* self, PyObject* arg) {
+  Py_ssize_t name_size;
+  char* name;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  const FileDescriptor* file_descriptor =
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindFileContainingSymbol(
+          absl::string_view(name, name_size));
+  if (file_descriptor == nullptr) {
+    return SetErrorFromCollector(
+        reinterpret_cast<PyDescriptorPool*>(self)->error_collector, name,
+        "symbol");
+  }
+
+
+  return PyFileDescriptor_FromDescriptor(file_descriptor);
+}
+
+static PyObject* FindExtensionByNumber(PyObject* self, PyObject* args) {
+  PyObject* message_descriptor;
+  int number;
+  if (!PyArg_ParseTuple(args, "Oi", &message_descriptor, &number)) {
+    return nullptr;
+  }
+  const Descriptor* descriptor = PyMessageDescriptor_AsDescriptor(
+      message_descriptor);
+  if (descriptor == nullptr) {
+    return nullptr;
+  }
+
+  const FieldDescriptor* extension_descriptor =
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindExtensionByNumber(
+          descriptor, number);
+  if (extension_descriptor == nullptr) {
+    BuildFileErrorCollector* error_collector =
+        reinterpret_cast<BuildFileErrorCollector*>(
+            reinterpret_cast<PyDescriptorPool*>(self)->error_collector);
+    if (error_collector && !error_collector->error_message.empty()) {
+      PyErr_Format(PyExc_KeyError, "Couldn't build file for Extension %.d\n%s",
+                   number, error_collector->error_message.c_str());
+      error_collector->Clear();
+      return nullptr;
+    }
+    PyErr_Format(PyExc_KeyError, "Couldn't find Extension %d", number);
+    return nullptr;
+  }
+
+
+  return PyFieldDescriptor_FromDescriptor(extension_descriptor);
+}
+
+static PyObject* FindAllExtensions(PyObject* self, PyObject* arg) {
+  const Descriptor* descriptor = PyMessageDescriptor_AsDescriptor(arg);
+  if (descriptor == nullptr) {
+    return nullptr;
+  }
+
+  std::vector<const FieldDescriptor*> extensions;
+  reinterpret_cast<PyDescriptorPool*>(self)->pool->FindAllExtensions(
+      descriptor, &extensions);
+
+  ScopedPyObjectPtr result(PyList_New(extensions.size()));
+  if (result == nullptr) {
+    return nullptr;
+  }
+  for (int i = 0; i < extensions.size(); i++) {
+    PyObject* extension = PyFieldDescriptor_FromDescriptor(extensions[i]);
+    if (extension == nullptr) {
+      return nullptr;
+    }
+    PyList_SET_ITEM(result.get(), i, extension);  // Steals the reference.
+  }
+  return result.release();
+}
+
+// These functions should not exist -- the only valid way to create
+// descriptors is to call Add() or AddSerializedFile().
+// But these AddDescriptor() functions were created in Python and some people
+// call them, so we support them for now for compatibility.
+// However we do check that the existing descriptor already exists in the pool,
+// which appears to always be true for existing calls -- but then why do people
+// call a function that will just be a no-op?
+// TODO(amauryfa): Need to investigate further.
+
+static PyObject* AddFileDescriptor(PyObject* self, PyObject* descriptor) {
+  const FileDescriptor* file_descriptor =
+      PyFileDescriptor_AsDescriptor(descriptor);
+  if (!file_descriptor) {
+    return nullptr;
+  }
+  if (file_descriptor !=
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindFileByName(
+          file_descriptor->name())) {
+    PyErr_Format(PyExc_ValueError,
+                 "The file descriptor %s does not belong to this pool",
+                 file_descriptor->name().c_str());
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* AddDescriptor(PyObject* self, PyObject* descriptor) {
+  const Descriptor* message_descriptor =
+      PyMessageDescriptor_AsDescriptor(descriptor);
+  if (!message_descriptor) {
+    return nullptr;
+  }
+  if (message_descriptor !=
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindMessageTypeByName(
+          message_descriptor->full_name())) {
+    PyErr_Format(PyExc_ValueError,
+                 "The message descriptor %s does not belong to this pool",
+                 message_descriptor->full_name().c_str());
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* AddEnumDescriptor(PyObject* self, PyObject* descriptor) {
+  const EnumDescriptor* enum_descriptor =
+      PyEnumDescriptor_AsDescriptor(descriptor);
+  if (!enum_descriptor) {
+    return nullptr;
+  }
+  if (enum_descriptor !=
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindEnumTypeByName(
+          enum_descriptor->full_name())) {
+    PyErr_Format(PyExc_ValueError,
+                 "The enum descriptor %s does not belong to this pool",
+                 enum_descriptor->full_name().c_str());
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* AddExtensionDescriptor(PyObject* self, PyObject* descriptor) {
+  const FieldDescriptor* extension_descriptor =
+      PyFieldDescriptor_AsDescriptor(descriptor);
+  if (!extension_descriptor) {
+    return nullptr;
+  }
+  if (extension_descriptor !=
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindExtensionByName(
+          extension_descriptor->full_name())) {
+    PyErr_Format(PyExc_ValueError,
+                 "The extension descriptor %s does not belong to this pool",
+                 extension_descriptor->full_name().c_str());
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* AddServiceDescriptor(PyObject* self, PyObject* descriptor) {
+  const ServiceDescriptor* service_descriptor =
+      PyServiceDescriptor_AsDescriptor(descriptor);
+  if (!service_descriptor) {
+    return nullptr;
+  }
+  if (service_descriptor !=
+      reinterpret_cast<PyDescriptorPool*>(self)->pool->FindServiceByName(
+          service_descriptor->full_name())) {
+    PyErr_Format(PyExc_ValueError,
+                 "The service descriptor %s does not belong to this pool",
+                 service_descriptor->full_name().c_str());
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+// The code below loads new Descriptors from a serialized FileDescriptorProto.
+static PyObject* AddSerializedFile(PyObject* pself, PyObject* serialized_pb) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
+  char* message_type;
+  Py_ssize_t message_len;
+
+  if (self->database != nullptr) {
+    PyErr_SetString(
+        PyExc_ValueError,
+        "Cannot call Add on a DescriptorPool that uses a DescriptorDatabase. "
+        "Add your file to the underlying database.");
+    return nullptr;
+  }
+  if (!self->is_mutable) {
+    PyErr_SetString(
+        PyExc_ValueError,
+        "This DescriptorPool is not mutable and cannot add new definitions.");
+    return nullptr;
+  }
+
+  if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) {
+    return nullptr;
+  }
+
+  FileDescriptorProto file_proto;
+  if (!file_proto.ParseFromArray(message_type, message_len)) {
+    PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!");
+    return nullptr;
+  }
+
+  // If the file was already part of a C++ library, all its descriptors are in
+  // the underlying pool.  No need to do anything else.
+  const FileDescriptor* generated_file = nullptr;
+  if (self->underlay) {
+    generated_file = self->underlay->FindFileByName(file_proto.name());
+  }
+  if (generated_file != nullptr) {
+    return PyFileDescriptor_FromDescriptorWithSerializedPb(
+        generated_file, serialized_pb);
+  }
+
+  BuildFileErrorCollector error_collector;
+  const FileDescriptor* descriptor =
+      // Pool is mutable, we can remove the "const".
+      const_cast<DescriptorPool*>(self->pool)
+          ->BuildFileCollectingErrors(file_proto, &error_collector);
+  if (descriptor == nullptr) {
+    PyErr_Format(PyExc_TypeError,
+                 "Couldn't build proto file into descriptor pool!\n%s",
+                 error_collector.error_message.c_str());
+    return nullptr;
+  }
+
+
+  return PyFileDescriptor_FromDescriptorWithSerializedPb(
+      descriptor, serialized_pb);
+}
+
+static PyObject* Add(PyObject* self, PyObject* file_descriptor_proto) {
+  ScopedPyObjectPtr serialized_pb(
+      PyObject_CallMethod(file_descriptor_proto, "SerializeToString", nullptr));
+  if (serialized_pb == nullptr) {
+    return nullptr;
+  }
+  return AddSerializedFile(self, serialized_pb.get());
+}
+
+static PyMethodDef Methods[] = {
+    {"Add", Add, METH_O,
+     "Adds the FileDescriptorProto and its types to this pool."},
+    {"AddSerializedFile", AddSerializedFile, METH_O,
+     "Adds a serialized FileDescriptorProto to this pool."},
+
+    // TODO(amauryfa): Understand why the Python implementation differs from
+    // this one, ask users to use another API and deprecate these functions.
+    {"AddFileDescriptor", AddFileDescriptor, METH_O,
+     "No-op. Add() must have been called before."},
+    {"AddDescriptor", AddDescriptor, METH_O,
+     "No-op. Add() must have been called before."},
+    {"AddEnumDescriptor", AddEnumDescriptor, METH_O,
+     "No-op. Add() must have been called before."},
+    {"AddExtensionDescriptor", AddExtensionDescriptor, METH_O,
+     "No-op. Add() must have been called before."},
+    {"AddServiceDescriptor", AddServiceDescriptor, METH_O,
+     "No-op. Add() must have been called before."},
+
+    {"FindFileByName", FindFileByName, METH_O,
+     "Searches for a file descriptor by its .proto name."},
+    {"FindMessageTypeByName", FindMessageByName, METH_O,
+     "Searches for a message descriptor by full name."},
+    {"FindFieldByName", FindFieldByNameMethod, METH_O,
+     "Searches for a field descriptor by full name."},
+    {"FindExtensionByName", FindExtensionByNameMethod, METH_O,
+     "Searches for extension descriptor by full name."},
+    {"FindEnumTypeByName", FindEnumTypeByNameMethod, METH_O,
+     "Searches for enum type descriptor by full name."},
+    {"FindOneofByName", FindOneofByNameMethod, METH_O,
+     "Searches for oneof descriptor by full name."},
+    {"FindServiceByName", FindServiceByName, METH_O,
+     "Searches for service descriptor by full name."},
+    {"FindMethodByName", FindMethodByName, METH_O,
+     "Searches for method descriptor by full name."},
+
+    {"FindFileContainingSymbol", FindFileContainingSymbol, METH_O,
+     "Gets the FileDescriptor containing the specified symbol."},
+    {"FindExtensionByNumber", FindExtensionByNumber, METH_VARARGS,
+     "Gets the extension descriptor for the given number."},
+    {"FindAllExtensions", FindAllExtensions, METH_O,
+     "Gets all known extensions of the given message descriptor."},
+    {nullptr},
+};
+
+}  // namespace cdescriptor_pool
+
+PyTypeObject PyDescriptorPool_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".DescriptorPool",          // tp_name
+    sizeof(PyDescriptorPool),   // tp_basicsize
+    0,                          // tp_itemsize
+    cdescriptor_pool::Dealloc,  // tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                                  // tp_getattr
+    nullptr,                                  // tp_setattr
+    nullptr,                                  // tp_compare
+    nullptr,                                  // tp_repr
+    nullptr,                                  // tp_as_number
+    nullptr,                                  // tp_as_sequence
+    nullptr,                                  // tp_as_mapping
+    nullptr,                                  // tp_hash
+    nullptr,                                  // tp_call
+    nullptr,                                  // tp_str
+    nullptr,                                  // tp_getattro
+    nullptr,                                  // tp_setattro
+    nullptr,                                  // tp_as_buffer
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,  // tp_flags
+    "A Descriptor Pool",                      // tp_doc
+    cdescriptor_pool::GcTraverse,             // tp_traverse
+    cdescriptor_pool::GcClear,                // tp_clear
+    nullptr,                                  // tp_richcompare
+    0,                                        // tp_weaklistoffset
+    nullptr,                                  // tp_iter
+    nullptr,                                  // tp_iternext
+    cdescriptor_pool::Methods,                // tp_methods
+    nullptr,                                  // tp_members
+    nullptr,                                  // tp_getset
+    nullptr,                                  // tp_base
+    nullptr,                                  // tp_dict
+    nullptr,                                  // tp_descr_get
+    nullptr,                                  // tp_descr_set
+    0,                                        // tp_dictoffset
+    nullptr,                                  // tp_init
+    nullptr,                                  // tp_alloc
+    cdescriptor_pool::New,                    // tp_new
+    PyObject_GC_Del,                          // tp_free
+};
+
+// This is the DescriptorPool which contains all the definitions from the
+// generated _pb2.py modules.
+static PyDescriptorPool* python_generated_pool = nullptr;
+
+bool InitDescriptorPool() {
+  if (PyType_Ready(&PyDescriptorPool_Type) < 0)
+    return false;
+
+  // The Pool of messages declared in Python libraries.
+  // generated_pool() contains all messages already linked in C++ libraries, and
+  // is used as underlay.
+  descriptor_pool_map =
+      new std::unordered_map<const DescriptorPool*, PyDescriptorPool*>;
+  python_generated_pool = cdescriptor_pool::PyDescriptorPool_NewWithUnderlay(
+      DescriptorPool::generated_pool());
+  if (python_generated_pool == nullptr) {
+    delete descriptor_pool_map;
+    return false;
+  }
+
+  // Register this pool to be found for C++-generated descriptors.
+  descriptor_pool_map->insert(
+      std::make_pair(DescriptorPool::generated_pool(),
+                     python_generated_pool));
+
+  return true;
+}
+
+// The default DescriptorPool used everywhere in this module.
+// Today it's the python_generated_pool.
+// TODO(amauryfa): Remove all usages of this function: the pool should be
+// derived from the context.
+PyDescriptorPool* GetDefaultDescriptorPool() {
+  return python_generated_pool;
+}
+
+PyDescriptorPool* GetDescriptorPool_FromPool(const DescriptorPool* pool) {
+  // Fast path for standard descriptors.
+  if (pool == python_generated_pool->pool ||
+      pool == DescriptorPool::generated_pool()) {
+    return python_generated_pool;
+  }
+  std::unordered_map<const DescriptorPool*, PyDescriptorPool*>::iterator it =
+      descriptor_pool_map->find(pool);
+  if (it == descriptor_pool_map->end()) {
+    PyErr_SetString(PyExc_KeyError, "Unknown descriptor pool");
+    return nullptr;
+  }
+  return it->second;
+}
+
+PyObject* PyDescriptorPool_FromPool(const DescriptorPool* pool) {
+  PyDescriptorPool* existing_pool = GetDescriptorPool_FromPool(pool);
+  if (existing_pool != nullptr) {
+    Py_INCREF(existing_pool);
+    return reinterpret_cast<PyObject*>(existing_pool);
+  } else {
+    PyErr_Clear();
+  }
+
+  PyDescriptorPool* cpool = cdescriptor_pool::_CreateDescriptorPool();
+  if (cpool == nullptr) {
+    return nullptr;
+  }
+  cpool->pool = const_cast<DescriptorPool*>(pool);
+  cpool->is_owned = false;
+  cpool->is_mutable = false;
+  cpool->underlay = nullptr;
+
+  if (!descriptor_pool_map->insert(std::make_pair(cpool->pool, cpool)).second) {
+    // Should never happen -- We already checked the existence above.
+    PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered");
+    return nullptr;
+  }
+
+  return reinterpret_cast<PyObject*>(cpool);
+}
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/descriptor_pool.h

@@ -0,0 +1,149 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <unordered_map>
+#include "google/protobuf/descriptor.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+struct PyMessageFactory;
+
+// The (meta) type of all Messages classes.
+struct CMessageClass;
+
+// Wraps operations to the global DescriptorPool which contains information
+// about all messages and fields.
+//
+// There is normally one pool per process. We make it a Python object only
+// because it contains many Python references.
+//
+// "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
+// namespace.
+typedef struct PyDescriptorPool {
+  PyObject_HEAD;
+
+  // The C++ pool containing Descriptors.
+  const DescriptorPool* pool;
+
+  // True if we should free the pointer above.
+  bool is_owned;
+
+  // True if this pool accepts new proto definitions.
+  // In this case it is allowed to const_cast<DescriptorPool*>(pool).
+  bool is_mutable;
+
+
+  // The error collector to store error info. Can be NULL. This pointer is
+  // owned.
+  DescriptorPool::ErrorCollector* error_collector;
+
+  // The C++ pool acting as an underlay. Can be NULL.
+  // This pointer is not owned and must stay alive.
+  const DescriptorPool* underlay;
+
+  // The C++ descriptor database used to fetch unknown protos. Can be NULL.
+  // This pointer is owned.
+  const DescriptorDatabase* database;
+
+  // The preferred MessageFactory to be used by descriptors.
+  // TODO(amauryfa): Don't create the Factory from the DescriptorPool, but
+  // use the one passed while creating message classes. And remove this member.
+  PyMessageFactory* py_message_factory;
+
+  // Cache the options for any kind of descriptor.
+  // Descriptor pointers are owned by the DescriptorPool above.
+  // Python objects are owned by the map.
+  std::unordered_map<const void*, PyObject*>* descriptor_options;
+} PyDescriptorPool;
+
+
+extern PyTypeObject PyDescriptorPool_Type;
+
+namespace cdescriptor_pool {
+
+
+// The functions below are also exposed as methods of the DescriptorPool type.
+
+// Looks up a field by name. Returns a PyFieldDescriptor corresponding to
+// the field on success, or NULL on failure.
+//
+// Returns a new reference.
+PyObject* FindFieldByName(PyDescriptorPool* self, PyObject* name);
+
+// Looks up an extension by name. Returns a PyFieldDescriptor corresponding
+// to the field on success, or NULL on failure.
+//
+// Returns a new reference.
+PyObject* FindExtensionByName(PyDescriptorPool* self, PyObject* arg);
+
+// Looks up an enum type by name. Returns a PyEnumDescriptor corresponding
+// to the field on success, or NULL on failure.
+//
+// Returns a new reference.
+PyObject* FindEnumTypeByName(PyDescriptorPool* self, PyObject* arg);
+
+// Looks up a oneof by name. Returns a COneofDescriptor corresponding
+// to the oneof on success, or NULL on failure.
+//
+// Returns a new reference.
+PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg);
+
+}  // namespace cdescriptor_pool
+
+// Retrieves the global descriptor pool owned by the _message module.
+// This is the one used by pb2.py generated modules.
+// Returns a *borrowed* reference.
+// "Default" pool used to register messages from _pb2.py modules.
+PyDescriptorPool* GetDefaultDescriptorPool();
+
+// Retrieves an existing python descriptor pool owning the C++ descriptor pool.
+// Returns a *borrowed* reference.
+PyDescriptorPool* GetDescriptorPool_FromPool(const DescriptorPool* pool);
+
+// Wraps a C++ descriptor pool in a Python object, creates it if necessary.
+// Returns a new reference.
+PyObject* PyDescriptorPool_FromPool(const DescriptorPool* pool);
+
+// Initialize objects used by this module.
+bool InitDescriptorPool();
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__

python/google/protobuf/pyext/extension_dict.cc

@@ -0,0 +1,487 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: anuraag@google.com (Anuraag Agrawal)
+// Author: tibell@google.com (Johan Tibell)
+
+#include "google/protobuf/pyext/extension_dict.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "google/protobuf/stubs/logging.h"
+#include "google/protobuf/stubs/common.h"
+#include "google/protobuf/descriptor.pb.h"
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/dynamic_message.h"
+#include "google/protobuf/message.h"
+#include "google/protobuf/pyext/descriptor.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/message_factory.h"
+#include "google/protobuf/pyext/repeated_composite_container.h"
+#include "google/protobuf/pyext/repeated_scalar_container.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+#include "absl/strings/string_view.h"
+
+#define PyString_AsStringAndSize(ob, charpp, sizep)              \
+  (PyUnicode_Check(ob)                                           \
+       ? ((*(charpp) = const_cast<char*>(                        \
+               PyUnicode_AsUTF8AndSize(ob, (sizep)))) == nullptr \
+              ? -1                                               \
+              : 0)                                               \
+       : PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+namespace extension_dict {
+
+static Py_ssize_t len(ExtensionDict* self) {
+  Py_ssize_t size = 0;
+  std::vector<const FieldDescriptor*> fields;
+  self->parent->message->GetReflection()->ListFields(*self->parent->message,
+                                                     &fields);
+
+  for (size_t i = 0; i < fields.size(); ++i) {
+    if (fields[i]->is_extension()) {
+      // With C++ descriptors, the field can always be retrieved, but for
+      // unknown extensions which have not been imported in Python code, there
+      // is no message class and we cannot retrieve the value.
+      // ListFields() has the same behavior.
+      if (fields[i]->message_type() != nullptr &&
+          message_factory::GetMessageClass(
+              cmessage::GetFactoryForMessage(self->parent),
+              fields[i]->message_type()) == nullptr) {
+        PyErr_Clear();
+        continue;
+      }
+      ++size;
+    }
+  }
+  return size;
+}
+
+struct ExtensionIterator {
+  PyObject_HEAD;
+  Py_ssize_t index;
+  std::vector<const FieldDescriptor*> fields;
+
+  // Owned reference, to keep the FieldDescriptors alive.
+  ExtensionDict* extension_dict;
+};
+
+PyObject* GetIter(PyObject* _self) {
+  ExtensionDict* self = reinterpret_cast<ExtensionDict*>(_self);
+
+  ScopedPyObjectPtr obj(PyType_GenericAlloc(&ExtensionIterator_Type, 0));
+  if (obj == nullptr) {
+    return PyErr_Format(PyExc_MemoryError,
+                        "Could not allocate extension iterator");
+  }
+
+  ExtensionIterator* iter = reinterpret_cast<ExtensionIterator*>(obj.get());
+
+  // Call "placement new" to initialize. So the constructor of
+  // std::vector<...> fields will be called.
+  new (iter) ExtensionIterator;
+
+  self->parent->message->GetReflection()->ListFields(*self->parent->message,
+                                                     &iter->fields);
+  iter->index = 0;
+  Py_INCREF(self);
+  iter->extension_dict = self;
+
+  return obj.release();
+}
+
+static void DeallocExtensionIterator(PyObject* _self) {
+  ExtensionIterator* self = reinterpret_cast<ExtensionIterator*>(_self);
+  self->fields.clear();
+  Py_XDECREF(self->extension_dict);
+  freefunc tp_free = Py_TYPE(_self)->tp_free;
+  self->~ExtensionIterator();
+  (*tp_free)(_self);
+}
+
+PyObject* subscript(ExtensionDict* self, PyObject* key) {
+  const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key);
+  if (descriptor == nullptr) {
+    return nullptr;
+  }
+  if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
+    return nullptr;
+  }
+
+  if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
+      descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
+    return cmessage::InternalGetScalar(self->parent->message, descriptor);
+  }
+
+  CMessage::CompositeFieldsMap::iterator iterator =
+      self->parent->composite_fields->find(descriptor);
+  if (iterator != self->parent->composite_fields->end()) {
+    Py_INCREF(iterator->second);
+    return iterator->second->AsPyObject();
+  }
+
+  if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
+      descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+    // TODO(plabatut): consider building the class on the fly!
+    ContainerBase* sub_message = cmessage::InternalGetSubMessage(
+        self->parent, descriptor);
+    if (sub_message == nullptr) {
+      return nullptr;
+    }
+    (*self->parent->composite_fields)[descriptor] = sub_message;
+    return sub_message->AsPyObject();
+  }
+
+  if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
+    if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+      // On the fly message class creation is needed to support the following
+      // situation:
+      // 1- add FileDescriptor to the pool that contains extensions of a message
+      //    defined by another proto file. Do not create any message classes.
+      // 2- instantiate an extended message, and access the extension using
+      //    the field descriptor.
+      // 3- the extension submessage fails to be returned, because no class has
+      //    been created.
+      // It happens when deserializing text proto format, or when enumerating
+      // fields of a deserialized message.
+      CMessageClass* message_class = message_factory::GetOrCreateMessageClass(
+          cmessage::GetFactoryForMessage(self->parent),
+          descriptor->message_type());
+      ScopedPyObjectPtr message_class_handler(
+        reinterpret_cast<PyObject*>(message_class));
+      if (message_class == nullptr) {
+        return nullptr;
+      }
+      ContainerBase* py_container = repeated_composite_container::NewContainer(
+          self->parent, descriptor, message_class);
+      if (py_container == nullptr) {
+        return nullptr;
+      }
+      (*self->parent->composite_fields)[descriptor] = py_container;
+      return py_container->AsPyObject();
+    } else {
+      ContainerBase* py_container = repeated_scalar_container::NewContainer(
+          self->parent, descriptor);
+      if (py_container == nullptr) {
+        return nullptr;
+      }
+      (*self->parent->composite_fields)[descriptor] = py_container;
+      return py_container->AsPyObject();
+    }
+  }
+  PyErr_SetString(PyExc_ValueError, "control reached unexpected line");
+  return nullptr;
+}
+
+int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
+  const FieldDescriptor* descriptor = cmessage::GetExtensionDescriptor(key);
+  if (descriptor == nullptr) {
+    return -1;
+  }
+  if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) {
+    return -1;
+  }
+
+  if (value == nullptr) {
+    return cmessage::ClearFieldByDescriptor(self->parent, descriptor);
+  }
+
+  if (descriptor->label() != FieldDescriptor::LABEL_OPTIONAL ||
+      descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+    PyErr_SetString(PyExc_TypeError, "Extension is repeated and/or composite "
+                    "type");
+    return -1;
+  }
+  cmessage::AssureWritable(self->parent);
+  if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) {
+    return -1;
+  }
+  return 0;
+}
+
+PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* arg) {
+  char* name;
+  Py_ssize_t name_size;
+  if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) {
+    return nullptr;
+  }
+
+  PyDescriptorPool* pool = cmessage::GetFactoryForMessage(self->parent)->pool;
+  const FieldDescriptor* message_extension =
+      pool->pool->FindExtensionByName(absl::string_view(name, name_size));
+  if (message_extension == nullptr) {
+    // Is is the name of a message set extension?
+    const Descriptor* message_descriptor =
+        pool->pool->FindMessageTypeByName(absl::string_view(name, name_size));
+    if (message_descriptor && message_descriptor->extension_count() > 0) {
+      const FieldDescriptor* extension = message_descriptor->extension(0);
+      if (extension->is_extension() &&
+          extension->containing_type()->options().message_set_wire_format() &&
+          extension->type() == FieldDescriptor::TYPE_MESSAGE &&
+          extension->label() == FieldDescriptor::LABEL_OPTIONAL) {
+        message_extension = extension;
+      }
+    }
+  }
+  if (message_extension == nullptr) {
+    Py_RETURN_NONE;
+  }
+
+  return PyFieldDescriptor_FromDescriptor(message_extension);
+}
+
+PyObject* _FindExtensionByNumber(ExtensionDict* self, PyObject* arg) {
+  int64_t number = PyLong_AsLong(arg);
+  if (number == -1 && PyErr_Occurred()) {
+    return nullptr;
+  }
+
+  PyDescriptorPool* pool = cmessage::GetFactoryForMessage(self->parent)->pool;
+  const FieldDescriptor* message_extension = pool->pool->FindExtensionByNumber(
+      self->parent->message->GetDescriptor(), number);
+  if (message_extension == nullptr) {
+    Py_RETURN_NONE;
+  }
+
+  return PyFieldDescriptor_FromDescriptor(message_extension);
+}
+
+static int Contains(PyObject* _self, PyObject* key) {
+  ExtensionDict* self = reinterpret_cast<ExtensionDict*>(_self);
+  const FieldDescriptor* field_descriptor =
+      cmessage::GetExtensionDescriptor(key);
+  if (field_descriptor == nullptr) {
+    return -1;
+  }
+
+  if (!field_descriptor->is_extension()) {
+    PyErr_Format(PyExc_KeyError, "%s is not an extension",
+                 field_descriptor->full_name().c_str());
+    return -1;
+  }
+
+  const Message* message = self->parent->message;
+  const Reflection* reflection = message->GetReflection();
+  if (field_descriptor->is_repeated()) {
+    if (reflection->FieldSize(*message, field_descriptor) > 0) {
+      return 1;
+    }
+  } else {
+    if (reflection->HasField(*message, field_descriptor)) {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+ExtensionDict* NewExtensionDict(CMessage *parent) {
+  ExtensionDict* self = reinterpret_cast<ExtensionDict*>(
+      PyType_GenericAlloc(&ExtensionDict_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  return self;
+}
+
+void dealloc(PyObject* pself) {
+  ExtensionDict* self = reinterpret_cast<ExtensionDict*>(pself);
+  Py_CLEAR(self->parent);
+  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+}
+
+static PyObject* RichCompare(ExtensionDict* self, PyObject* other, int opid) {
+  // Only equality comparisons are implemented.
+  if (opid != Py_EQ && opid != Py_NE) {
+    Py_INCREF(Py_NotImplemented);
+    return Py_NotImplemented;
+  }
+  bool equals = false;
+  if (PyObject_TypeCheck(other, &ExtensionDict_Type)) {
+    equals = self->parent == reinterpret_cast<ExtensionDict*>(other)->parent;
+  }
+  if (equals ^ (opid == Py_EQ)) {
+    Py_RETURN_FALSE;
+  } else {
+    Py_RETURN_TRUE;
+  }
+}
+static PySequenceMethods SeqMethods = {
+    (lenfunc)len,          // sq_length
+    nullptr,               // sq_concat
+    nullptr,               // sq_repeat
+    nullptr,               // sq_item
+    nullptr,               // sq_slice
+    nullptr,               // sq_ass_item
+    nullptr,               // sq_ass_slice
+    (objobjproc)Contains,  // sq_contains
+};
+
+static PyMappingMethods MpMethods = {
+  (lenfunc)len,                /* mp_length */
+  (binaryfunc)subscript,       /* mp_subscript */
+  (objobjargproc)ass_subscript,/* mp_ass_subscript */
+};
+
+#define EDMETHOD(name, args, doc) { #name, (PyCFunction)name, args, doc }
+static PyMethodDef Methods[] = {
+    EDMETHOD(_FindExtensionByName, METH_O, "Finds an extension by name."),
+    EDMETHOD(_FindExtensionByNumber, METH_O,
+             "Finds an extension by field number."),
+    {nullptr, nullptr},
+};
+
+}  // namespace extension_dict
+
+PyTypeObject ExtensionDict_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)  //
+    FULL_MODULE_NAME ".ExtensionDict",      // tp_name
+    sizeof(ExtensionDict),                  // tp_basicsize
+    0,                                      //  tp_itemsize
+    (destructor)extension_dict::dealloc,    //  tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                                   //  tp_getattr
+    nullptr,                                   //  tp_setattr
+    nullptr,                                   //  tp_compare
+    nullptr,                                   //  tp_repr
+    nullptr,                                   //  tp_as_number
+    &extension_dict::SeqMethods,               //  tp_as_sequence
+    &extension_dict::MpMethods,                //  tp_as_mapping
+    PyObject_HashNotImplemented,               //  tp_hash
+    nullptr,                                   //  tp_call
+    nullptr,                                   //  tp_str
+    nullptr,                                   //  tp_getattro
+    nullptr,                                   //  tp_setattro
+    nullptr,                                   //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,                        //  tp_flags
+    "An extension dict",                       //  tp_doc
+    nullptr,                                   //  tp_traverse
+    nullptr,                                   //  tp_clear
+    (richcmpfunc)extension_dict::RichCompare,  //  tp_richcompare
+    0,                                         //  tp_weaklistoffset
+    extension_dict::GetIter,                   //  tp_iter
+    nullptr,                                   //  tp_iternext
+    extension_dict::Methods,                   //  tp_methods
+    nullptr,                                   //  tp_members
+    nullptr,                                   //  tp_getset
+    nullptr,                                   //  tp_base
+    nullptr,                                   //  tp_dict
+    nullptr,                                   //  tp_descr_get
+    nullptr,                                   //  tp_descr_set
+    0,                                         //  tp_dictoffset
+    nullptr,                                   //  tp_init
+};
+
+PyObject* IterNext(PyObject* _self) {
+  extension_dict::ExtensionIterator* self =
+      reinterpret_cast<extension_dict::ExtensionIterator*>(_self);
+  Py_ssize_t total_size = self->fields.size();
+  Py_ssize_t index = self->index;
+  while (self->index < total_size) {
+    index = self->index;
+    ++self->index;
+    if (self->fields[index]->is_extension()) {
+      // With C++ descriptors, the field can always be retrieved, but for
+      // unknown extensions which have not been imported in Python code, there
+      // is no message class and we cannot retrieve the value.
+      // ListFields() has the same behavior.
+      if (self->fields[index]->message_type() != nullptr &&
+          message_factory::GetMessageClass(
+              cmessage::GetFactoryForMessage(self->extension_dict->parent),
+              self->fields[index]->message_type()) == nullptr) {
+        PyErr_Clear();
+        continue;
+      }
+
+      return PyFieldDescriptor_FromDescriptor(self->fields[index]);
+    }
+  }
+
+  return nullptr;
+}
+
+PyTypeObject ExtensionIterator_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)      //
+    FULL_MODULE_NAME ".ExtensionIterator",      //  tp_name
+    sizeof(extension_dict::ExtensionIterator),  //  tp_basicsize
+    0,                                          //  tp_itemsize
+    extension_dict::DeallocExtensionIterator,   //  tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                  //  tp_getattr
+    nullptr,                  //  tp_setattr
+    nullptr,                  //  tp_compare
+    nullptr,                  //  tp_repr
+    nullptr,                  //  tp_as_number
+    nullptr,                  //  tp_as_sequence
+    nullptr,                  //  tp_as_mapping
+    nullptr,                  //  tp_hash
+    nullptr,                  //  tp_call
+    nullptr,                  //  tp_str
+    nullptr,                  //  tp_getattro
+    nullptr,                  //  tp_setattro
+    nullptr,                  //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,       //  tp_flags
+    "A scalar map iterator",  //  tp_doc
+    nullptr,                  //  tp_traverse
+    nullptr,                  //  tp_clear
+    nullptr,                  //  tp_richcompare
+    0,                        //  tp_weaklistoffset
+    PyObject_SelfIter,        //  tp_iter
+    IterNext,                 //  tp_iternext
+    nullptr,                  //  tp_methods
+    nullptr,                  //  tp_members
+    nullptr,                  //  tp_getset
+    nullptr,                  //  tp_base
+    nullptr,                  //  tp_dict
+    nullptr,                  //  tp_descr_get
+    nullptr,                  //  tp_descr_set
+    0,                        //  tp_dictoffset
+    nullptr,                  //  tp_init
+};
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/extension_dict.h

@@ -0,0 +1,70 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: anuraag@google.com (Anuraag Agrawal)
+// Author: tibell@google.com (Johan Tibell)
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "google/protobuf/pyext/message.h"
+
+namespace google {
+namespace protobuf {
+
+class Message;
+class FieldDescriptor;
+
+namespace python {
+
+typedef struct ExtensionDict {
+  PyObject_HEAD;
+
+  // Strong, owned reference to the parent message. Never NULL.
+  CMessage* parent;
+} ExtensionDict;
+
+extern PyTypeObject ExtensionDict_Type;
+extern PyTypeObject ExtensionIterator_Type;
+
+namespace extension_dict {
+
+// Builds an Extensions dict for a specific message.
+ExtensionDict* NewExtensionDict(CMessage *parent);
+
+}  // namespace extension_dict
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__

python/google/protobuf/pyext/field.cc

@@ -0,0 +1,143 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "google/protobuf/pyext/field.h"
+
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/pyext/descriptor.h"
+#include "google/protobuf/pyext/message.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+namespace field {
+
+static PyObject* Repr(PyMessageFieldProperty* self) {
+  return PyUnicode_FromFormat("<field property '%s'>",
+                              self->field_descriptor->full_name().c_str());
+}
+
+static PyObject* DescrGet(PyMessageFieldProperty* self, PyObject* obj,
+                          PyObject* type) {
+  if (obj == nullptr) {
+    Py_INCREF(self);
+    return reinterpret_cast<PyObject*>(self);
+  }
+  return cmessage::GetFieldValue(reinterpret_cast<CMessage*>(obj),
+                                 self->field_descriptor);
+}
+
+static int DescrSet(PyMessageFieldProperty* self, PyObject* obj,
+                    PyObject* value) {
+  if (value == nullptr) {
+    PyErr_SetString(PyExc_AttributeError, "Cannot delete field attribute");
+    return -1;
+  }
+  return cmessage::SetFieldValue(reinterpret_cast<CMessage*>(obj),
+                                 self->field_descriptor, value);
+}
+
+static PyObject* GetDescriptor(PyMessageFieldProperty* self, void* closure) {
+  return PyFieldDescriptor_FromDescriptor(self->field_descriptor);
+}
+
+static PyObject* GetDoc(PyMessageFieldProperty* self, void* closure) {
+  return PyUnicode_FromFormat("Field %s",
+                              self->field_descriptor->full_name().c_str());
+}
+
+static PyGetSetDef Getters[] = {
+    {"DESCRIPTOR", (getter)GetDescriptor, nullptr, "Field descriptor"},
+    {"__doc__", (getter)GetDoc, nullptr, nullptr},
+    {nullptr},
+};
+}  // namespace field
+
+static PyTypeObject _CFieldProperty_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)  // head
+    FULL_MODULE_NAME ".FieldProperty",      // tp_name
+    sizeof(PyMessageFieldProperty),         // tp_basicsize
+    0,                                      // tp_itemsize
+    nullptr,                                // tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                        // tp_getattr
+    nullptr,                        // tp_setattr
+    nullptr,                        // tp_compare
+    (reprfunc)field::Repr,          // tp_repr
+    nullptr,                        // tp_as_number
+    nullptr,                        // tp_as_sequence
+    nullptr,                        // tp_as_mapping
+    nullptr,                        // tp_hash
+    nullptr,                        // tp_call
+    nullptr,                        // tp_str
+    nullptr,                        // tp_getattro
+    nullptr,                        // tp_setattro
+    nullptr,                        // tp_as_buffer
+    Py_TPFLAGS_DEFAULT,             // tp_flags
+    "Field property of a Message",  // tp_doc
+    nullptr,                        // tp_traverse
+    nullptr,                        // tp_clear
+    nullptr,                        // tp_richcompare
+    0,                              // tp_weaklistoffset
+    nullptr,                        // tp_iter
+    nullptr,                        // tp_iternext
+    nullptr,                        // tp_methods
+    nullptr,                        // tp_members
+    field::Getters,                 // tp_getset
+    nullptr,                        // tp_base
+    nullptr,                        // tp_dict
+    (descrgetfunc)field::DescrGet,  // tp_descr_get
+    (descrsetfunc)field::DescrSet,  // tp_descr_set
+    0,                              // tp_dictoffset
+    nullptr,                        // tp_init
+    nullptr,                        // tp_alloc
+    nullptr,                        // tp_new
+};
+PyTypeObject* CFieldProperty_Type = &_CFieldProperty_Type;
+
+PyObject* NewFieldProperty(const FieldDescriptor* field_descriptor) {
+  // Create a new descriptor object
+  PyMessageFieldProperty* property =
+      PyObject_New(PyMessageFieldProperty, CFieldProperty_Type);
+  if (property == nullptr) {
+    return nullptr;
+  }
+  property->field_descriptor = field_descriptor;
+  return reinterpret_cast<PyObject*>(property);
+}
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/field.h

@@ -0,0 +1,60 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+namespace google {
+namespace protobuf {
+
+class FieldDescriptor;
+
+namespace python {
+
+// A data descriptor that represents a field in a Message class.
+struct PyMessageFieldProperty {
+  PyObject_HEAD;
+
+  // This pointer is owned by the same pool as the Message class it belongs to.
+  const FieldDescriptor* field_descriptor;
+};
+
+extern PyTypeObject* CFieldProperty_Type;
+
+PyObject* NewFieldProperty(const FieldDescriptor* field_descriptor);
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__

python/google/protobuf/pyext/map_container.cc

@@ -0,0 +1,931 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: haberman@google.com (Josh Haberman)
+
+#include "google/protobuf/pyext/map_container.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "google/protobuf/stubs/logging.h"
+#include "google/protobuf/stubs/common.h"
+#include "google/protobuf/map.h"
+#include "google/protobuf/map_field.h"
+#include "google/protobuf/message.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/message_factory.h"
+#include "google/protobuf/pyext/repeated_composite_container.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+// Functions that need access to map reflection functionality.
+// They need to be contained in this class because it is friended.
+class MapReflectionFriend {
+ public:
+  // Methods that are in common between the map types.
+  static PyObject* Contains(PyObject* _self, PyObject* key);
+  static Py_ssize_t Length(PyObject* _self);
+  static PyObject* GetIterator(PyObject *_self);
+  static PyObject* IterNext(PyObject* _self);
+  static PyObject* MergeFrom(PyObject* _self, PyObject* arg);
+
+  // Methods that differ between the map types.
+  static PyObject* ScalarMapGetItem(PyObject* _self, PyObject* key);
+  static PyObject* MessageMapGetItem(PyObject* _self, PyObject* key);
+  static int ScalarMapSetItem(PyObject* _self, PyObject* key, PyObject* v);
+  static int MessageMapSetItem(PyObject* _self, PyObject* key, PyObject* v);
+  static PyObject* ScalarMapToStr(PyObject* _self);
+  static PyObject* MessageMapToStr(PyObject* _self);
+};
+
+struct MapIterator {
+  PyObject_HEAD;
+
+  std::unique_ptr<::google::protobuf::MapIterator> iter;
+
+  // A pointer back to the container, so we can notice changes to the version.
+  // We own a ref on this.
+  MapContainer* container;
+
+  // We need to keep a ref on the parent Message too, because
+  // MapIterator::~MapIterator() accesses it.  Normally this would be ok because
+  // the ref on container (above) would guarantee outlive semantics.  However in
+  // the case of ClearField(), the MapContainer points to a different message,
+  // a copy of the original.  But our iterator still points to the original,
+  // which could now get deleted before us.
+  //
+  // To prevent this, we ensure that the Message will always stay alive as long
+  // as this iterator does.  This is solely for the benefit of the MapIterator
+  // destructor -- we should never actually access the iterator in this state
+  // except to delete it.
+  CMessage* parent;
+  // The version of the map when we took the iterator to it.
+  //
+  // We store this so that if the map is modified during iteration we can throw
+  // an error.
+  uint64_t version;
+};
+
+Message* MapContainer::GetMutableMessage() {
+  cmessage::AssureWritable(parent);
+  return parent->message;
+}
+
+// Consumes a reference on the Python string object.
+static bool PyStringToSTL(PyObject* py_string, std::string* stl_string) {
+  char *value;
+  Py_ssize_t value_len;
+
+  if (!py_string) {
+    return false;
+  }
+  if (PyBytes_AsStringAndSize(py_string, &value, &value_len) < 0) {
+    Py_DECREF(py_string);
+    return false;
+  } else {
+    stl_string->assign(value, value_len);
+    Py_DECREF(py_string);
+    return true;
+  }
+}
+
+static bool PythonToMapKey(MapContainer* self, PyObject* obj, MapKey* key) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_key();
+  switch (field_descriptor->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32: {
+      GOOGLE_CHECK_GET_INT32(obj, value, false);
+      key->SetInt32Value(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_INT64: {
+      GOOGLE_CHECK_GET_INT64(obj, value, false);
+      key->SetInt64Value(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT32: {
+      GOOGLE_CHECK_GET_UINT32(obj, value, false);
+      key->SetUInt32Value(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT64: {
+      GOOGLE_CHECK_GET_UINT64(obj, value, false);
+      key->SetUInt64Value(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_BOOL: {
+      GOOGLE_CHECK_GET_BOOL(obj, value, false);
+      key->SetBoolValue(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_STRING: {
+      std::string str;
+      if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) {
+        return false;
+      }
+      key->SetStringValue(str);
+      break;
+    }
+    default:
+      PyErr_Format(
+          PyExc_SystemError, "Type %d cannot be a map key",
+          field_descriptor->cpp_type());
+      return false;
+  }
+  return true;
+}
+
+static PyObject* MapKeyToPython(MapContainer* self, const MapKey& key) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_key();
+  switch (field_descriptor->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32:
+      return PyLong_FromLong(key.GetInt32Value());
+    case FieldDescriptor::CPPTYPE_INT64:
+      return PyLong_FromLongLong(key.GetInt64Value());
+    case FieldDescriptor::CPPTYPE_UINT32:
+      return PyLong_FromSize_t(key.GetUInt32Value());
+    case FieldDescriptor::CPPTYPE_UINT64:
+      return PyLong_FromUnsignedLongLong(key.GetUInt64Value());
+    case FieldDescriptor::CPPTYPE_BOOL:
+      return PyBool_FromLong(key.GetBoolValue());
+    case FieldDescriptor::CPPTYPE_STRING:
+      return ToStringObject(field_descriptor, key.GetStringValue());
+    default:
+      PyErr_Format(
+          PyExc_SystemError, "Couldn't convert type %d to value",
+          field_descriptor->cpp_type());
+      return nullptr;
+  }
+}
+
+// This is only used for ScalarMap, so we don't need to handle the
+// CPPTYPE_MESSAGE case.
+PyObject* MapValueRefToPython(MapContainer* self, const MapValueRef& value) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_value();
+  switch (field_descriptor->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32:
+      return PyLong_FromLong(value.GetInt32Value());
+    case FieldDescriptor::CPPTYPE_INT64:
+      return PyLong_FromLongLong(value.GetInt64Value());
+    case FieldDescriptor::CPPTYPE_UINT32:
+      return PyLong_FromSize_t(value.GetUInt32Value());
+    case FieldDescriptor::CPPTYPE_UINT64:
+      return PyLong_FromUnsignedLongLong(value.GetUInt64Value());
+    case FieldDescriptor::CPPTYPE_FLOAT:
+      return PyFloat_FromDouble(value.GetFloatValue());
+    case FieldDescriptor::CPPTYPE_DOUBLE:
+      return PyFloat_FromDouble(value.GetDoubleValue());
+    case FieldDescriptor::CPPTYPE_BOOL:
+      return PyBool_FromLong(value.GetBoolValue());
+    case FieldDescriptor::CPPTYPE_STRING:
+      return ToStringObject(field_descriptor, value.GetStringValue());
+    case FieldDescriptor::CPPTYPE_ENUM:
+      return PyLong_FromLong(value.GetEnumValue());
+    default:
+      PyErr_Format(
+          PyExc_SystemError, "Couldn't convert type %d to value",
+          field_descriptor->cpp_type());
+      return nullptr;
+  }
+}
+
+// This is only used for ScalarMap, so we don't need to handle the
+// CPPTYPE_MESSAGE case.
+static bool PythonToMapValueRef(MapContainer* self, PyObject* obj,
+                                bool allow_unknown_enum_values,
+                                MapValueRef* value_ref) {
+  const FieldDescriptor* field_descriptor =
+      self->parent_field_descriptor->message_type()->map_value();
+  switch (field_descriptor->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32: {
+      GOOGLE_CHECK_GET_INT32(obj, value, false);
+      value_ref->SetInt32Value(value);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_INT64: {
+      GOOGLE_CHECK_GET_INT64(obj, value, false);
+      value_ref->SetInt64Value(value);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_UINT32: {
+      GOOGLE_CHECK_GET_UINT32(obj, value, false);
+      value_ref->SetUInt32Value(value);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_UINT64: {
+      GOOGLE_CHECK_GET_UINT64(obj, value, false);
+      value_ref->SetUInt64Value(value);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_FLOAT: {
+      GOOGLE_CHECK_GET_FLOAT(obj, value, false);
+      value_ref->SetFloatValue(value);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_DOUBLE: {
+      GOOGLE_CHECK_GET_DOUBLE(obj, value, false);
+      value_ref->SetDoubleValue(value);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_BOOL: {
+      GOOGLE_CHECK_GET_BOOL(obj, value, false);
+      value_ref->SetBoolValue(value);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_STRING: {
+      std::string str;
+      if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) {
+        return false;
+      }
+      value_ref->SetStringValue(str);
+      return true;
+    }
+    case FieldDescriptor::CPPTYPE_ENUM: {
+      GOOGLE_CHECK_GET_INT32(obj, value, false);
+      if (allow_unknown_enum_values) {
+        value_ref->SetEnumValue(value);
+        return true;
+      } else {
+        const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
+        const EnumValueDescriptor* enum_value =
+            enum_descriptor->FindValueByNumber(value);
+        if (enum_value != nullptr) {
+          value_ref->SetEnumValue(value);
+          return true;
+        } else {
+          PyErr_Format(PyExc_ValueError, "Unknown enum value: %d", value);
+          return false;
+        }
+      }
+      break;
+    }
+    default:
+      PyErr_Format(
+          PyExc_SystemError, "Setting value to a field of unknown type %d",
+          field_descriptor->cpp_type());
+      return false;
+  }
+}
+
+// Map methods common to ScalarMap and MessageMap //////////////////////////////
+
+static MapContainer* GetMap(PyObject* obj) {
+  return reinterpret_cast<MapContainer*>(obj);
+}
+
+Py_ssize_t MapReflectionFriend::Length(PyObject* _self) {
+  MapContainer* self = GetMap(_self);
+  const google::protobuf::Message* message = self->parent->message;
+  return message->GetReflection()->MapSize(*message,
+                                           self->parent_field_descriptor);
+}
+
+PyObject* Clear(PyObject* _self) {
+  MapContainer* self = GetMap(_self);
+  Message* message = self->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+
+  reflection->ClearField(message, self->parent_field_descriptor);
+
+  Py_RETURN_NONE;
+}
+
+PyObject* GetEntryClass(PyObject* _self) {
+  MapContainer* self = GetMap(_self);
+  CMessageClass* message_class = message_factory::GetMessageClass(
+      cmessage::GetFactoryForMessage(self->parent),
+      self->parent_field_descriptor->message_type());
+  Py_XINCREF(message_class);
+  return reinterpret_cast<PyObject*>(message_class);
+}
+
+PyObject* MapReflectionFriend::MergeFrom(PyObject* _self, PyObject* arg) {
+  MapContainer* self = GetMap(_self);
+  if (!PyObject_TypeCheck(arg, ScalarMapContainer_Type) &&
+      !PyObject_TypeCheck(arg, MessageMapContainer_Type)) {
+    PyErr_SetString(PyExc_AttributeError, "Not a map field");
+    return nullptr;
+  }
+  MapContainer* other_map = GetMap(arg);
+  Message* message = self->GetMutableMessage();
+  const Message* other_message = other_map->parent->message;
+  const Reflection* reflection = message->GetReflection();
+  const Reflection* other_reflection = other_message->GetReflection();
+  internal::MapFieldBase* field = reflection->MutableMapData(
+      message, self->parent_field_descriptor);
+  const internal::MapFieldBase* other_field = other_reflection->GetMapData(
+      *other_message, other_map->parent_field_descriptor);
+  field->MergeFrom(*other_field);
+  self->version++;
+  Py_RETURN_NONE;
+}
+
+PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
+  MapContainer* self = GetMap(_self);
+
+  const Message* message = self->parent->message;
+  const Reflection* reflection = message->GetReflection();
+  MapKey map_key;
+
+  if (!PythonToMapKey(self, key, &map_key)) {
+    return nullptr;
+  }
+
+  if (reflection->ContainsMapKey(*message, self->parent_field_descriptor,
+                                 map_key)) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+// ScalarMap ///////////////////////////////////////////////////////////////////
+
+MapContainer* NewScalarMapContainer(
+    CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return nullptr;
+  }
+
+  PyObject* obj(PyType_GenericAlloc(ScalarMapContainer_Type, 0));
+  if (obj == nullptr) {
+    PyErr_Format(PyExc_RuntimeError,
+                 "Could not allocate new container.");
+    return nullptr;
+  }
+
+  MapContainer* self = GetMap(obj);
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+  self->version = 0;
+
+  return self;
+}
+
+PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self,
+                                                PyObject* key) {
+  MapContainer* self = GetMap(_self);
+
+  Message* message = self->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+  MapKey map_key;
+  MapValueRef value;
+
+  if (!PythonToMapKey(self, key, &map_key)) {
+    return nullptr;
+  }
+
+  if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor,
+                                         map_key, &value)) {
+    self->version++;
+  }
+
+  return MapValueRefToPython(self, value);
+}
+
+int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key,
+                                          PyObject* v) {
+  MapContainer* self = GetMap(_self);
+
+  Message* message = self->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+  MapKey map_key;
+  MapValueRef value;
+
+  if (!PythonToMapKey(self, key, &map_key)) {
+    return -1;
+  }
+
+  if (v) {
+    // Set item to v.
+    if (reflection->InsertOrLookupMapValue(
+            message, self->parent_field_descriptor, map_key, &value)) {
+      self->version++;
+    }
+
+    if (!PythonToMapValueRef(self, v, reflection->SupportsUnknownEnumValues(),
+                             &value)) {
+      return -1;
+    }
+    return 0;
+  } else {
+    // Delete key from map.
+    if (reflection->DeleteMapValue(message, self->parent_field_descriptor,
+                                   map_key)) {
+      self->version++;
+      return 0;
+    } else {
+      PyErr_Format(PyExc_KeyError, "Key not present in map");
+      return -1;
+    }
+  }
+}
+
+static PyObject* ScalarMapGet(PyObject* self, PyObject* args,
+                              PyObject* kwargs) {
+  static const char* kwlist[] = {"key", "default", nullptr};
+  PyObject* key;
+  PyObject* default_value = nullptr;
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O",
+                                   const_cast<char**>(kwlist), &key,
+                                   &default_value)) {
+    return nullptr;
+  }
+
+  ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key));
+  if (is_present.get() == nullptr) {
+    return nullptr;
+  }
+
+  if (PyObject_IsTrue(is_present.get())) {
+    return MapReflectionFriend::ScalarMapGetItem(self, key);
+  } else {
+    if (default_value != nullptr) {
+      Py_INCREF(default_value);
+      return default_value;
+    } else {
+      Py_RETURN_NONE;
+    }
+  }
+}
+
+PyObject* MapReflectionFriend::ScalarMapToStr(PyObject* _self) {
+  ScopedPyObjectPtr dict(PyDict_New());
+  if (dict == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr key;
+  ScopedPyObjectPtr value;
+
+  MapContainer* self = GetMap(_self);
+  Message* message = self->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+  for (google::protobuf::MapIterator it = reflection->MapBegin(
+           message, self->parent_field_descriptor);
+       it != reflection->MapEnd(message, self->parent_field_descriptor);
+       ++it) {
+    key.reset(MapKeyToPython(self, it.GetKey()));
+    if (key == nullptr) {
+      return nullptr;
+    }
+    value.reset(MapValueRefToPython(self, it.GetValueRef()));
+    if (value == nullptr) {
+      return nullptr;
+    }
+    if (PyDict_SetItem(dict.get(), key.get(), value.get()) < 0) {
+      return nullptr;
+    }
+  }
+  return PyObject_Repr(dict.get());
+}
+
+static void ScalarMapDealloc(PyObject* _self) {
+  MapContainer* self = GetMap(_self);
+  self->RemoveFromParentCache();
+  PyTypeObject *type = Py_TYPE(_self);
+  type->tp_free(_self);
+  if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) {
+    // With Python3, the Map class is not static, and must be managed.
+    Py_DECREF(type);
+  }
+}
+
+static PyMethodDef ScalarMapMethods[] = {
+    {"__contains__", MapReflectionFriend::Contains, METH_O,
+     "Tests whether a key is a member of the map."},
+    {"clear", (PyCFunction)Clear, METH_NOARGS,
+     "Removes all elements from the map."},
+    {"get", (PyCFunction)ScalarMapGet, METH_VARARGS | METH_KEYWORDS,
+     "Gets the value for the given key if present, or otherwise a default"},
+    {"GetEntryClass", (PyCFunction)GetEntryClass, METH_NOARGS,
+     "Return the class used to build Entries of (key, value) pairs."},
+    {"MergeFrom", (PyCFunction)MapReflectionFriend::MergeFrom, METH_O,
+     "Merges a map into the current map."},
+    /*
+    { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+      "Makes a deep copy of the class." },
+    { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+      "Outputs picklable representation of the repeated field." },
+    */
+    {nullptr, nullptr},
+};
+
+PyTypeObject* ScalarMapContainer_Type;
+static PyType_Slot ScalarMapContainer_Type_slots[] = {
+    {Py_tp_dealloc, (void*)ScalarMapDealloc},
+    {Py_mp_length, (void*)MapReflectionFriend::Length},
+    {Py_mp_subscript, (void*)MapReflectionFriend::ScalarMapGetItem},
+    {Py_mp_ass_subscript, (void*)MapReflectionFriend::ScalarMapSetItem},
+    {Py_tp_methods, (void*)ScalarMapMethods},
+    {Py_tp_iter, (void*)MapReflectionFriend::GetIterator},
+    {Py_tp_repr, (void*)MapReflectionFriend::ScalarMapToStr},
+    {0, nullptr},
+};
+
+PyType_Spec ScalarMapContainer_Type_spec = {
+    FULL_MODULE_NAME ".ScalarMapContainer", sizeof(MapContainer), 0,
+    Py_TPFLAGS_DEFAULT, ScalarMapContainer_Type_slots};
+
+// MessageMap //////////////////////////////////////////////////////////////////
+
+static MessageMapContainer* GetMessageMap(PyObject* obj) {
+  return reinterpret_cast<MessageMapContainer*>(obj);
+}
+
+static PyObject* GetCMessage(MessageMapContainer* self, Message* message) {
+  // Get or create the CMessage object corresponding to this message.
+  return self->parent
+      ->BuildSubMessageFromPointer(self->parent_field_descriptor, message,
+                                   self->message_class)
+      ->AsPyObject();
+}
+
+MessageMapContainer* NewMessageMapContainer(
+    CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor,
+    CMessageClass* message_class) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return nullptr;
+  }
+
+  PyObject* obj = PyType_GenericAlloc(MessageMapContainer_Type, 0);
+  if (obj == nullptr) {
+    PyErr_SetString(PyExc_RuntimeError, "Could not allocate new container.");
+    return nullptr;
+  }
+
+  MessageMapContainer* self = GetMessageMap(obj);
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+  self->version = 0;
+
+  Py_INCREF(message_class);
+  self->message_class = message_class;
+
+  return self;
+}
+
+int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key,
+                                           PyObject* v) {
+  if (v) {
+    PyErr_Format(PyExc_ValueError,
+                 "Direct assignment of submessage not allowed");
+    return -1;
+  }
+
+  // Now we know that this is a delete, not a set.
+
+  MessageMapContainer* self = GetMessageMap(_self);
+  Message* message = self->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+  MapKey map_key;
+  MapValueRef value;
+
+  self->version++;
+
+  if (!PythonToMapKey(self, key, &map_key)) {
+    return -1;
+  }
+
+  // Delete key from map.
+  if (reflection->ContainsMapKey(*message, self->parent_field_descriptor,
+                                 map_key)) {
+    // Delete key from CMessage dict.
+    MapValueRef value;
+    reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor,
+                                       map_key, &value);
+    Message* sub_message = value.MutableMessageValue();
+    // If there is a living weak reference to an item, we "Release" it,
+    // otherwise we just discard the C++ value.
+    if (CMessage* released =
+            self->parent->MaybeReleaseSubMessage(sub_message)) {
+      Message* msg = released->message;
+      released->message = msg->New();
+      msg->GetReflection()->Swap(msg, released->message);
+    }
+
+    // Delete key from map.
+    reflection->DeleteMapValue(message, self->parent_field_descriptor,
+                               map_key);
+    return 0;
+  } else {
+    PyErr_Format(PyExc_KeyError, "Key not present in map");
+    return -1;
+  }
+}
+
+PyObject* MapReflectionFriend::MessageMapGetItem(PyObject* _self,
+                                                 PyObject* key) {
+  MessageMapContainer* self = GetMessageMap(_self);
+
+  Message* message = self->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+  MapKey map_key;
+  MapValueRef value;
+
+  if (!PythonToMapKey(self, key, &map_key)) {
+    return nullptr;
+  }
+
+  if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor,
+                                         map_key, &value)) {
+    self->version++;
+  }
+
+  return GetCMessage(self, value.MutableMessageValue());
+}
+
+PyObject* MapReflectionFriend::MessageMapToStr(PyObject* _self) {
+  ScopedPyObjectPtr dict(PyDict_New());
+  if (dict == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr key;
+  ScopedPyObjectPtr value;
+
+  MessageMapContainer* self = GetMessageMap(_self);
+  Message* message = self->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+  for (google::protobuf::MapIterator it = reflection->MapBegin(
+           message, self->parent_field_descriptor);
+       it != reflection->MapEnd(message, self->parent_field_descriptor);
+       ++it) {
+    key.reset(MapKeyToPython(self, it.GetKey()));
+    if (key == nullptr) {
+      return nullptr;
+    }
+    value.reset(GetCMessage(self, it.MutableValueRef()->MutableMessageValue()));
+    if (value == nullptr) {
+      return nullptr;
+    }
+    if (PyDict_SetItem(dict.get(), key.get(), value.get()) < 0) {
+      return nullptr;
+    }
+  }
+  return PyObject_Repr(dict.get());
+}
+
+PyObject* MessageMapGet(PyObject* self, PyObject* args, PyObject* kwargs) {
+  static const char* kwlist[] = {"key", "default", nullptr};
+  PyObject* key;
+  PyObject* default_value = nullptr;
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O",
+                                   const_cast<char**>(kwlist), &key,
+                                   &default_value)) {
+    return nullptr;
+  }
+
+  ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key));
+  if (is_present.get() == nullptr) {
+    return nullptr;
+  }
+
+  if (PyObject_IsTrue(is_present.get())) {
+    return MapReflectionFriend::MessageMapGetItem(self, key);
+  } else {
+    if (default_value != nullptr) {
+      Py_INCREF(default_value);
+      return default_value;
+    } else {
+      Py_RETURN_NONE;
+    }
+  }
+}
+
+static void MessageMapDealloc(PyObject* _self) {
+  MessageMapContainer* self = GetMessageMap(_self);
+  self->RemoveFromParentCache();
+  Py_DECREF(self->message_class);
+  PyTypeObject *type = Py_TYPE(_self);
+  type->tp_free(_self);
+  if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) {
+    // With Python3, the Map class is not static, and must be managed.
+    Py_DECREF(type);
+  }
+}
+
+static PyMethodDef MessageMapMethods[] = {
+    {"__contains__", (PyCFunction)MapReflectionFriend::Contains, METH_O,
+     "Tests whether the map contains this element."},
+    {"clear", (PyCFunction)Clear, METH_NOARGS,
+     "Removes all elements from the map."},
+    {"get", (PyCFunction)MessageMapGet, METH_VARARGS | METH_KEYWORDS,
+     "Gets the value for the given key if present, or otherwise a default"},
+    {"get_or_create", MapReflectionFriend::MessageMapGetItem, METH_O,
+     "Alias for getitem, useful to make explicit that the map is mutated."},
+    {"GetEntryClass", (PyCFunction)GetEntryClass, METH_NOARGS,
+     "Return the class used to build Entries of (key, value) pairs."},
+    {"MergeFrom", (PyCFunction)MapReflectionFriend::MergeFrom, METH_O,
+     "Merges a map into the current map."},
+    /*
+    { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+      "Makes a deep copy of the class." },
+    { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+      "Outputs picklable representation of the repeated field." },
+    */
+    {nullptr, nullptr},
+};
+
+PyTypeObject* MessageMapContainer_Type;
+static PyType_Slot MessageMapContainer_Type_slots[] = {
+    {Py_tp_dealloc, (void*)MessageMapDealloc},
+    {Py_mp_length, (void*)MapReflectionFriend::Length},
+    {Py_mp_subscript, (void*)MapReflectionFriend::MessageMapGetItem},
+    {Py_mp_ass_subscript, (void*)MapReflectionFriend::MessageMapSetItem},
+    {Py_tp_methods, (void*)MessageMapMethods},
+    {Py_tp_iter, (void*)MapReflectionFriend::GetIterator},
+    {Py_tp_repr, (void*)MapReflectionFriend::MessageMapToStr},
+    {0, nullptr}};
+
+PyType_Spec MessageMapContainer_Type_spec = {
+    FULL_MODULE_NAME ".MessageMapContainer", sizeof(MessageMapContainer), 0,
+    Py_TPFLAGS_DEFAULT, MessageMapContainer_Type_slots};
+
+// MapIterator /////////////////////////////////////////////////////////////////
+
+static MapIterator* GetIter(PyObject* obj) {
+  return reinterpret_cast<MapIterator*>(obj);
+}
+
+PyObject* MapReflectionFriend::GetIterator(PyObject *_self) {
+  MapContainer* self = GetMap(_self);
+
+  ScopedPyObjectPtr obj(PyType_GenericAlloc(&MapIterator_Type, 0));
+  if (obj == nullptr) {
+    return PyErr_Format(PyExc_KeyError, "Could not allocate iterator");
+  }
+
+  MapIterator* iter = GetIter(obj.get());
+
+  Py_INCREF(self);
+  iter->container = self;
+  iter->version = self->version;
+  Py_INCREF(self->parent);
+  iter->parent = self->parent;
+
+  if (MapReflectionFriend::Length(_self) > 0) {
+    Message* message = self->GetMutableMessage();
+    const Reflection* reflection = message->GetReflection();
+
+    iter->iter.reset(new ::google::protobuf::MapIterator(
+        reflection->MapBegin(message, self->parent_field_descriptor)));
+  }
+
+  return obj.release();
+}
+
+PyObject* MapReflectionFriend::IterNext(PyObject* _self) {
+  MapIterator* self = GetIter(_self);
+
+  // This won't catch mutations to the map performed by MergeFrom(); no easy way
+  // to address that.
+  if (self->version != self->container->version) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Map modified during iteration.");
+  }
+  if (self->parent != self->container->parent) {
+    return PyErr_Format(PyExc_RuntimeError,
+                        "Map cleared during iteration.");
+  }
+
+  if (self->iter.get() == nullptr) {
+    return nullptr;
+  }
+
+  Message* message = self->container->GetMutableMessage();
+  const Reflection* reflection = message->GetReflection();
+
+  if (*self->iter ==
+      reflection->MapEnd(message, self->container->parent_field_descriptor)) {
+    return nullptr;
+  }
+
+  PyObject* ret = MapKeyToPython(self->container, self->iter->GetKey());
+
+  ++(*self->iter);
+
+  return ret;
+}
+
+static void DeallocMapIterator(PyObject* _self) {
+  MapIterator* self = GetIter(_self);
+  self->iter.reset();
+  Py_CLEAR(self->container);
+  Py_CLEAR(self->parent);
+  Py_TYPE(_self)->tp_free(_self);
+}
+
+PyTypeObject MapIterator_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".MapIterator",       //  tp_name
+    sizeof(MapIterator),  //  tp_basicsize
+    0,                    //  tp_itemsize
+    DeallocMapIterator,   //  tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                        //  tp_getattr
+    nullptr,                        //  tp_setattr
+    nullptr,                        //  tp_compare
+    nullptr,                        //  tp_repr
+    nullptr,                        //  tp_as_number
+    nullptr,                        //  tp_as_sequence
+    nullptr,                        //  tp_as_mapping
+    nullptr,                        //  tp_hash
+    nullptr,                        //  tp_call
+    nullptr,                        //  tp_str
+    nullptr,                        //  tp_getattro
+    nullptr,                        //  tp_setattro
+    nullptr,                        //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,             //  tp_flags
+    "A scalar map iterator",        //  tp_doc
+    nullptr,                        //  tp_traverse
+    nullptr,                        //  tp_clear
+    nullptr,                        //  tp_richcompare
+    0,                              //  tp_weaklistoffset
+    PyObject_SelfIter,              //  tp_iter
+    MapReflectionFriend::IterNext,  //  tp_iternext
+    nullptr,                        //  tp_methods
+    nullptr,                        //  tp_members
+    nullptr,                        //  tp_getset
+    nullptr,                        //  tp_base
+    nullptr,                        //  tp_dict
+    nullptr,                        //  tp_descr_get
+    nullptr,                        //  tp_descr_set
+    0,                              //  tp_dictoffset
+    nullptr,                        //  tp_init
+};
+
+bool InitMapContainers() {
+  // ScalarMapContainer_Type derives from our MutableMapping type.
+  ScopedPyObjectPtr abc(PyImport_ImportModule("collections.abc"));
+  if (abc == nullptr) {
+    return false;
+  }
+
+  ScopedPyObjectPtr mutable_mapping(
+      PyObject_GetAttrString(abc.get(), "MutableMapping"));
+  if (mutable_mapping == nullptr) {
+    return false;
+  }
+
+  Py_INCREF(mutable_mapping.get());
+  ScopedPyObjectPtr bases(PyTuple_Pack(1, mutable_mapping.get()));
+  if (bases == nullptr) {
+    return false;
+  }
+
+  ScalarMapContainer_Type = reinterpret_cast<PyTypeObject*>(
+      PyType_FromSpecWithBases(&ScalarMapContainer_Type_spec, bases.get()));
+
+  if (PyType_Ready(&MapIterator_Type) < 0) {
+    return false;
+  }
+
+  MessageMapContainer_Type = reinterpret_cast<PyTypeObject*>(
+      PyType_FromSpecWithBases(&MessageMapContainer_Type_spec, bases.get()));
+  return true;
+}
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/map_container.h

@@ -0,0 +1,89 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <cstdint>
+
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/message.h"
+#include "google/protobuf/pyext/message.h"
+
+namespace google {
+namespace protobuf {
+
+class Message;
+
+namespace python {
+
+struct CMessageClass;
+
+// This struct is used directly for ScalarMap, and is the base class of
+// MessageMapContainer, which is used for MessageMap.
+struct MapContainer : public ContainerBase {
+  // Use to get a mutable message when necessary.
+  Message* GetMutableMessage();
+
+  // We bump this whenever we perform a mutation, to invalidate existing
+  // iterators.
+  uint64_t version;
+};
+
+struct MessageMapContainer : public MapContainer {
+  // The type used to create new child messages.
+  CMessageClass* message_class;
+};
+
+bool InitMapContainers();
+
+extern PyTypeObject* MessageMapContainer_Type;
+extern PyTypeObject* ScalarMapContainer_Type;
+extern PyTypeObject MapIterator_Type;  // Both map types use the same iterator.
+
+// Builds a MapContainer object, from a parent message and a
+// field descriptor.
+extern MapContainer* NewScalarMapContainer(
+    CMessage* parent, const FieldDescriptor* parent_field_descriptor);
+
+// Builds a MessageMap object, from a parent message and a
+// field descriptor.
+extern MessageMapContainer* NewMessageMapContainer(
+    CMessage* parent, const FieldDescriptor* parent_field_descriptor,
+    CMessageClass* message_class);
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__

python/google/protobuf/pyext/message.h

@@ -0,0 +1,377 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: anuraag@google.com (Anuraag Agrawal)
+// Author: tibell@google.com (Johan Tibell)
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include "google/protobuf/stubs/common.h"
+
+namespace google {
+namespace protobuf {
+
+class Message;
+class Reflection;
+class FieldDescriptor;
+class Descriptor;
+class DescriptorPool;
+class MessageFactory;
+
+namespace python {
+
+struct ExtensionDict;
+struct PyMessageFactory;
+struct CMessageClass;
+
+// Most of the complexity of the Message class comes from the "Release"
+// behavior:
+//
+// When a field is cleared, it is only detached from its message. Existing
+// references to submessages, to repeated container etc. won't see any change,
+// as if the data was effectively managed by these containers.
+//
+// ExtensionDicts and UnknownFields containers do NOT follow this rule. They
+// don't store any data, and always refer to their parent message.
+
+struct ContainerBase {
+  PyObject_HEAD;
+
+  // Strong reference to a parent message object. For a CMessage there are three
+  // cases:
+  // - For a top-level message, this pointer is NULL.
+  // - For a sub-message, this points to the parent message.
+  // - For a message managed externally, this is a owned reference to Py_None.
+  //
+  // For all other types: repeated containers, maps, it always point to a
+  // valid parent CMessage.
+  struct CMessage* parent;
+
+  // If this object belongs to a parent message, describes which field it comes
+  // from.
+  // The pointer is owned by the DescriptorPool (which is kept alive
+  // through the message's Python class)
+  const FieldDescriptor* parent_field_descriptor;
+
+  PyObject* AsPyObject() { return reinterpret_cast<PyObject*>(this); }
+
+  // The Three methods below are only used by Repeated containers, and Maps.
+
+  // This implementation works for all containers which have a parent.
+  PyObject* DeepCopy();
+  // Delete this container object from its parent. Does not work for messages.
+  void RemoveFromParentCache();
+};
+
+typedef struct CMessage : public ContainerBase {
+  // Pointer to the C++ Message object for this CMessage.
+  // - If this object has no parent, we own this pointer.
+  // - If this object has a parent message, the parent owns this pointer.
+  Message* message;
+
+  // Indicates this submessage is pointing to a default instance of a message.
+  // Submessages are always first created as read only messages and are then
+  // made writable, at which point this field is set to false.
+  bool read_only;
+
+  // A mapping indexed by field, containing weak references to contained objects
+  // which need to implement the "Release" mechanism:
+  // direct submessages, RepeatedCompositeContainer, RepeatedScalarContainer
+  // and MapContainer.
+  typedef std::unordered_map<const FieldDescriptor*, ContainerBase*>
+      CompositeFieldsMap;
+  CompositeFieldsMap* composite_fields;
+
+  // A mapping containing weak references to indirect child messages, accessed
+  // through containers: repeated messages, and values of message maps.
+  // This avoid the creation of similar maps in each of those containers.
+  typedef std::unordered_map<const Message*, CMessage*> SubMessagesMap;
+  SubMessagesMap* child_submessages;
+
+  // A reference to PyUnknownFields.
+  PyObject* unknown_field_set;
+
+  // Implements the "weakref" protocol for this object.
+  PyObject* weakreflist;
+
+  // Return a *borrowed* reference to the message class.
+  CMessageClass* GetMessageClass() {
+    return reinterpret_cast<CMessageClass*>(Py_TYPE(this));
+  }
+
+  // For container containing messages, return a Python object for the given
+  // pointer to a message.
+  CMessage* BuildSubMessageFromPointer(const FieldDescriptor* field_descriptor,
+                                       Message* sub_message,
+                                       CMessageClass* message_class);
+  CMessage* MaybeReleaseSubMessage(Message* sub_message);
+} CMessage;
+
+// The (meta) type of all Messages classes.
+// It allows us to cache some C++ pointers in the class object itself, they are
+// faster to extract than from the type's dictionary.
+
+struct CMessageClass {
+  // This is how CPython subclasses C structures: the base structure must be
+  // the first member of the object.
+  PyHeapTypeObject super;
+
+  // C++ descriptor of this message.
+  const Descriptor* message_descriptor;
+
+  // Owned reference, used to keep the pointer above alive.
+  // This reference must stay alive until all message pointers are destructed.
+  PyObject* py_message_descriptor;
+
+  // The Python MessageFactory used to create the class. It is needed to resolve
+  // fields descriptors, including extensions fields; its C++ MessageFactory is
+  // used to instantiate submessages.
+  // This reference must stay alive until all message pointers are destructed.
+  PyMessageFactory* py_message_factory;
+
+  PyObject* AsPyObject() {
+    return reinterpret_cast<PyObject*>(this);
+  }
+};
+
+extern PyTypeObject* CMessageClass_Type;
+extern PyTypeObject* CMessage_Type;
+
+namespace cmessage {
+
+// Internal function to create a new empty Message Python object, but with empty
+// pointers to the C++ objects.
+// The caller must fill self->message, self->owner and eventually self->parent.
+CMessage* NewEmptyMessage(CMessageClass* type);
+
+// Retrieves the C++ descriptor of a Python Extension descriptor.
+// On error, return NULL with an exception set.
+const FieldDescriptor* GetExtensionDescriptor(PyObject* extension);
+
+// Initializes a new CMessage instance for a submessage. Only called once per
+// submessage as the result is cached in composite_fields.
+//
+// Corresponds to reflection api method GetMessage.
+CMessage* InternalGetSubMessage(
+    CMessage* self, const FieldDescriptor* field_descriptor);
+
+// Deletes a range of items in a repeated field (following a
+// removal in a RepeatedCompositeContainer).
+//
+// Corresponds to reflection api method RemoveLast.
+int DeleteRepeatedField(CMessage* self,
+                        const FieldDescriptor* field_descriptor,
+                        PyObject* slice);
+
+// Sets the specified scalar value to the message.
+int InternalSetScalar(CMessage* self,
+                      const FieldDescriptor* field_descriptor,
+                      PyObject* value);
+
+// Sets the specified scalar value to the message.  Requires it is not a Oneof.
+int InternalSetNonOneofScalar(Message* message,
+                              const FieldDescriptor* field_descriptor,
+                              PyObject* arg);
+
+// Retrieves the specified scalar value from the message.
+//
+// Returns a new python reference.
+PyObject* InternalGetScalar(const Message* message,
+                            const FieldDescriptor* field_descriptor);
+
+bool SetCompositeField(CMessage* self, const FieldDescriptor* field,
+                       ContainerBase* value);
+
+bool SetSubmessage(CMessage* self, CMessage* submessage);
+
+// Clears the message, removing all contained data. Extension dictionary and
+// submessages are released first if there are remaining external references.
+//
+// Corresponds to message api method Clear.
+PyObject* Clear(CMessage* self);
+
+// Clears the data described by the given descriptor.
+// Returns -1 on error.
+//
+// Corresponds to reflection api method ClearField.
+int ClearFieldByDescriptor(CMessage* self, const FieldDescriptor* descriptor);
+
+// Checks if the message has the field described by the descriptor. Used for
+// extensions (which have no name).
+// Returns 1 if true, 0 if false, and -1 on error.
+//
+// Corresponds to reflection api method HasField
+int HasFieldByDescriptor(CMessage* self,
+                         const FieldDescriptor* field_descriptor);
+
+// Checks if the message has the named field.
+//
+// Corresponds to reflection api method HasField.
+PyObject* HasField(CMessage* self, PyObject* arg);
+
+// Initializes values of fields on a newly constructed message.
+// Note that positional arguments are disallowed: 'args' must be NULL or the
+// empty tuple.
+int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs);
+
+PyObject* MergeFrom(CMessage* self, PyObject* arg);
+
+// This method does not do anything beyond checking that no other extension
+// has been registered with the same field number on this class.
+PyObject* RegisterExtension(PyObject* cls, PyObject* extension_handle);
+
+// Get a field from a message.
+PyObject* GetFieldValue(CMessage* self,
+                        const FieldDescriptor* field_descriptor);
+// Sets the value of a scalar field in a message.
+// On error, return -1 with an extension set.
+int SetFieldValue(CMessage* self, const FieldDescriptor* field_descriptor,
+                  PyObject* value);
+
+PyObject* FindInitializationErrors(CMessage* self);
+
+int AssureWritable(CMessage* self);
+
+// Returns the message factory for the given message.
+// This is equivalent to message.MESSAGE_FACTORY
+//
+// The returned factory is suitable for finding fields and building submessages,
+// even in the case of extensions.
+// Returns a *borrowed* reference, and never fails because we pass a CMessage.
+PyMessageFactory* GetFactoryForMessage(CMessage* message);
+
+PyObject* SetAllowOversizeProtos(PyObject* m, PyObject* arg);
+
+}  // namespace cmessage
+
+
+/* Is 64bit */
+#define IS_64BIT (SIZEOF_LONG == 8)
+
+#define FIELD_IS_REPEATED(field_descriptor) \
+  ((field_descriptor)->label() == FieldDescriptor::LABEL_REPEATED)
+
+#define GOOGLE_CHECK_GET_INT32(arg, value, err)  \
+  int32_t value;                          \
+  if (!CheckAndGetInteger(arg, &value)) { \
+    return err;                           \
+  }
+
+#define GOOGLE_CHECK_GET_INT64(arg, value, err)  \
+  int64_t value;                          \
+  if (!CheckAndGetInteger(arg, &value)) { \
+    return err;                           \
+  }
+
+#define GOOGLE_CHECK_GET_UINT32(arg, value, err) \
+  uint32_t value;                         \
+  if (!CheckAndGetInteger(arg, &value)) { \
+    return err;                           \
+  }
+
+#define GOOGLE_CHECK_GET_UINT64(arg, value, err) \
+  uint64_t value;                         \
+  if (!CheckAndGetInteger(arg, &value)) { \
+    return err;                           \
+  }
+
+#define GOOGLE_CHECK_GET_FLOAT(arg, value, err) \
+  float value;                           \
+  if (!CheckAndGetFloat(arg, &value)) {  \
+    return err;                          \
+  }
+
+#define GOOGLE_CHECK_GET_DOUBLE(arg, value, err) \
+  double value;                           \
+  if (!CheckAndGetDouble(arg, &value)) {  \
+    return err;                           \
+  }
+
+#define GOOGLE_CHECK_GET_BOOL(arg, value, err) \
+  bool value;                           \
+  if (!CheckAndGetBool(arg, &value)) {  \
+    return err;                         \
+  }
+
+#define FULL_MODULE_NAME "google.protobuf.pyext._message"
+
+void FormatTypeError(PyObject* arg, const char* expected_types);
+template<class T>
+bool CheckAndGetInteger(PyObject* arg, T* value);
+bool CheckAndGetDouble(PyObject* arg, double* value);
+bool CheckAndGetFloat(PyObject* arg, float* value);
+bool CheckAndGetBool(PyObject* arg, bool* value);
+PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor);
+bool CheckAndSetString(
+    PyObject* arg, Message* message,
+    const FieldDescriptor* descriptor,
+    const Reflection* reflection,
+    bool append,
+    int index);
+PyObject* ToStringObject(const FieldDescriptor* descriptor,
+                         const std::string& value);
+
+// Check if the passed field descriptor belongs to the given message.
+// If not, return false and set a Python exception (a KeyError)
+bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor,
+                                const Message* message);
+
+extern PyObject* PickleError_class;
+
+PyObject* PyMessage_New(const Descriptor* descriptor,
+                        PyObject* py_message_factory);
+const Message* PyMessage_GetMessagePointer(PyObject* msg);
+Message* PyMessage_GetMutableMessagePointer(PyObject* msg);
+PyObject* PyMessage_NewMessageOwnedExternally(Message* message,
+                                              PyObject* py_message_factory);
+
+bool InitProto2MessageModule(PyObject *m);
+
+// These are referenced by repeated_scalar_container, and must
+// be explicitly instantiated.
+extern template bool CheckAndGetInteger<int32>(PyObject*, int32*);
+extern template bool CheckAndGetInteger<int64>(PyObject*, int64*);
+extern template bool CheckAndGetInteger<uint32>(PyObject*, uint32*);
+extern template bool CheckAndGetInteger<uint64>(PyObject*, uint64*);
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__

python/google/protobuf/pyext/message_factory.cc

@@ -0,0 +1,307 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <unordered_map>
+#include <utility>
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "google/protobuf/dynamic_message.h"
+#include "google/protobuf/pyext/descriptor.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/message_factory.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+
+#define PyString_AsStringAndSize(ob, charpp, sizep)              \
+  (PyUnicode_Check(ob)                                           \
+       ? ((*(charpp) = const_cast<char*>(                        \
+               PyUnicode_AsUTF8AndSize(ob, (sizep)))) == nullptr \
+              ? -1                                               \
+              : 0)                                               \
+       : PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+namespace message_factory {
+
+PyMessageFactory* NewMessageFactory(PyTypeObject* type, PyDescriptorPool* pool) {
+  PyMessageFactory* factory = reinterpret_cast<PyMessageFactory*>(
+      PyType_GenericAlloc(type, 0));
+  if (factory == nullptr) {
+    return nullptr;
+  }
+
+  DynamicMessageFactory* message_factory = new DynamicMessageFactory();
+  // This option might be the default some day.
+  message_factory->SetDelegateToGeneratedFactory(true);
+  factory->message_factory = message_factory;
+
+  factory->pool = pool;
+  Py_INCREF(pool);
+
+  factory->classes_by_descriptor = new PyMessageFactory::ClassesByMessageMap();
+
+  return factory;
+}
+
+PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
+  static const char* kwlist[] = {"pool", nullptr};
+  PyObject* pool = nullptr;
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O",
+                                   const_cast<char**>(kwlist), &pool)) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr owned_pool;
+  if (pool == nullptr || pool == Py_None) {
+    owned_pool.reset(PyObject_CallFunction(
+        reinterpret_cast<PyObject*>(&PyDescriptorPool_Type), nullptr));
+    if (owned_pool == nullptr) {
+      return nullptr;
+    }
+    pool = owned_pool.get();
+  } else {
+    if (!PyObject_TypeCheck(pool, &PyDescriptorPool_Type)) {
+      PyErr_Format(PyExc_TypeError, "Expected a DescriptorPool, got %s",
+                   pool->ob_type->tp_name);
+      return nullptr;
+    }
+  }
+
+  return reinterpret_cast<PyObject*>(
+      NewMessageFactory(type, reinterpret_cast<PyDescriptorPool*>(pool)));
+}
+
+static void Dealloc(PyObject* pself) {
+  PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
+
+  typedef PyMessageFactory::ClassesByMessageMap::iterator iterator;
+  for (iterator it = self->classes_by_descriptor->begin();
+       it != self->classes_by_descriptor->end(); ++it) {
+    Py_CLEAR(it->second);
+  }
+  delete self->classes_by_descriptor;
+  delete self->message_factory;
+  Py_CLEAR(self->pool);
+  Py_TYPE(self)->tp_free(pself);
+}
+
+static int GcTraverse(PyObject* pself, visitproc visit, void* arg) {
+  PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
+  Py_VISIT(self->pool);
+  for (const auto& desc_and_class : *self->classes_by_descriptor) {
+    Py_VISIT(desc_and_class.second);
+  }
+  return 0;
+}
+
+static int GcClear(PyObject* pself) {
+  PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
+  // Here it's important to not clear self->pool, so that the C++ DescriptorPool
+  // is still alive when self->message_factory is destructed.
+  for (auto& desc_and_class : *self->classes_by_descriptor) {
+    Py_CLEAR(desc_and_class.second);
+  }
+
+  return 0;
+}
+
+// Add a message class to our database.
+int RegisterMessageClass(PyMessageFactory* self,
+                         const Descriptor* message_descriptor,
+                         CMessageClass* message_class) {
+  Py_INCREF(message_class);
+  typedef PyMessageFactory::ClassesByMessageMap::iterator iterator;
+  std::pair<iterator, bool> ret = self->classes_by_descriptor->insert(
+      std::make_pair(message_descriptor, message_class));
+  if (!ret.second) {
+    // Update case: DECREF the previous value.
+    Py_DECREF(ret.first->second);
+    ret.first->second = message_class;
+  }
+  return 0;
+}
+
+CMessageClass* GetOrCreateMessageClass(PyMessageFactory* self,
+                                       const Descriptor* descriptor) {
+  // This is the same implementation as MessageFactory.GetPrototype().
+
+  // Do not create a MessageClass that already exists.
+  std::unordered_map<const Descriptor*, CMessageClass*>::iterator it =
+      self->classes_by_descriptor->find(descriptor);
+  if (it != self->classes_by_descriptor->end()) {
+    Py_INCREF(it->second);
+    return it->second;
+  }
+  ScopedPyObjectPtr py_descriptor(
+      PyMessageDescriptor_FromDescriptor(descriptor));
+  if (py_descriptor == nullptr) {
+    return nullptr;
+  }
+  // Create a new message class.
+  ScopedPyObjectPtr args(Py_BuildValue(
+      "s(){sOsOsO}", descriptor->name().c_str(),
+      "DESCRIPTOR", py_descriptor.get(),
+      "__module__", Py_None,
+      "message_factory", self));
+  if (args == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr message_class(PyObject_CallObject(
+      reinterpret_cast<PyObject*>(CMessageClass_Type), args.get()));
+  if (message_class == nullptr) {
+    return nullptr;
+  }
+  // Create messages class for the messages used by the fields, and registers
+  // all extensions for these messages during the recursion.
+  for (int field_idx = 0; field_idx < descriptor->field_count(); field_idx++) {
+    const Descriptor* sub_descriptor =
+        descriptor->field(field_idx)->message_type();
+    // It is null if the field type is not a message.
+    if (sub_descriptor != nullptr) {
+      CMessageClass* result = GetOrCreateMessageClass(self, sub_descriptor);
+      if (result == nullptr) {
+        return nullptr;
+      }
+      Py_DECREF(result);
+    }
+  }
+
+  // Register extensions defined in this message.
+  for (int ext_idx = 0 ; ext_idx < descriptor->extension_count() ; ext_idx++) {
+    const FieldDescriptor* extension = descriptor->extension(ext_idx);
+    ScopedPyObjectPtr py_extended_class(
+        GetOrCreateMessageClass(self, extension->containing_type())
+            ->AsPyObject());
+    if (py_extended_class == nullptr) {
+      return nullptr;
+    }
+    ScopedPyObjectPtr py_extension(PyFieldDescriptor_FromDescriptor(extension));
+    if (py_extension == nullptr) {
+      return nullptr;
+    }
+    ScopedPyObjectPtr result(cmessage::RegisterExtension(
+        py_extended_class.get(), py_extension.get()));
+    if (result == nullptr) {
+      return nullptr;
+    }
+  }
+  return reinterpret_cast<CMessageClass*>(message_class.release());
+}
+
+// Retrieve the message class added to our database.
+CMessageClass* GetMessageClass(PyMessageFactory* self,
+                               const Descriptor* message_descriptor) {
+  typedef PyMessageFactory::ClassesByMessageMap::iterator iterator;
+  iterator ret = self->classes_by_descriptor->find(message_descriptor);
+  if (ret == self->classes_by_descriptor->end()) {
+    PyErr_Format(PyExc_TypeError, "No message class registered for '%s'",
+                 message_descriptor->full_name().c_str());
+    return nullptr;
+  } else {
+    return ret->second;
+  }
+}
+
+static PyMethodDef Methods[] = {
+    {nullptr},
+};
+
+static PyObject* GetPool(PyMessageFactory* self, void* closure) {
+  Py_INCREF(self->pool);
+  return reinterpret_cast<PyObject*>(self->pool);
+}
+
+static PyGetSetDef Getters[] = {
+    {"pool", (getter)GetPool, nullptr, "DescriptorPool"},
+    {nullptr},
+};
+
+}  // namespace message_factory
+
+PyTypeObject PyMessageFactory_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".MessageFactory",         // tp_name
+    sizeof(PyMessageFactory),  // tp_basicsize
+    0,                         // tp_itemsize
+    message_factory::Dealloc,  // tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,  // tp_getattr
+    nullptr,  // tp_setattr
+    nullptr,  // tp_compare
+    nullptr,  // tp_repr
+    nullptr,  // tp_as_number
+    nullptr,  // tp_as_sequence
+    nullptr,  // tp_as_mapping
+    nullptr,  // tp_hash
+    nullptr,  // tp_call
+    nullptr,  // tp_str
+    nullptr,  // tp_getattro
+    nullptr,  // tp_setattro
+    nullptr,  // tp_as_buffer
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,  // tp_flags
+    "A static Message Factory",                                     // tp_doc
+    message_factory::GcTraverse,  // tp_traverse
+    message_factory::GcClear,     // tp_clear
+    nullptr,                      // tp_richcompare
+    0,                            // tp_weaklistoffset
+    nullptr,                      // tp_iter
+    nullptr,                      // tp_iternext
+    message_factory::Methods,     // tp_methods
+    nullptr,                      // tp_members
+    message_factory::Getters,     // tp_getset
+    nullptr,                      // tp_base
+    nullptr,                      // tp_dict
+    nullptr,                      // tp_descr_get
+    nullptr,                      // tp_descr_set
+    0,                            // tp_dictoffset
+    nullptr,                      // tp_init
+    nullptr,                      // tp_alloc
+    message_factory::New,         // tp_new
+    PyObject_GC_Del,              // tp_free
+};
+
+bool InitMessageFactory() {
+  if (PyType_Ready(&PyMessageFactory_Type) < 0) {
+    return false;
+  }
+
+  return true;
+}
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/message_factory.h

@@ -0,0 +1,104 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_FACTORY_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_FACTORY_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <unordered_map>
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/pyext/descriptor_pool.h"
+
+namespace google {
+namespace protobuf {
+class MessageFactory;
+
+namespace python {
+
+// The (meta) type of all Messages classes.
+struct CMessageClass;
+
+struct PyMessageFactory {
+  PyObject_HEAD
+
+  // DynamicMessageFactory used to create C++ instances of messages.
+  // This object cache the descriptors that were used, so the DescriptorPool
+  // needs to get rid of it before it can delete itself.
+  //
+  // Note: A C++ MessageFactory is different from the PyMessageFactory.
+  // The C++ one creates messages, when the Python one creates classes.
+  MessageFactory* message_factory;
+
+  // Owned reference to a Python DescriptorPool.
+  // This reference must stay until the message_factory is destructed.
+  PyDescriptorPool* pool;
+
+  // Make our own mapping to retrieve Python classes from C++ descriptors.
+  //
+  // Descriptor pointers stored here are owned by the DescriptorPool above.
+  // Python references to classes are owned by this PyDescriptorPool.
+  typedef std::unordered_map<const Descriptor*, CMessageClass*>
+      ClassesByMessageMap;
+  ClassesByMessageMap* classes_by_descriptor;
+};
+
+extern PyTypeObject PyMessageFactory_Type;
+
+namespace message_factory {
+
+// Creates a new MessageFactory instance.
+PyMessageFactory* NewMessageFactory(PyTypeObject* type, PyDescriptorPool* pool);
+
+// Registers a new Python class for the given message descriptor.
+// On error, returns -1 with a Python exception set.
+int RegisterMessageClass(PyMessageFactory* self,
+                         const Descriptor* message_descriptor,
+                         CMessageClass* message_class);
+// Retrieves the Python class registered with the given message descriptor, or
+// fail with a TypeError. Returns a *borrowed* reference.
+CMessageClass* GetMessageClass(PyMessageFactory* self,
+                               const Descriptor* message_descriptor);
+// Retrieves the Python class registered with the given message descriptor.
+// The class is created if not done yet. Returns a *new* reference.
+CMessageClass* GetOrCreateMessageClass(PyMessageFactory* self,
+                                       const Descriptor* message_descriptor);
+}  // namespace message_factory
+
+// Initialize objects used by this module.
+// On error, returns false with a Python exception set.
+bool InitMessageFactory();
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_FACTORY_H__

python/google/protobuf/pyext/message_module.cc

@@ -0,0 +1,134 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "google/protobuf/message_lite.h"
+#include "google/protobuf/pyext/descriptor.h"
+#include "google/protobuf/pyext/descriptor_pool.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/message_factory.h"
+#include "google/protobuf/proto_api.h"
+
+namespace {
+
+// C++ API.  Clients get at this via proto_api.h
+struct ApiImplementation : google::protobuf::python::PyProto_API {
+  const google::protobuf::Message* GetMessagePointer(PyObject* msg) const override {
+    return google::protobuf::python::PyMessage_GetMessagePointer(msg);
+  }
+  google::protobuf::Message* GetMutableMessagePointer(PyObject* msg) const override {
+    return google::protobuf::python::PyMessage_GetMutableMessagePointer(msg);
+  }
+  const google::protobuf::Descriptor* MessageDescriptor_AsDescriptor(
+      PyObject* desc) const override {
+    return google::protobuf::python::PyMessageDescriptor_AsDescriptor(desc);
+  }
+  const google::protobuf::EnumDescriptor* EnumDescriptor_AsDescriptor(
+      PyObject* enum_desc) const override {
+    return google::protobuf::python::PyEnumDescriptor_AsDescriptor(enum_desc);
+  }
+
+  const google::protobuf::DescriptorPool* GetDefaultDescriptorPool() const override {
+    return google::protobuf::python::GetDefaultDescriptorPool()->pool;
+  }
+
+  google::protobuf::MessageFactory* GetDefaultMessageFactory() const override {
+    return google::protobuf::python::GetDefaultDescriptorPool()
+        ->py_message_factory->message_factory;
+  }
+  PyObject* NewMessage(const google::protobuf::Descriptor* descriptor,
+                       PyObject* py_message_factory) const override {
+    return google::protobuf::python::PyMessage_New(descriptor, py_message_factory);
+  }
+  PyObject* NewMessageOwnedExternally(
+      google::protobuf::Message* msg, PyObject* py_message_factory) const override {
+    return google::protobuf::python::PyMessage_NewMessageOwnedExternally(
+        msg, py_message_factory);
+  }
+  PyObject* DescriptorPool_FromPool(
+      const google::protobuf::DescriptorPool* pool) const override {
+    return google::protobuf::python::PyDescriptorPool_FromPool(pool);
+  }
+};
+
+}  // namespace
+
+static const char module_docstring[] =
+    "python-proto2 is a module that can be used to enhance proto2 Python API\n"
+    "performance.\n"
+    "\n"
+    "It provides access to the protocol buffers C++ reflection API that\n"
+    "implements the basic protocol buffer functions.";
+
+static PyMethodDef ModuleMethods[] = {
+    {"SetAllowOversizeProtos",
+     (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos, METH_O,
+     "Enable/disable oversize proto parsing."},
+    // DO NOT USE: For migration and testing only.
+    {nullptr, nullptr}};
+
+static struct PyModuleDef _module = {PyModuleDef_HEAD_INIT,
+                                     "_message",
+                                     module_docstring,
+                                     -1,
+                                     ModuleMethods, /* m_methods */
+                                     nullptr,
+                                     nullptr,
+                                     nullptr,
+                                     nullptr};
+
+PyMODINIT_FUNC PyInit__message() {
+  PyObject* m;
+  m = PyModule_Create(&_module);
+  if (m == nullptr) {
+    return nullptr;
+  }
+
+  if (!google::protobuf::python::InitProto2MessageModule(m)) {
+    Py_DECREF(m);
+    return nullptr;
+  }
+
+  // Adds the C++ API
+  if (PyObject* api = PyCapsule_New(
+          new ApiImplementation(), google::protobuf::python::PyProtoAPICapsuleName(),
+          [](PyObject* o) {
+            delete (ApiImplementation*)PyCapsule_GetPointer(
+                o, google::protobuf::python::PyProtoAPICapsuleName());
+          })) {
+    PyModule_AddObject(m, "proto_API", api);
+  } else {
+    return nullptr;
+  }
+
+  return m;
+}

python/google/protobuf/pyext/proto2_api_test.proto

@@ -0,0 +1,40 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+package google.protobuf.python.internal;
+
+import "google/protobuf/internal/cpp/proto1_api_test.proto";
+
+message TestNestedProto1APIMessage {
+  optional int32 a = 1;
+  optional TestMessage.NestedMessage b = 2;
+}

python/google/protobuf/pyext/python.proto

@@ -0,0 +1,68 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: tibell@google.com (Johan Tibell)
+//
+// These message definitions are used to exercises known corner cases
+// in the C++ implementation of the Python API.
+
+syntax = "proto2";
+
+package google.protobuf.python.internal;
+
+// Protos optimized for SPEED use a strict superset of the generated code
+// of equivalent ones optimized for CODE_SIZE, so we should optimize all our
+// tests for speed unless explicitly testing code size optimization.
+option optimize_for = SPEED;
+
+message TestAllTypes {
+  message NestedMessage {
+    optional int32 bb = 1;
+    optional ForeignMessage cc = 2;
+  }
+
+  repeated NestedMessage repeated_nested_message = 1;
+  optional NestedMessage optional_nested_message = 2;
+  optional int32 optional_int32 = 3;
+}
+
+message ForeignMessage {
+  optional int32 c = 1;
+  repeated int32 d = 2;
+}
+
+message TestAllExtensions {  // extension begin
+  extensions 1 to max;
+}  // extension end
+
+extend TestAllExtensions {  // extension begin
+  optional TestAllTypes.NestedMessage optional_nested_message_extension = 1;
+  repeated TestAllTypes.NestedMessage repeated_nested_message_extension = 2;
+}  // extension end

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -0,0 +1,590 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: anuraag@google.com (Anuraag Agrawal)
+// Author: tibell@google.com (Johan Tibell)
+
+#include "google/protobuf/pyext/repeated_composite_container.h"
+
+#include <memory>
+
+#include "google/protobuf/stubs/logging.h"
+#include "google/protobuf/stubs/common.h"
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/dynamic_message.h"
+#include "google/protobuf/message.h"
+#include "google/protobuf/reflection.h"
+#include "google/protobuf/pyext/descriptor.h"
+#include "google/protobuf/pyext/descriptor_pool.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/message_factory.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+namespace repeated_composite_container {
+
+// ---------------------------------------------------------------------
+// len()
+
+static Py_ssize_t Length(PyObject* pself) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
+  Message* message = self->parent->message;
+  return message->GetReflection()->FieldSize(*message,
+                                             self->parent_field_descriptor);
+}
+
+// ---------------------------------------------------------------------
+// add()
+
+PyObject* Add(RepeatedCompositeContainer* self, PyObject* args,
+              PyObject* kwargs) {
+  if (cmessage::AssureWritable(self->parent) == -1) return nullptr;
+  Message* message = self->parent->message;
+
+  Message* sub_message = message->GetReflection()->AddMessage(
+      message, self->parent_field_descriptor,
+      self->child_message_class->py_message_factory->message_factory);
+  CMessage* cmsg = self->parent->BuildSubMessageFromPointer(
+      self->parent_field_descriptor, sub_message, self->child_message_class);
+
+  if (cmessage::InitAttributes(cmsg, args, kwargs) < 0) {
+    message->GetReflection()->RemoveLast(message,
+                                         self->parent_field_descriptor);
+    Py_DECREF(cmsg);
+    return nullptr;
+  }
+
+  return cmsg->AsPyObject();
+}
+
+static PyObject* AddMethod(PyObject* self, PyObject* args, PyObject* kwargs) {
+  return Add(reinterpret_cast<RepeatedCompositeContainer*>(self), args, kwargs);
+}
+
+// ---------------------------------------------------------------------
+// append()
+
+static PyObject* AddMessage(RepeatedCompositeContainer* self, PyObject* value) {
+  cmessage::AssureWritable(self->parent);
+  PyObject* py_cmsg;
+  Message* message = self->parent->message;
+  const Reflection* reflection = message->GetReflection();
+  py_cmsg = Add(self, nullptr, nullptr);
+  if (py_cmsg == nullptr) return nullptr;
+  CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
+  if (ScopedPyObjectPtr(cmessage::MergeFrom(cmsg, value)) == nullptr) {
+    reflection->RemoveLast(message, self->parent_field_descriptor);
+    Py_DECREF(cmsg);
+    return nullptr;
+  }
+  return py_cmsg;
+}
+
+static PyObject* AppendMethod(PyObject* pself, PyObject* value) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+  ScopedPyObjectPtr py_cmsg(AddMessage(self, value));
+  if (py_cmsg == nullptr) {
+    return nullptr;
+  }
+
+  Py_RETURN_NONE;
+}
+
+// ---------------------------------------------------------------------
+// insert()
+static PyObject* Insert(PyObject* pself, PyObject* args) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
+  Py_ssize_t index;
+  PyObject* value;
+  if (!PyArg_ParseTuple(args, "nO", &index, &value)) {
+    return nullptr;
+  }
+
+  ScopedPyObjectPtr py_cmsg(AddMessage(self, value));
+  if (py_cmsg == nullptr) {
+    return nullptr;
+  }
+
+  // Swap the element to right position.
+  Message* message = self->parent->message;
+  const Reflection* reflection = message->GetReflection();
+  const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
+  Py_ssize_t length = reflection->FieldSize(*message, field_descriptor) - 1;
+  Py_ssize_t end_index = index;
+  if (end_index < 0) end_index += length;
+  if (end_index < 0) end_index = 0;
+  for (Py_ssize_t i = length; i > end_index; i--) {
+    reflection->SwapElements(message, field_descriptor, i, i - 1);
+  }
+
+  Py_RETURN_NONE;
+}
+
+// ---------------------------------------------------------------------
+// extend()
+
+PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value) {
+  cmessage::AssureWritable(self->parent);
+  ScopedPyObjectPtr iter(PyObject_GetIter(value));
+  if (iter == nullptr) {
+    PyErr_SetString(PyExc_TypeError, "Value must be iterable");
+    return nullptr;
+  }
+  ScopedPyObjectPtr next;
+  while ((next.reset(PyIter_Next(iter.get()))) != nullptr) {
+    if (!PyObject_TypeCheck(next.get(), CMessage_Type)) {
+      PyErr_SetString(PyExc_TypeError, "Not a cmessage");
+      return nullptr;
+    }
+    ScopedPyObjectPtr new_message(Add(self, nullptr, nullptr));
+    if (new_message == nullptr) {
+      return nullptr;
+    }
+    CMessage* new_cmessage = reinterpret_cast<CMessage*>(new_message.get());
+    if (ScopedPyObjectPtr(cmessage::MergeFrom(new_cmessage, next.get())) ==
+        nullptr) {
+      return nullptr;
+    }
+  }
+  if (PyErr_Occurred()) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* ExtendMethod(PyObject* self, PyObject* value) {
+  return Extend(reinterpret_cast<RepeatedCompositeContainer*>(self), value);
+}
+
+PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other) {
+  return Extend(self, other);
+}
+
+static PyObject* MergeFromMethod(PyObject* self, PyObject* other) {
+  return MergeFrom(reinterpret_cast<RepeatedCompositeContainer*>(self), other);
+}
+
+// This function does not check the bounds.
+static PyObject* GetItem(RepeatedCompositeContainer* self, Py_ssize_t index,
+                         Py_ssize_t length = -1) {
+  if (length == -1) {
+    Message* message = self->parent->message;
+    const Reflection* reflection = message->GetReflection();
+    length = reflection->FieldSize(*message, self->parent_field_descriptor);
+  }
+  if (index < 0 || index >= length) {
+    PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index);
+    return nullptr;
+  }
+  Message* message = self->parent->message;
+  Message* sub_message = message->GetReflection()->MutableRepeatedMessage(
+      message, self->parent_field_descriptor, index);
+  return self->parent
+      ->BuildSubMessageFromPointer(self->parent_field_descriptor, sub_message,
+                                   self->child_message_class)
+      ->AsPyObject();
+}
+
+PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* item) {
+  Message* message = self->parent->message;
+  const Reflection* reflection = message->GetReflection();
+  Py_ssize_t length =
+      reflection->FieldSize(*message, self->parent_field_descriptor);
+
+  if (PyIndex_Check(item)) {
+    Py_ssize_t index;
+    index = PyNumber_AsSsize_t(item, PyExc_IndexError);
+    if (index == -1 && PyErr_Occurred()) return nullptr;
+    if (index < 0) index += length;
+    return GetItem(self, index, length);
+  } else if (PySlice_Check(item)) {
+    Py_ssize_t from, to, step, slicelength, cur, i;
+    PyObject* result;
+
+    if (PySlice_GetIndicesEx(item, length, &from, &to, &step, &slicelength) ==
+        -1) {
+      return nullptr;
+    }
+
+    if (slicelength <= 0) {
+      return PyList_New(0);
+    } else {
+      result = PyList_New(slicelength);
+      if (!result) return nullptr;
+
+      for (cur = from, i = 0; i < slicelength; cur += step, i++) {
+        PyList_SET_ITEM(result, i, GetItem(self, cur, length));
+      }
+
+      return result;
+    }
+  } else {
+    PyErr_Format(PyExc_TypeError, "indices must be integers, not %.200s",
+                 item->ob_type->tp_name);
+    return nullptr;
+  }
+}
+
+static PyObject* SubscriptMethod(PyObject* self, PyObject* slice) {
+  return Subscript(reinterpret_cast<RepeatedCompositeContainer*>(self), slice);
+}
+
+int AssignSubscript(RepeatedCompositeContainer* self, PyObject* slice,
+                    PyObject* value) {
+  if (value != nullptr) {
+    PyErr_SetString(PyExc_TypeError, "does not support assignment");
+    return -1;
+  }
+
+  return cmessage::DeleteRepeatedField(self->parent,
+                                       self->parent_field_descriptor, slice);
+}
+
+static int AssignSubscriptMethod(PyObject* self, PyObject* slice,
+                                 PyObject* value) {
+  return AssignSubscript(reinterpret_cast<RepeatedCompositeContainer*>(self),
+                         slice, value);
+}
+
+static PyObject* Remove(PyObject* pself, PyObject* value) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+  Py_ssize_t len = Length(reinterpret_cast<PyObject*>(self));
+
+  for (Py_ssize_t i = 0; i < len; i++) {
+    ScopedPyObjectPtr item(GetItem(self, i, len));
+    if (item == nullptr) {
+      return nullptr;
+    }
+    int result = PyObject_RichCompareBool(item.get(), value, Py_EQ);
+    if (result < 0) {
+      return nullptr;
+    }
+    if (result) {
+      ScopedPyObjectPtr py_index(PyLong_FromSsize_t(i));
+      if (AssignSubscript(self, py_index.get(), nullptr) < 0) {
+        return nullptr;
+      }
+      Py_RETURN_NONE;
+    }
+  }
+  PyErr_SetString(PyExc_ValueError, "Item to delete not in list");
+  return nullptr;
+}
+
+static PyObject* RichCompare(PyObject* pself, PyObject* other, int opid) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
+  if (!PyObject_TypeCheck(other, &RepeatedCompositeContainer_Type)) {
+    PyErr_SetString(PyExc_TypeError,
+                    "Can only compare repeated composite fields "
+                    "against other repeated composite fields.");
+    return nullptr;
+  }
+  if (opid == Py_EQ || opid == Py_NE) {
+    // TODO(anuraag): Don't make new lists just for this...
+    ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+    if (full_slice == nullptr) {
+      return nullptr;
+    }
+    ScopedPyObjectPtr list(Subscript(self, full_slice.get()));
+    if (list == nullptr) {
+      return nullptr;
+    }
+    ScopedPyObjectPtr other_list(
+        Subscript(reinterpret_cast<RepeatedCompositeContainer*>(other),
+                  full_slice.get()));
+    if (other_list == nullptr) {
+      return nullptr;
+    }
+    return PyObject_RichCompare(list.get(), other_list.get(), opid);
+  } else {
+    Py_INCREF(Py_NotImplemented);
+    return Py_NotImplemented;
+  }
+}
+
+static PyObject* ToStr(PyObject* pself) {
+  ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+  if (full_slice == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr list(Subscript(
+      reinterpret_cast<RepeatedCompositeContainer*>(pself), full_slice.get()));
+  if (list == nullptr) {
+    return nullptr;
+  }
+  return PyObject_Repr(list.get());
+}
+
+// ---------------------------------------------------------------------
+// sort()
+
+static void ReorderAttached(RepeatedCompositeContainer* self,
+                            PyObject* child_list) {
+  Message* message = self->parent->message;
+  const Reflection* reflection = message->GetReflection();
+  const FieldDescriptor* descriptor = self->parent_field_descriptor;
+  const Py_ssize_t length = Length(reinterpret_cast<PyObject*>(self));
+
+  // We need to rearrange things to match python's sort order.
+  for (Py_ssize_t i = 0; i < length; ++i) {
+    reflection->UnsafeArenaReleaseLast(message, descriptor);
+  }
+  for (Py_ssize_t i = 0; i < length; ++i) {
+    Message* child_message =
+        reinterpret_cast<CMessage*>(PyList_GET_ITEM(child_list, i))->message;
+    reflection->UnsafeArenaAddAllocatedMessage(message, descriptor,
+                                               child_message);
+  }
+}
+
+// Returns 0 if successful; returns -1 and sets an exception if
+// unsuccessful.
+static int SortPythonMessages(RepeatedCompositeContainer* self, PyObject* args,
+                              PyObject* kwds) {
+  ScopedPyObjectPtr child_list(
+      PySequence_List(reinterpret_cast<PyObject*>(self)));
+  if (child_list == nullptr) {
+    return -1;
+  }
+  ScopedPyObjectPtr m(PyObject_GetAttrString(child_list.get(), "sort"));
+  if (m == nullptr) return -1;
+  if (ScopedPyObjectPtr(PyObject_Call(m.get(), args, kwds)) == nullptr)
+    return -1;
+  ReorderAttached(self, child_list.get());
+  return 0;
+}
+
+static PyObject* Sort(PyObject* pself, PyObject* args, PyObject* kwds) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
+  // Support the old sort_function argument for backwards
+  // compatibility.
+  if (kwds != nullptr) {
+    PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function");
+    if (sort_func != nullptr) {
+      // Must set before deleting as sort_func is a borrowed reference
+      // and kwds might be the only thing keeping it alive.
+      PyDict_SetItemString(kwds, "cmp", sort_func);
+      PyDict_DelItemString(kwds, "sort_function");
+    }
+  }
+
+  if (SortPythonMessages(self, args, kwds) < 0) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+// ---------------------------------------------------------------------
+// reverse()
+
+// Returns 0 if successful; returns -1 and sets an exception if
+// unsuccessful.
+static int ReversePythonMessages(RepeatedCompositeContainer* self) {
+  ScopedPyObjectPtr child_list(
+      PySequence_List(reinterpret_cast<PyObject*>(self)));
+  if (child_list == nullptr) {
+    return -1;
+  }
+  if (ScopedPyObjectPtr(
+          PyObject_CallMethod(child_list.get(), "reverse", nullptr)) == nullptr)
+    return -1;
+  ReorderAttached(self, child_list.get());
+  return 0;
+}
+
+static PyObject* Reverse(PyObject* pself) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
+  if (ReversePythonMessages(self) < 0) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+// ---------------------------------------------------------------------
+
+static PyObject* Item(PyObject* pself, Py_ssize_t index) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+  return GetItem(self, index);
+}
+
+static PyObject* Pop(PyObject* pself, PyObject* args) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+
+  Py_ssize_t index = -1;
+  if (!PyArg_ParseTuple(args, "|n", &index)) {
+    return nullptr;
+  }
+  Py_ssize_t length = Length(pself);
+  if (index < 0) index += length;
+  PyObject* item = GetItem(self, index, length);
+  if (item == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr py_index(PyLong_FromSsize_t(index));
+  if (AssignSubscript(self, py_index.get(), nullptr) < 0) {
+    return nullptr;
+  }
+  return item;
+}
+
+PyObject* DeepCopy(PyObject* pself, PyObject* arg) {
+  return reinterpret_cast<RepeatedCompositeContainer*>(pself)->DeepCopy();
+}
+
+// The private constructor of RepeatedCompositeContainer objects.
+RepeatedCompositeContainer* NewContainer(
+    CMessage* parent, const FieldDescriptor* parent_field_descriptor,
+    CMessageClass* child_message_class) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return nullptr;
+  }
+
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(
+          PyType_GenericAlloc(&RepeatedCompositeContainer_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+  Py_INCREF(child_message_class);
+  self->child_message_class = child_message_class;
+  return self;
+}
+
+static void Dealloc(PyObject* pself) {
+  RepeatedCompositeContainer* self =
+      reinterpret_cast<RepeatedCompositeContainer*>(pself);
+  self->RemoveFromParentCache();
+  Py_CLEAR(self->child_message_class);
+  Py_TYPE(self)->tp_free(pself);
+}
+
+static PySequenceMethods SqMethods = {
+    Length,  /* sq_length */
+    nullptr, /* sq_concat */
+    nullptr, /* sq_repeat */
+    Item     /* sq_item */
+};
+
+static PyMappingMethods MpMethods = {
+    Length,                /* mp_length */
+    SubscriptMethod,       /* mp_subscript */
+    AssignSubscriptMethod, /* mp_ass_subscript */
+};
+
+static PyMethodDef Methods[] = {
+    {"__deepcopy__", DeepCopy, METH_VARARGS, "Makes a deep copy of the class."},
+    {"add", reinterpret_cast<PyCFunction>(AddMethod),
+     METH_VARARGS | METH_KEYWORDS, "Adds an object to the repeated container."},
+    {"append", AppendMethod, METH_O,
+     "Appends a message to the end of the repeated container."},
+    {"insert", Insert, METH_VARARGS,
+     "Inserts a message before the specified index."},
+    {"extend", ExtendMethod, METH_O, "Adds objects to the repeated container."},
+    {"pop", Pop, METH_VARARGS,
+     "Removes an object from the repeated container and returns it."},
+    {"remove", Remove, METH_O,
+     "Removes an object from the repeated container."},
+    {"sort", reinterpret_cast<PyCFunction>(Sort), METH_VARARGS | METH_KEYWORDS,
+     "Sorts the repeated container."},
+    {"reverse", reinterpret_cast<PyCFunction>(Reverse), METH_NOARGS,
+     "Reverses elements order of the repeated container."},
+    {"MergeFrom", MergeFromMethod, METH_O,
+     "Adds objects to the repeated container."},
+    {nullptr, nullptr}};
+
+}  // namespace repeated_composite_container
+
+PyTypeObject RepeatedCompositeContainer_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".RepeatedCompositeContainer",          // tp_name
+    sizeof(RepeatedCompositeContainer),     // tp_basicsize
+    0,                                      //  tp_itemsize
+    repeated_composite_container::Dealloc,  //  tp_dealloc
+#if PY_VERSION_HEX >= 0x03080000
+    0,  //  tp_vectorcall_offset
+#else
+    nullptr,  //  tp_print
+#endif
+    nullptr,                                    //  tp_getattr
+    nullptr,                                    //  tp_setattr
+    nullptr,                                    //  tp_compare
+    repeated_composite_container::ToStr,        //  tp_repr
+    nullptr,                                    //  tp_as_number
+    &repeated_composite_container::SqMethods,   //  tp_as_sequence
+    &repeated_composite_container::MpMethods,   //  tp_as_mapping
+    PyObject_HashNotImplemented,                //  tp_hash
+    nullptr,                                    //  tp_call
+    nullptr,                                    //  tp_str
+    nullptr,                                    //  tp_getattro
+    nullptr,                                    //  tp_setattro
+    nullptr,                                    //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,                         //  tp_flags
+    "A Repeated scalar container",              //  tp_doc
+    nullptr,                                    //  tp_traverse
+    nullptr,                                    //  tp_clear
+    repeated_composite_container::RichCompare,  //  tp_richcompare
+    0,                                          //  tp_weaklistoffset
+    nullptr,                                    //  tp_iter
+    nullptr,                                    //  tp_iternext
+    repeated_composite_container::Methods,      //  tp_methods
+    nullptr,                                    //  tp_members
+    nullptr,                                    //  tp_getset
+    nullptr,                                    //  tp_base
+    nullptr,                                    //  tp_dict
+    nullptr,                                    //  tp_descr_get
+    nullptr,                                    //  tp_descr_set
+    0,                                          //  tp_dictoffset
+    nullptr,                                    //  tp_init
+};
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/repeated_composite_container.h

@@ -0,0 +1,109 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: anuraag@google.com (Anuraag Agrawal)
+// Author: tibell@google.com (Johan Tibell)
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "google/protobuf/pyext/message.h"
+
+namespace google {
+namespace protobuf {
+
+class FieldDescriptor;
+class Message;
+
+namespace python {
+
+struct CMessageClass;
+
+// A RepeatedCompositeContainer always has a parent message.
+// The parent message also caches reference to items of the container.
+typedef struct RepeatedCompositeContainer : public ContainerBase {
+  // The type used to create new child messages.
+  CMessageClass* child_message_class;
+} RepeatedCompositeContainer;
+
+extern PyTypeObject RepeatedCompositeContainer_Type;
+
+namespace repeated_composite_container {
+
+// Builds a RepeatedCompositeContainer object, from a parent message and a
+// field descriptor.
+RepeatedCompositeContainer* NewContainer(
+    CMessage* parent,
+    const FieldDescriptor* parent_field_descriptor,
+    CMessageClass *child_message_class);
+
+// Appends a new CMessage to the container and returns it.  The
+// CMessage is initialized using the content of kwargs.
+//
+// Returns a new reference if successful; returns NULL and sets an
+// exception if unsuccessful.
+PyObject* Add(RepeatedCompositeContainer* self,
+              PyObject* args,
+              PyObject* kwargs);
+
+// Appends all the CMessages in the input iterator to the container.
+//
+// Returns None if successful; returns NULL and sets an exception if
+// unsuccessful.
+PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value);
+
+// Appends a new message to the container for each message in the
+// input iterator, merging each data element in. Equivalent to extend.
+//
+// Returns None if successful; returns NULL and sets an exception if
+// unsuccessful.
+PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other);
+
+// Accesses messages in the container.
+//
+// Returns a new reference to the message for an integer parameter.
+// Returns a new reference to a list of messages for a slice.
+PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice);
+
+// Deletes items from the container (cannot be used for assignment).
+//
+// Returns 0 on success, -1 on failure.
+int AssignSubscript(RepeatedCompositeContainer* self,
+                    PyObject* slice,
+                    PyObject* value);
+}  // namespace repeated_composite_container
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__

python/google/protobuf/pyext/repeated_scalar_container.cc

@@ -0,0 +1,775 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: anuraag@google.com (Anuraag Agrawal)
+// Author: tibell@google.com (Johan Tibell)
+
+#include "google/protobuf/pyext/repeated_scalar_container.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "google/protobuf/stubs/common.h"
+#include "google/protobuf/stubs/logging.h"
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/dynamic_message.h"
+#include "google/protobuf/message.h"
+#include "google/protobuf/pyext/descriptor.h"
+#include "google/protobuf/pyext/descriptor_pool.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+
+#define PyString_AsString(ob) \
+  (PyUnicode_Check(ob) ? PyUnicode_AsUTF8(ob) : PyBytes_AsString(ob))
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+namespace repeated_scalar_container {
+
+static int InternalAssignRepeatedField(RepeatedScalarContainer* self,
+                                       PyObject* list) {
+  Message* message = self->parent->message;
+  message->GetReflection()->ClearField(message, self->parent_field_descriptor);
+  for (Py_ssize_t i = 0; i < PyList_GET_SIZE(list); ++i) {
+    PyObject* value = PyList_GET_ITEM(list, i);
+    if (ScopedPyObjectPtr(Append(self, value)) == nullptr) {
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static Py_ssize_t Len(PyObject* pself) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+  Message* message = self->parent->message;
+  return message->GetReflection()->FieldSize(*message,
+                                             self->parent_field_descriptor);
+}
+
+static int AssignItem(PyObject* pself, Py_ssize_t index, PyObject* arg) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
+  cmessage::AssureWritable(self->parent);
+  Message* message = self->parent->message;
+  const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
+
+  const Reflection* reflection = message->GetReflection();
+  int field_size = reflection->FieldSize(*message, field_descriptor);
+  if (index < 0) {
+    index = field_size + index;
+  }
+  if (index < 0 || index >= field_size) {
+    PyErr_Format(PyExc_IndexError, "list assignment index (%d) out of range",
+                 static_cast<int>(index));
+    return -1;
+  }
+
+  if (arg == nullptr) {
+    ScopedPyObjectPtr py_index(PyLong_FromLong(index));
+    return cmessage::DeleteRepeatedField(self->parent, field_descriptor,
+                                         py_index.get());
+  }
+
+  if (PySequence_Check(arg) && !(PyBytes_Check(arg) || PyUnicode_Check(arg))) {
+    PyErr_SetString(PyExc_TypeError, "Value must be scalar");
+    return -1;
+  }
+
+  switch (field_descriptor->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32: {
+      GOOGLE_CHECK_GET_INT32(arg, value, -1);
+      reflection->SetRepeatedInt32(message, field_descriptor, index, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_INT64: {
+      GOOGLE_CHECK_GET_INT64(arg, value, -1);
+      reflection->SetRepeatedInt64(message, field_descriptor, index, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT32: {
+      GOOGLE_CHECK_GET_UINT32(arg, value, -1);
+      reflection->SetRepeatedUInt32(message, field_descriptor, index, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT64: {
+      GOOGLE_CHECK_GET_UINT64(arg, value, -1);
+      reflection->SetRepeatedUInt64(message, field_descriptor, index, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_FLOAT: {
+      GOOGLE_CHECK_GET_FLOAT(arg, value, -1);
+      reflection->SetRepeatedFloat(message, field_descriptor, index, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_DOUBLE: {
+      GOOGLE_CHECK_GET_DOUBLE(arg, value, -1);
+      reflection->SetRepeatedDouble(message, field_descriptor, index, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_BOOL: {
+      GOOGLE_CHECK_GET_BOOL(arg, value, -1);
+      reflection->SetRepeatedBool(message, field_descriptor, index, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_STRING: {
+      if (!CheckAndSetString(arg, message, field_descriptor, reflection, false,
+                             index)) {
+        return -1;
+      }
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_ENUM: {
+      GOOGLE_CHECK_GET_INT32(arg, value, -1);
+      if (reflection->SupportsUnknownEnumValues()) {
+        reflection->SetRepeatedEnumValue(message, field_descriptor, index,
+                                         value);
+      } else {
+        const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
+        const EnumValueDescriptor* enum_value =
+            enum_descriptor->FindValueByNumber(value);
+        if (enum_value != nullptr) {
+          reflection->SetRepeatedEnum(message, field_descriptor, index,
+                                      enum_value);
+        } else {
+          ScopedPyObjectPtr s(PyObject_Str(arg));
+          if (s != nullptr) {
+            PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
+                         PyString_AsString(s.get()));
+          }
+          return -1;
+        }
+      }
+      break;
+    }
+    default:
+      PyErr_Format(PyExc_SystemError,
+                   "Adding value to a field of unknown type %d",
+                   field_descriptor->cpp_type());
+      return -1;
+  }
+  return 0;
+}
+
+static PyObject* Item(PyObject* pself, Py_ssize_t index) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
+  Message* message = self->parent->message;
+  const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
+  const Reflection* reflection = message->GetReflection();
+
+  int field_size = reflection->FieldSize(*message, field_descriptor);
+  if (index < 0) {
+    index = field_size + index;
+  }
+  if (index < 0 || index >= field_size) {
+    PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index);
+    return nullptr;
+  }
+
+  PyObject* result = nullptr;
+  switch (field_descriptor->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32: {
+      int32_t value =
+          reflection->GetRepeatedInt32(*message, field_descriptor, index);
+      result = PyLong_FromLong(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_INT64: {
+      int64_t value =
+          reflection->GetRepeatedInt64(*message, field_descriptor, index);
+      result = PyLong_FromLongLong(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT32: {
+      uint32_t value =
+          reflection->GetRepeatedUInt32(*message, field_descriptor, index);
+      result = PyLong_FromLongLong(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT64: {
+      uint64_t value =
+          reflection->GetRepeatedUInt64(*message, field_descriptor, index);
+      result = PyLong_FromUnsignedLongLong(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_FLOAT: {
+      float value =
+          reflection->GetRepeatedFloat(*message, field_descriptor, index);
+      result = PyFloat_FromDouble(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_DOUBLE: {
+      double value =
+          reflection->GetRepeatedDouble(*message, field_descriptor, index);
+      result = PyFloat_FromDouble(value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_BOOL: {
+      bool value =
+          reflection->GetRepeatedBool(*message, field_descriptor, index);
+      result = PyBool_FromLong(value ? 1 : 0);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_ENUM: {
+      const EnumValueDescriptor* enum_value =
+          message->GetReflection()->GetRepeatedEnum(*message, field_descriptor,
+                                                    index);
+      result = PyLong_FromLong(enum_value->number());
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_STRING: {
+      std::string scratch;
+      const std::string& value = reflection->GetRepeatedStringReference(
+          *message, field_descriptor, index, &scratch);
+      result = ToStringObject(field_descriptor, value);
+      break;
+    }
+    default:
+      PyErr_Format(PyExc_SystemError,
+                   "Getting value from a repeated field of unknown type %d",
+                   field_descriptor->cpp_type());
+  }
+
+  return result;
+}
+
+static PyObject* Subscript(PyObject* pself, PyObject* slice) {
+  Py_ssize_t from;
+  Py_ssize_t to;
+  Py_ssize_t step;
+  Py_ssize_t length;
+  Py_ssize_t slicelength;
+  bool return_list = false;
+  if (PyLong_Check(slice)) {
+    from = to = PyLong_AsLong(slice);
+  } else if (PyIndex_Check(slice)) {
+    from = to = PyNumber_AsSsize_t(slice, PyExc_ValueError);
+    if (from == -1 && PyErr_Occurred()) {
+      return nullptr;
+    }
+  } else if (PySlice_Check(slice)) {
+    length = Len(pself);
+    if (PySlice_GetIndicesEx(slice, length, &from, &to, &step, &slicelength) ==
+        -1) {
+      return nullptr;
+    }
+    return_list = true;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "list indices must be integers");
+    return nullptr;
+  }
+
+  if (!return_list) {
+    return Item(pself, from);
+  }
+
+  PyObject* list = PyList_New(0);
+  if (list == nullptr) {
+    return nullptr;
+  }
+  if (from <= to) {
+    if (step < 0) {
+      return list;
+    }
+    for (Py_ssize_t index = from; index < to; index += step) {
+      if (index < 0 || index >= length) {
+        break;
+      }
+      ScopedPyObjectPtr s(Item(pself, index));
+      PyList_Append(list, s.get());
+    }
+  } else {
+    if (step > 0) {
+      return list;
+    }
+    for (Py_ssize_t index = from; index > to; index += step) {
+      if (index < 0 || index >= length) {
+        break;
+      }
+      ScopedPyObjectPtr s(Item(pself, index));
+      PyList_Append(list, s.get());
+    }
+  }
+  return list;
+}
+
+PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
+  cmessage::AssureWritable(self->parent);
+  Message* message = self->parent->message;
+  const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
+
+  const Reflection* reflection = message->GetReflection();
+  switch (field_descriptor->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32: {
+      GOOGLE_CHECK_GET_INT32(item, value, nullptr);
+      reflection->AddInt32(message, field_descriptor, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_INT64: {
+      GOOGLE_CHECK_GET_INT64(item, value, nullptr);
+      reflection->AddInt64(message, field_descriptor, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT32: {
+      GOOGLE_CHECK_GET_UINT32(item, value, nullptr);
+      reflection->AddUInt32(message, field_descriptor, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_UINT64: {
+      GOOGLE_CHECK_GET_UINT64(item, value, nullptr);
+      reflection->AddUInt64(message, field_descriptor, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_FLOAT: {
+      GOOGLE_CHECK_GET_FLOAT(item, value, nullptr);
+      reflection->AddFloat(message, field_descriptor, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_DOUBLE: {
+      GOOGLE_CHECK_GET_DOUBLE(item, value, nullptr);
+      reflection->AddDouble(message, field_descriptor, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_BOOL: {
+      GOOGLE_CHECK_GET_BOOL(item, value, nullptr);
+      reflection->AddBool(message, field_descriptor, value);
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_STRING: {
+      if (!CheckAndSetString(item, message, field_descriptor, reflection, true,
+                             -1)) {
+        return nullptr;
+      }
+      break;
+    }
+    case FieldDescriptor::CPPTYPE_ENUM: {
+      GOOGLE_CHECK_GET_INT32(item, value, nullptr);
+      if (reflection->SupportsUnknownEnumValues()) {
+        reflection->AddEnumValue(message, field_descriptor, value);
+      } else {
+        const EnumDescriptor* enum_descriptor = field_descriptor->enum_type();
+        const EnumValueDescriptor* enum_value =
+            enum_descriptor->FindValueByNumber(value);
+        if (enum_value != nullptr) {
+          reflection->AddEnum(message, field_descriptor, enum_value);
+        } else {
+          ScopedPyObjectPtr s(PyObject_Str(item));
+          if (s != nullptr) {
+            PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
+                         PyString_AsString(s.get()));
+          }
+          return nullptr;
+        }
+      }
+      break;
+    }
+    default:
+      PyErr_Format(PyExc_SystemError,
+                   "Adding value to a field of unknown type %d",
+                   field_descriptor->cpp_type());
+      return nullptr;
+  }
+
+  Py_RETURN_NONE;
+}
+
+static PyObject* AppendMethod(PyObject* self, PyObject* item) {
+  return Append(reinterpret_cast<RepeatedScalarContainer*>(self), item);
+}
+
+static int AssSubscript(PyObject* pself, PyObject* slice, PyObject* value) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
+  Py_ssize_t from;
+  Py_ssize_t to;
+  Py_ssize_t step;
+  Py_ssize_t length;
+  Py_ssize_t slicelength;
+  bool create_list = false;
+
+  cmessage::AssureWritable(self->parent);
+  Message* message = self->parent->message;
+  const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
+
+  if (PyLong_Check(slice)) {
+    from = to = PyLong_AsLong(slice);
+  } else if (PySlice_Check(slice)) {
+    const Reflection* reflection = message->GetReflection();
+    length = reflection->FieldSize(*message, field_descriptor);
+    if (PySlice_GetIndicesEx(slice, length, &from, &to, &step, &slicelength) ==
+        -1) {
+      return -1;
+    }
+    create_list = true;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "list indices must be integers");
+    return -1;
+  }
+
+  if (value == nullptr) {
+    return cmessage::DeleteRepeatedField(self->parent, field_descriptor, slice);
+  }
+
+  if (!create_list) {
+    return AssignItem(pself, from, value);
+  }
+
+  ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+  if (full_slice == nullptr) {
+    return -1;
+  }
+  ScopedPyObjectPtr new_list(Subscript(pself, full_slice.get()));
+  if (new_list == nullptr) {
+    return -1;
+  }
+  if (PySequence_SetSlice(new_list.get(), from, to, value) < 0) {
+    return -1;
+  }
+
+  return InternalAssignRepeatedField(self, new_list.get());
+}
+
+PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) {
+  cmessage::AssureWritable(self->parent);
+
+  // TODO(ptucker): Deprecate this behavior. b/18413862
+  if (value == Py_None) {
+    Py_RETURN_NONE;
+  }
+  if ((Py_TYPE(value)->tp_as_sequence == nullptr) && PyObject_Not(value)) {
+    Py_RETURN_NONE;
+  }
+
+  ScopedPyObjectPtr iter(PyObject_GetIter(value));
+  if (iter == nullptr) {
+    PyErr_SetString(PyExc_TypeError, "Value must be iterable");
+    return nullptr;
+  }
+  ScopedPyObjectPtr next;
+  while ((next.reset(PyIter_Next(iter.get()))) != nullptr) {
+    if (ScopedPyObjectPtr(Append(self, next.get())) == nullptr) {
+      return nullptr;
+    }
+  }
+  if (PyErr_Occurred()) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* Insert(PyObject* pself, PyObject* args) {
+  RepeatedScalarContainer* self =
+      reinterpret_cast<RepeatedScalarContainer*>(pself);
+
+  Py_ssize_t index;
+  PyObject* value;
+  if (!PyArg_ParseTuple(args, "lO", &index, &value)) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+  ScopedPyObjectPtr new_list(Subscript(pself, full_slice.get()));
+  if (PyList_Insert(new_list.get(), index, value) < 0) {
+    return nullptr;
+  }
+  int ret = InternalAssignRepeatedField(self, new_list.get());
+  if (ret < 0) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* Remove(PyObject* pself, PyObject* value) {
+  Py_ssize_t match_index = -1;
+  for (Py_ssize_t i = 0; i < Len(pself); ++i) {
+    ScopedPyObjectPtr elem(Item(pself, i));
+    if (PyObject_RichCompareBool(elem.get(), value, Py_EQ)) {
+      match_index = i;
+      break;
+    }
+  }
+  if (match_index == -1) {
+    PyErr_SetString(PyExc_ValueError, "remove(x): x not in container");
+    return nullptr;
+  }
+  if (AssignItem(pself, match_index, nullptr) < 0) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* ExtendMethod(PyObject* self, PyObject* value) {
+  return Extend(reinterpret_cast<RepeatedScalarContainer*>(self), value);
+}
+
+static PyObject* RichCompare(PyObject* pself, PyObject* other, int opid) {
+  if (opid != Py_EQ && opid != Py_NE) {
+    Py_INCREF(Py_NotImplemented);
+    return Py_NotImplemented;
+  }
+
+  // Copy the contents of this repeated scalar container, and other if it is
+  // also a repeated scalar container, into Python lists so we can delegate
+  // to the list's compare method.
+
+  ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+  if (full_slice == nullptr) {
+    return nullptr;
+  }
+
+  ScopedPyObjectPtr other_list_deleter;
+  if (PyObject_TypeCheck(other, &RepeatedScalarContainer_Type)) {
+    other_list_deleter.reset(Subscript(other, full_slice.get()));
+    other = other_list_deleter.get();
+  }
+
+  ScopedPyObjectPtr list(Subscript(pself, full_slice.get()));
+  if (list == nullptr) {
+    return nullptr;
+  }
+  return PyObject_RichCompare(list.get(), other, opid);
+}
+
+PyObject* Reduce(PyObject* unused_self, PyObject* unused_other) {
+  PyErr_Format(PickleError_class,
+               "can't pickle repeated message fields, convert to list first");
+  return nullptr;
+}
+
+static PyObject* Sort(PyObject* pself, PyObject* args, PyObject* kwds) {
+  // Support the old sort_function argument for backwards
+  // compatibility.
+  if (kwds != nullptr) {
+    PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function");
+    if (sort_func != nullptr) {
+      // Must set before deleting as sort_func is a borrowed reference
+      // and kwds might be the only thing keeping it alive.
+      if (PyDict_SetItemString(kwds, "cmp", sort_func) == -1) return nullptr;
+      if (PyDict_DelItemString(kwds, "sort_function") == -1) return nullptr;
+    }
+  }
+
+  ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+  if (full_slice == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr list(Subscript(pself, full_slice.get()));
+  if (list == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr m(PyObject_GetAttrString(list.get(), "sort"));
+  if (m == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr res(PyObject_Call(m.get(), args, kwds));
+  if (res == nullptr) {
+    return nullptr;
+  }
+  int ret = InternalAssignRepeatedField(
+      reinterpret_cast<RepeatedScalarContainer*>(pself), list.get());
+  if (ret < 0) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* Reverse(PyObject* pself) {
+  ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+  if (full_slice == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr list(Subscript(pself, full_slice.get()));
+  if (list == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr res(PyObject_CallMethod(list.get(), "reverse", nullptr));
+  if (res == nullptr) {
+    return nullptr;
+  }
+  int ret = InternalAssignRepeatedField(
+      reinterpret_cast<RepeatedScalarContainer*>(pself), list.get());
+  if (ret < 0) {
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* Pop(PyObject* pself, PyObject* args) {
+  Py_ssize_t index = -1;
+  if (!PyArg_ParseTuple(args, "|n", &index)) {
+    return nullptr;
+  }
+  PyObject* item = Item(pself, index);
+  if (item == nullptr) {
+    PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index);
+    return nullptr;
+  }
+  if (AssignItem(pself, index, nullptr) < 0) {
+    return nullptr;
+  }
+  return item;
+}
+
+static PyObject* ToStr(PyObject* pself) {
+  ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
+  if (full_slice == nullptr) {
+    return nullptr;
+  }
+  ScopedPyObjectPtr list(Subscript(pself, full_slice.get()));
+  if (list == nullptr) {
+    return nullptr;
+  }
+  return PyObject_Repr(list.get());
+}
+
+static PyObject* MergeFrom(PyObject* pself, PyObject* arg) {
+  return Extend(reinterpret_cast<RepeatedScalarContainer*>(pself), arg);
+}
+
+// The private constructor of RepeatedScalarContainer objects.
+RepeatedScalarContainer* NewContainer(
+    CMessage* parent, const FieldDescriptor* parent_field_descriptor) {
+  if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
+    return nullptr;
+  }
+
+  RepeatedScalarContainer* self = reinterpret_cast<RepeatedScalarContainer*>(
+      PyType_GenericAlloc(&RepeatedScalarContainer_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  self->parent_field_descriptor = parent_field_descriptor;
+
+  return self;
+}
+
+PyObject* DeepCopy(PyObject* pself, PyObject* arg) {
+  return reinterpret_cast<RepeatedScalarContainer*>(pself)->DeepCopy();
+}
+
+static void Dealloc(PyObject* pself) {
+  reinterpret_cast<RepeatedScalarContainer*>(pself)->RemoveFromParentCache();
+  Py_TYPE(pself)->tp_free(pself);
+}
+
+static PySequenceMethods SqMethods = {
+    Len,       /* sq_length */
+    nullptr,   /* sq_concat */
+    nullptr,   /* sq_repeat */
+    Item,      /* sq_item */
+    nullptr,   /* sq_slice */
+    AssignItem /* sq_ass_item */
+};
+
+static PyMappingMethods MpMethods = {
+    Len,          /* mp_length */
+    Subscript,    /* mp_subscript */
+    AssSubscript, /* mp_ass_subscript */
+};
+
+static PyMethodDef Methods[] = {
+    {"__deepcopy__", DeepCopy, METH_VARARGS, "Makes a deep copy of the class."},
+    {"__reduce__", Reduce, METH_NOARGS,
+     "Outputs picklable representation of the repeated field."},
+    {"append", AppendMethod, METH_O,
+     "Appends an object to the repeated container."},
+    {"extend", ExtendMethod, METH_O,
+     "Appends objects to the repeated container."},
+    {"insert", Insert, METH_VARARGS,
+     "Inserts an object at the specified position in the container."},
+    {"pop", Pop, METH_VARARGS,
+     "Removes an object from the repeated container and returns it."},
+    {"remove", Remove, METH_O,
+     "Removes an object from the repeated container."},
+    {"sort", reinterpret_cast<PyCFunction>(Sort), METH_VARARGS | METH_KEYWORDS,
+     "Sorts the repeated container."},
+    {"reverse", reinterpret_cast<PyCFunction>(Reverse), METH_NOARGS,
+     "Reverses elements order of the repeated container."},
+    {"MergeFrom", static_cast<PyCFunction>(MergeFrom), METH_O,
+     "Merges a repeated container into the current container."},
+    {nullptr, nullptr}};
+
+}  // namespace repeated_scalar_container
+
+PyTypeObject RepeatedScalarContainer_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".RepeatedScalarContainer",              // tp_name
+    sizeof(RepeatedScalarContainer),         // tp_basicsize
+    0,                                       //  tp_itemsize
+    repeated_scalar_container::Dealloc,      //  tp_dealloc
+#if PY_VERSION_HEX >= 0x03080000
+    0,                                       //  tp_vectorcall_offset
+#else
+    nullptr,                                 //  tp_print
+#endif
+    nullptr,                                 //  tp_getattr
+    nullptr,                                 //  tp_setattr
+    nullptr,                                 //  tp_compare
+    repeated_scalar_container::ToStr,        //  tp_repr
+    nullptr,                                 //  tp_as_number
+    &repeated_scalar_container::SqMethods,   //  tp_as_sequence
+    &repeated_scalar_container::MpMethods,   //  tp_as_mapping
+    PyObject_HashNotImplemented,             //  tp_hash
+    nullptr,                                 //  tp_call
+    nullptr,                                 //  tp_str
+    nullptr,                                 //  tp_getattro
+    nullptr,                                 //  tp_setattro
+    nullptr,                                 //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,                      //  tp_flags
+    "A Repeated scalar container",           //  tp_doc
+    nullptr,                                 //  tp_traverse
+    nullptr,                                 //  tp_clear
+    repeated_scalar_container::RichCompare,  //  tp_richcompare
+    0,                                       //  tp_weaklistoffset
+    nullptr,                                 //  tp_iter
+    nullptr,                                 //  tp_iternext
+    repeated_scalar_container::Methods,      //  tp_methods
+    nullptr,                                 //  tp_members
+    nullptr,                                 //  tp_getset
+    nullptr,                                 //  tp_base
+    nullptr,                                 //  tp_dict
+    nullptr,                                 //  tp_descr_get
+    nullptr,                                 //  tp_descr_set
+    0,                                       //  tp_dictoffset
+    nullptr,                                 //  tp_init
+};
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/repeated_scalar_container.h

@@ -0,0 +1,76 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: anuraag@google.com (Anuraag Agrawal)
+// Author: tibell@google.com (Johan Tibell)
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "google/protobuf/descriptor.h"
+#include "google/protobuf/pyext/message.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+typedef struct RepeatedScalarContainer : public ContainerBase {
+} RepeatedScalarContainer;
+
+extern PyTypeObject RepeatedScalarContainer_Type;
+
+namespace repeated_scalar_container {
+
+// Builds a RepeatedScalarContainer object, from a parent message and a
+// field descriptor.
+extern RepeatedScalarContainer* NewContainer(
+    CMessage* parent, const FieldDescriptor* parent_field_descriptor);
+
+// Appends the scalar 'item' to the end of the container 'self'.
+//
+// Returns None if successful; returns NULL and sets an exception if
+// unsuccessful.
+PyObject* Append(RepeatedScalarContainer* self, PyObject* item);
+
+// Appends all the elements in the input iterator to the container.
+//
+// Returns None if successful; returns NULL and sets an exception if
+// unsuccessful.
+PyObject* Extend(RepeatedScalarContainer* self, PyObject* value);
+
+}  // namespace repeated_scalar_container
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__

python/google/protobuf/pyext/safe_numerics.h

@@ -0,0 +1,164 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SAFE_NUMERICS_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_SAFE_NUMERICS_H__
+// Copied from chromium with only changes to the namespace.
+
+#include <limits>
+
+#include "google/protobuf/stubs/logging.h"
+#include "google/protobuf/stubs/common.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+template <bool SameSize, bool DestLarger,
+          bool DestIsSigned, bool SourceIsSigned>
+struct IsValidNumericCastImpl;
+
+#define BASE_NUMERIC_CAST_CASE_SPECIALIZATION(A, B, C, D, Code) \
+template <> struct IsValidNumericCastImpl<A, B, C, D> { \
+  template <class Source, class DestBounds> static inline bool Test( \
+      Source source, DestBounds min, DestBounds max) { \
+    return Code; \
+  } \
+}
+
+#define BASE_NUMERIC_CAST_CASE_SAME_SIZE(DestSigned, SourceSigned, Code) \
+  BASE_NUMERIC_CAST_CASE_SPECIALIZATION( \
+      true, true, DestSigned, SourceSigned, Code); \
+  BASE_NUMERIC_CAST_CASE_SPECIALIZATION( \
+      true, false, DestSigned, SourceSigned, Code)
+
+#define BASE_NUMERIC_CAST_CASE_SOURCE_LARGER(DestSigned, SourceSigned, Code) \
+  BASE_NUMERIC_CAST_CASE_SPECIALIZATION( \
+      false, false, DestSigned, SourceSigned, Code); \
+
+#define BASE_NUMERIC_CAST_CASE_DEST_LARGER(DestSigned, SourceSigned, Code) \
+  BASE_NUMERIC_CAST_CASE_SPECIALIZATION( \
+      false, true, DestSigned, SourceSigned, Code); \
+
+// The three top level cases are:
+// - Same size
+// - Source larger
+// - Dest larger
+// And for each of those three cases, we handle the 4 different possibilities
+// of signed and unsigned. This gives 12 cases to handle, which we enumerate
+// below.
+//
+// The last argument in each of the macros is the actual comparison code. It
+// has three arguments available, source (the value), and min/max which are
+// the ranges of the destination.
+
+
+// These are the cases where both types have the same size.
+
+// Both signed.
+BASE_NUMERIC_CAST_CASE_SAME_SIZE(true, true, true);
+// Both unsigned.
+BASE_NUMERIC_CAST_CASE_SAME_SIZE(false, false, true);
+// Dest unsigned, Source signed.
+BASE_NUMERIC_CAST_CASE_SAME_SIZE(false, true, source >= 0);
+// Dest signed, Source unsigned.
+// This cast is OK because Dest's max must be less than Source's.
+BASE_NUMERIC_CAST_CASE_SAME_SIZE(true, false,
+                                 source <= static_cast<Source>(max));
+
+
+// These are the cases where Source is larger.
+
+// Both unsigned.
+BASE_NUMERIC_CAST_CASE_SOURCE_LARGER(false, false, source <= max);
+// Both signed.
+BASE_NUMERIC_CAST_CASE_SOURCE_LARGER(true, true,
+                                     source >= min && source <= max);
+// Dest is unsigned, Source is signed.
+BASE_NUMERIC_CAST_CASE_SOURCE_LARGER(false, true,
+                                     source >= 0 && source <= max);
+// Dest is signed, Source is unsigned.
+// This cast is OK because Dest's max must be less than Source's.
+BASE_NUMERIC_CAST_CASE_SOURCE_LARGER(true, false,
+                                     source <= static_cast<Source>(max));
+
+
+// These are the cases where Dest is larger.
+
+// Both unsigned.
+BASE_NUMERIC_CAST_CASE_DEST_LARGER(false, false, true);
+// Both signed.
+BASE_NUMERIC_CAST_CASE_DEST_LARGER(true, true, true);
+// Dest is unsigned, Source is signed.
+BASE_NUMERIC_CAST_CASE_DEST_LARGER(false, true, source >= 0);
+// Dest is signed, Source is unsigned.
+BASE_NUMERIC_CAST_CASE_DEST_LARGER(true, false, true);
+
+#undef BASE_NUMERIC_CAST_CASE_SPECIALIZATION
+#undef BASE_NUMERIC_CAST_CASE_SAME_SIZE
+#undef BASE_NUMERIC_CAST_CASE_SOURCE_LARGER
+#undef BASE_NUMERIC_CAST_CASE_DEST_LARGER
+
+
+// The main test for whether the conversion will under or overflow.
+template <class Dest, class Source>
+inline bool IsValidNumericCast(Source source) {
+  typedef std::numeric_limits<Source> SourceLimits;
+  typedef std::numeric_limits<Dest> DestLimits;
+  static_assert(SourceLimits::is_specialized, "argument must be numeric");
+  static_assert(SourceLimits::is_integer, "argument must be integral");
+  static_assert(DestLimits::is_specialized, "result must be numeric");
+  static_assert(DestLimits::is_integer, "result must be integral");
+
+  return IsValidNumericCastImpl<
+      sizeof(Dest) == sizeof(Source),
+      (sizeof(Dest) > sizeof(Source)),
+      DestLimits::is_signed,
+      SourceLimits::is_signed>::Test(
+          source,
+          DestLimits::min(),
+          DestLimits::max());
+}
+
+// checked_numeric_cast<> is analogous to static_cast<> for numeric types,
+// except that it CHECKs that the specified numeric conversion will not
+// overflow or underflow. Floating point arguments are not currently allowed
+// (this is static_asserted), though this could be supported if necessary.
+template <class Dest, class Source>
+inline Dest checked_numeric_cast(Source source) {
+  GOOGLE_CHECK(IsValidNumericCast<Dest>(source));
+  return static_cast<Dest>(source);
+}
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_SAFE_NUMERICS_H__

python/google/protobuf/pyext/scoped_pyobject_ptr.h

@@ -0,0 +1,99 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: tibell@google.com (Johan Tibell)
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+namespace google {
+namespace protobuf {
+namespace python {
+
+// Owns a python object and decrements the reference count on destruction.
+// This class is not threadsafe.
+template <typename PyObjectStruct>
+class ScopedPythonPtr {
+ public:
+  // Takes the ownership of the specified object to ScopedPythonPtr.
+  // The reference count of the specified py_object is not incremented.
+  explicit ScopedPythonPtr(PyObjectStruct* py_object = nullptr)
+      : ptr_(py_object) {}
+  ScopedPythonPtr(const ScopedPythonPtr&) = delete;
+  ScopedPythonPtr& operator=(const ScopedPythonPtr&) = delete;
+
+  // If a PyObject is owned, decrement its reference count.
+  ~ScopedPythonPtr() { Py_XDECREF(ptr_); }
+
+  // Deletes the current owned object, if any.
+  // Then takes ownership of a new object without incrementing the reference
+  // count.
+  // This function must be called with a reference that you own.
+  //   this->reset(this->get()) is wrong!
+  //   this->reset(this->release()) is OK.
+  PyObjectStruct* reset(PyObjectStruct* p = nullptr) {
+    Py_XDECREF(ptr_);
+    ptr_ = p;
+    return ptr_;
+  }
+
+  // Releases ownership of the object without decrementing the reference count.
+  // The caller now owns the returned reference.
+  PyObjectStruct* release() {
+    PyObject* p = ptr_;
+    ptr_ = nullptr;
+    return p;
+  }
+
+  PyObjectStruct* get() const { return ptr_; }
+
+  PyObject* as_pyobject() const { return reinterpret_cast<PyObject*>(ptr_); }
+
+  // Increments the reference count of the current object.
+  // Should not be called when no object is held.
+  void inc() const { Py_INCREF(ptr_); }
+
+  // True when a ScopedPyObjectPtr and a raw pointer refer to the same object.
+  // Comparison operators are non reflexive.
+  bool operator==(const PyObjectStruct* p) const { return ptr_ == p; }
+  bool operator!=(const PyObjectStruct* p) const { return ptr_ != p; }
+
+ private:
+  PyObjectStruct* ptr_;
+};
+
+typedef ScopedPythonPtr<PyObject> ScopedPyObjectPtr;
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__

python/google/protobuf/pyext/unknown_field_set.cc

@@ -0,0 +1,355 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "google/protobuf/pyext/unknown_field_set.h"
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <memory>
+#include <set>
+
+#include "google/protobuf/message.h"
+#include "google/protobuf/unknown_field_set.h"
+#include "google/protobuf/wire_format_lite.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+namespace unknown_field_set {
+
+static Py_ssize_t Len(PyObject* pself) {
+  PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(pself);
+  if (self->fields == nullptr) {
+    PyErr_Format(PyExc_ValueError, "UnknownFieldSet does not exist. ");
+    return -1;
+  }
+  return self->fields->field_count();
+}
+
+PyObject* NewPyUnknownField(PyUnknownFieldSet* parent, Py_ssize_t index);
+
+static PyObject* Item(PyObject* pself, Py_ssize_t index) {
+  PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(pself);
+  if (self->fields == nullptr) {
+    PyErr_Format(PyExc_ValueError, "UnknownFieldSet does not exist. ");
+    return nullptr;
+  }
+  Py_ssize_t total_size = self->fields->field_count();
+  if (index < 0) {
+    index = total_size + index;
+  }
+  if (index < 0 || index >= total_size) {
+    PyErr_Format(PyExc_IndexError, "index (%zd) out of range", index);
+    return nullptr;
+  }
+  return unknown_field_set::NewPyUnknownField(self, index);
+}
+
+PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
+  if (args == nullptr || PyTuple_Size(args) != 1) {
+    PyErr_SetString(PyExc_TypeError,
+                    "Must provide a message to create UnknownFieldSet");
+    return nullptr;
+  }
+
+  PyObject* c_message;
+  if (!PyArg_ParseTuple(args, "O", &c_message)) {
+    PyErr_SetString(PyExc_TypeError,
+                    "Must provide a message to create UnknownFieldSet");
+    return nullptr;
+  }
+
+  if (!PyObject_TypeCheck(c_message, CMessage_Type)) {
+    PyErr_Format(PyExc_TypeError,
+                 "Parameter to UnknownFieldSet() must be a message "
+                 "got %s.",
+                 Py_TYPE(c_message)->tp_name);
+    return nullptr;
+  }
+
+  PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(
+      PyType_GenericAlloc(&PyUnknownFieldSet_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+
+  // Top UnknownFieldSet should set parent nullptr.
+  self->parent = nullptr;
+
+  // Copy c_message's UnknownFieldSet.
+  Message* message = reinterpret_cast<CMessage*>(c_message)->message;
+  const Reflection* reflection = message->GetReflection();
+  self->fields = new google::protobuf::UnknownFieldSet;
+  self->fields->MergeFrom(reflection->GetUnknownFields(*message));
+  return reinterpret_cast<PyObject*>(self);
+}
+
+PyObject* NewPyUnknownField(PyUnknownFieldSet* parent, Py_ssize_t index) {
+  PyUnknownField* self = reinterpret_cast<PyUnknownField*>(
+      PyType_GenericAlloc(&PyUnknownField_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  self->index = index;
+
+  return reinterpret_cast<PyObject*>(self);
+}
+
+static void Dealloc(PyObject* pself) {
+  PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(pself);
+  if (self->parent == nullptr) {
+    delete self->fields;
+  } else {
+    Py_CLEAR(self->parent);
+  }
+  auto* py_type = Py_TYPE(pself);
+  self->~PyUnknownFieldSet();
+  py_type->tp_free(pself);
+}
+
+static PySequenceMethods SqMethods = {
+    Len,     /* sq_length */
+    nullptr, /* sq_concat */
+    nullptr, /* sq_repeat */
+    Item,    /* sq_item */
+    nullptr, /* sq_slice */
+    nullptr, /* sq_ass_item */
+};
+
+}  // namespace unknown_field_set
+
+PyTypeObject PyUnknownFieldSet_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".PyUnknownFieldSet",        // tp_name
+    sizeof(PyUnknownFieldSet),   // tp_basicsize
+    0,                           //  tp_itemsize
+    unknown_field_set::Dealloc,  //  tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                        //  tp_getattr
+    nullptr,                        //  tp_setattr
+    nullptr,                        //  tp_compare
+    nullptr,                        //  tp_repr
+    nullptr,                        //  tp_as_number
+    &unknown_field_set::SqMethods,  //  tp_as_sequence
+    nullptr,                        //  tp_as_mapping
+    PyObject_HashNotImplemented,    //  tp_hash
+    nullptr,                        //  tp_call
+    nullptr,                        //  tp_str
+    nullptr,                        //  tp_getattro
+    nullptr,                        //  tp_setattro
+    nullptr,                        //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,             //  tp_flags
+    "unknown field set",            //  tp_doc
+    nullptr,                        //  tp_traverse
+    nullptr,                        //  tp_clear
+    nullptr,                        //  tp_richcompare
+    0,                              //  tp_weaklistoffset
+    nullptr,                        //  tp_iter
+    nullptr,                        //  tp_iternext
+    nullptr,                        //  tp_methods
+    nullptr,                        //  tp_members
+    nullptr,                        //  tp_getset
+    nullptr,                        //  tp_base
+    nullptr,                        //  tp_dict
+    nullptr,                        //  tp_descr_get
+    nullptr,                        //  tp_descr_set
+    0,                              //  tp_dictoffset
+    nullptr,                        //  tp_init
+    nullptr,                        //  tp_alloc
+    unknown_field_set::New,         //  tp_new
+};
+
+namespace unknown_field {
+static PyObject* PyUnknownFieldSet_FromUnknownFieldSet(
+    PyUnknownFieldSet* parent, const UnknownFieldSet& fields) {
+  PyUnknownFieldSet* self = reinterpret_cast<PyUnknownFieldSet*>(
+      PyType_GenericAlloc(&PyUnknownFieldSet_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  self->fields = const_cast<UnknownFieldSet*>(&fields);
+
+  return reinterpret_cast<PyObject*>(self);
+}
+
+const UnknownField* GetUnknownField(PyUnknownField* self) {
+  const UnknownFieldSet* fields = self->parent->fields;
+  if (fields == nullptr) {
+    PyErr_Format(PyExc_ValueError, "UnknownField does not exist. ");
+    return nullptr;
+  }
+  Py_ssize_t total_size = fields->field_count();
+  if (self->index >= total_size) {
+    PyErr_Format(PyExc_ValueError, "UnknownField does not exist. ");
+    return nullptr;
+  }
+  return &fields->field(self->index);
+}
+
+static PyObject* GetFieldNumber(PyUnknownField* self, void* closure) {
+  const UnknownField* unknown_field = GetUnknownField(self);
+  if (unknown_field == nullptr) {
+    return nullptr;
+  }
+  return PyLong_FromLong(unknown_field->number());
+}
+
+using internal::WireFormatLite;
+static PyObject* GetWireType(PyUnknownField* self, void* closure) {
+  const UnknownField* unknown_field = GetUnknownField(self);
+  if (unknown_field == nullptr) {
+    return nullptr;
+  }
+
+  // Assign a default value to suppress may-uninitialized warnings (errors
+  // when built in some places).
+  WireFormatLite::WireType wire_type = WireFormatLite::WIRETYPE_VARINT;
+  switch (unknown_field->type()) {
+    case UnknownField::TYPE_VARINT:
+      wire_type = WireFormatLite::WIRETYPE_VARINT;
+      break;
+    case UnknownField::TYPE_FIXED32:
+      wire_type = WireFormatLite::WIRETYPE_FIXED32;
+      break;
+    case UnknownField::TYPE_FIXED64:
+      wire_type = WireFormatLite::WIRETYPE_FIXED64;
+      break;
+    case UnknownField::TYPE_LENGTH_DELIMITED:
+      wire_type = WireFormatLite::WIRETYPE_LENGTH_DELIMITED;
+      break;
+    case UnknownField::TYPE_GROUP:
+      wire_type = WireFormatLite::WIRETYPE_START_GROUP;
+      break;
+  }
+  return PyLong_FromLong(wire_type);
+}
+
+static PyObject* GetData(PyUnknownField* self, void* closure) {
+  const UnknownField* field = GetUnknownField(self);
+  if (field == nullptr) {
+    return nullptr;
+  }
+  PyObject* data = nullptr;
+  switch (field->type()) {
+    case UnknownField::TYPE_VARINT:
+      data = PyLong_FromUnsignedLongLong(field->varint());
+      break;
+    case UnknownField::TYPE_FIXED32:
+      data = PyLong_FromUnsignedLong(field->fixed32());
+      break;
+    case UnknownField::TYPE_FIXED64:
+      data = PyLong_FromUnsignedLongLong(field->fixed64());
+      break;
+    case UnknownField::TYPE_LENGTH_DELIMITED:
+      data = PyBytes_FromStringAndSize(field->length_delimited().data(),
+                                       field->GetLengthDelimitedSize());
+      break;
+    case UnknownField::TYPE_GROUP:
+      data =
+          PyUnknownFieldSet_FromUnknownFieldSet(self->parent, field->group());
+      break;
+  }
+  return data;
+}
+
+static void Dealloc(PyObject* pself) {
+  PyUnknownField* self = reinterpret_cast<PyUnknownField*>(pself);
+  Py_CLEAR(self->parent);
+}
+
+static PyGetSetDef Getters[] = {
+    {"field_number", (getter)GetFieldNumber, nullptr},
+    {"wire_type", (getter)GetWireType, nullptr},
+    {"data", (getter)GetData, nullptr},
+    {nullptr},
+};
+
+}  // namespace unknown_field
+
+PyTypeObject PyUnknownField_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".PyUnknownField",       // tp_name
+    sizeof(PyUnknownField),  //  tp_basicsize
+    0,                       //  tp_itemsize
+    unknown_field::Dealloc,  //  tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                      //  tp_getattr
+    nullptr,                      //  tp_setattr
+    nullptr,                      //  tp_compare
+    nullptr,                      //  tp_repr
+    nullptr,                      //  tp_as_number
+    nullptr,                      //  tp_as_sequence
+    nullptr,                      //  tp_as_mapping
+    PyObject_HashNotImplemented,  //  tp_hash
+    nullptr,                      //  tp_call
+    nullptr,                      //  tp_str
+    nullptr,                      //  tp_getattro
+    nullptr,                      //  tp_setattro
+    nullptr,                      //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,           //  tp_flags
+    "unknown field",              //  tp_doc
+    nullptr,                      //  tp_traverse
+    nullptr,                      //  tp_clear
+    nullptr,                      //  tp_richcompare
+    0,                            //  tp_weaklistoffset
+    nullptr,                      //  tp_iter
+    nullptr,                      //  tp_iternext
+    nullptr,                      //  tp_methods
+    nullptr,                      //  tp_members
+    unknown_field::Getters,       //  tp_getset
+    nullptr,                      //  tp_base
+    nullptr,                      //  tp_dict
+    nullptr,                      //  tp_descr_get
+    nullptr,                      //  tp_descr_set
+    0,                            //  tp_dictoffset
+    nullptr,                      //  tp_init
+};
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/unknown_field_set.h

@@ -0,0 +1,78 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELD_SET_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELD_SET_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <memory>
+#include <set>
+
+#include "google/protobuf/pyext/message.h"
+
+namespace google {
+namespace protobuf {
+
+class UnknownField;
+class UnknownFieldSet;
+
+namespace python {
+struct CMessage;
+
+struct PyUnknownFieldSet {
+  PyObject_HEAD;
+  // If parent is nullptr, it is a top UnknownFieldSet.
+  PyUnknownFieldSet* parent;
+
+  // Top UnknownFieldSet owns fields pointer. Sub UnknownFieldSet
+  // does not own fields pointer.
+  UnknownFieldSet* fields;
+};
+
+struct PyUnknownField {
+  PyObject_HEAD;
+  // Every Python PyUnknownField holds a reference to its parent
+  // PyUnknownFieldSet in order to keep it alive.
+  PyUnknownFieldSet* parent;
+
+  // The UnknownField index in UnknownFieldSet.
+  Py_ssize_t index;
+};
+
+extern PyTypeObject PyUnknownFieldSet_Type;
+extern PyTypeObject PyUnknownField_Type;
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELD_SET_H__

python/google/protobuf/pyext/unknown_fields.cc

@@ -0,0 +1,363 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "google/protobuf/pyext/unknown_fields.h"
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <set>
+#include <memory>
+
+#include "google/protobuf/message.h"
+#include "google/protobuf/pyext/message.h"
+#include "google/protobuf/pyext/scoped_pyobject_ptr.h"
+#include "google/protobuf/unknown_field_set.h"
+#include "google/protobuf/wire_format_lite.h"
+
+namespace google {
+namespace protobuf {
+namespace python {
+
+namespace unknown_fields {
+
+static Py_ssize_t Len(PyObject* pself) {
+  PyUnknownFields* self =
+      reinterpret_cast<PyUnknownFields*>(pself);
+  if (self->fields == nullptr) {
+    PyErr_Format(PyExc_ValueError,
+                 "UnknownFields does not exist. "
+                 "The parent message might be cleared.");
+    return -1;
+  }
+  return self->fields->field_count();
+}
+
+void Clear(PyUnknownFields* self) {
+  for (std::set<PyUnknownFields*>::iterator it =
+           self->sub_unknown_fields.begin();
+       it != self->sub_unknown_fields.end(); it++) {
+    Clear(*it);
+  }
+  self->fields = nullptr;
+  self->sub_unknown_fields.clear();
+}
+
+PyObject* NewPyUnknownFieldRef(PyUnknownFields* parent,
+                               Py_ssize_t index);
+
+static PyObject* Item(PyObject* pself, Py_ssize_t index) {
+  PyUnknownFields* self =
+      reinterpret_cast<PyUnknownFields*>(pself);
+  if (self->fields == nullptr) {
+    PyErr_Format(PyExc_ValueError,
+                 "UnknownFields does not exist. "
+                 "The parent message might be cleared.");
+    return nullptr;
+  }
+  Py_ssize_t total_size = self->fields->field_count();
+  if (index < 0) {
+    index = total_size + index;
+  }
+  if (index < 0 || index >= total_size) {
+    PyErr_Format(PyExc_IndexError,
+                 "index (%zd) out of range",
+                 index);
+    return nullptr;
+  }
+
+  return unknown_fields::NewPyUnknownFieldRef(self, index);
+}
+
+PyObject* NewPyUnknownFields(CMessage* c_message) {
+  PyUnknownFields* self = reinterpret_cast<PyUnknownFields*>(
+      PyType_GenericAlloc(&PyUnknownFields_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+  // Call "placement new" to initialize PyUnknownFields.
+  new (self) PyUnknownFields;
+
+  Py_INCREF(c_message);
+  self->parent = reinterpret_cast<PyObject*>(c_message);
+  Message* message = c_message->message;
+  const Reflection* reflection = message->GetReflection();
+  self->fields = &reflection->GetUnknownFields(*message);
+
+  return reinterpret_cast<PyObject*>(self);
+}
+
+PyObject* NewPyUnknownFieldRef(PyUnknownFields* parent,
+                               Py_ssize_t index) {
+  PyUnknownFieldRef* self = reinterpret_cast<PyUnknownFieldRef*>(
+      PyType_GenericAlloc(&PyUnknownFieldRef_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+
+  Py_INCREF(parent);
+  self->parent = parent;
+  self->index = index;
+
+  return reinterpret_cast<PyObject*>(self);
+}
+
+static void Dealloc(PyObject* pself) {
+  PyUnknownFields* self =
+      reinterpret_cast<PyUnknownFields*>(pself);
+  if (PyObject_TypeCheck(self->parent, &PyUnknownFields_Type)) {
+    reinterpret_cast<PyUnknownFields*>(
+        self->parent)->sub_unknown_fields.erase(self);
+  } else {
+    reinterpret_cast<CMessage*>(self->parent)->unknown_field_set = nullptr;
+  }
+  Py_CLEAR(self->parent);
+  auto* py_type = Py_TYPE(pself);
+  self->~PyUnknownFields();
+  py_type->tp_free(pself);
+}
+
+static PySequenceMethods SqMethods = {
+    Len,     /* sq_length */
+    nullptr, /* sq_concat */
+    nullptr, /* sq_repeat */
+    Item,    /* sq_item */
+    nullptr, /* sq_slice */
+    nullptr, /* sq_ass_item */
+};
+
+}  // namespace unknown_fields
+
+PyTypeObject PyUnknownFields_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".PyUnknownFields",       // tp_name
+    sizeof(PyUnknownFields),  // tp_basicsize
+    0,                        //  tp_itemsize
+    unknown_fields::Dealloc,  //  tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                      //  tp_getattr
+    nullptr,                      //  tp_setattr
+    nullptr,                      //  tp_compare
+    nullptr,                      //  tp_repr
+    nullptr,                      //  tp_as_number
+    &unknown_fields::SqMethods,   //  tp_as_sequence
+    nullptr,                      //  tp_as_mapping
+    PyObject_HashNotImplemented,  //  tp_hash
+    nullptr,                      //  tp_call
+    nullptr,                      //  tp_str
+    nullptr,                      //  tp_getattro
+    nullptr,                      //  tp_setattro
+    nullptr,                      //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,           //  tp_flags
+    "unknown field set",          //  tp_doc
+    nullptr,                      //  tp_traverse
+    nullptr,                      //  tp_clear
+    nullptr,                      //  tp_richcompare
+    0,                            //  tp_weaklistoffset
+    nullptr,                      //  tp_iter
+    nullptr,                      //  tp_iternext
+    nullptr,                      //  tp_methods
+    nullptr,                      //  tp_members
+    nullptr,                      //  tp_getset
+    nullptr,                      //  tp_base
+    nullptr,                      //  tp_dict
+    nullptr,                      //  tp_descr_get
+    nullptr,                      //  tp_descr_set
+    0,                            //  tp_dictoffset
+    nullptr,                      //  tp_init
+};
+
+namespace unknown_field {
+static PyObject* PyUnknownFields_FromUnknownFieldSet(
+    PyUnknownFields* parent, const UnknownFieldSet& fields) {
+  PyUnknownFields* self = reinterpret_cast<PyUnknownFields*>(
+      PyType_GenericAlloc(&PyUnknownFields_Type, 0));
+  if (self == nullptr) {
+    return nullptr;
+  }
+  // Call "placement new" to initialize PyUnknownFields.
+  new (self) PyUnknownFields;
+
+  Py_INCREF(parent);
+  self->parent = reinterpret_cast<PyObject*>(parent);
+  self->fields = &fields;
+  parent->sub_unknown_fields.emplace(self);
+
+  return reinterpret_cast<PyObject*>(self);
+}
+
+const UnknownField* GetUnknownField(PyUnknownFieldRef* self) {
+  const UnknownFieldSet* fields = self->parent->fields;
+  if (fields == nullptr) {
+    PyErr_Format(PyExc_ValueError,
+                 "UnknownField does not exist. "
+                 "The parent message might be cleared.");
+    return nullptr;
+  }
+  Py_ssize_t total_size = fields->field_count();
+  if (self->index >= total_size) {
+    PyErr_Format(PyExc_ValueError,
+                 "UnknownField does not exist. "
+                 "The parent message might be cleared.");
+    return nullptr;
+  }
+  return &fields->field(self->index);
+}
+
+static PyObject* GetFieldNumber(PyUnknownFieldRef* self, void *closure) {
+  const UnknownField* unknown_field = GetUnknownField(self);
+  if (unknown_field == nullptr) {
+    return nullptr;
+  }
+  return PyLong_FromLong(unknown_field->number());
+}
+
+using internal::WireFormatLite;
+static PyObject* GetWireType(PyUnknownFieldRef* self, void *closure) {
+  const UnknownField* unknown_field = GetUnknownField(self);
+  if (unknown_field == nullptr) {
+    return nullptr;
+  }
+
+  // Assign a default value to suppress may-uninitialized warnings (errors
+  // when built in some places).
+  WireFormatLite::WireType wire_type = WireFormatLite::WIRETYPE_VARINT;
+  switch (unknown_field->type()) {
+    case UnknownField::TYPE_VARINT:
+      wire_type = WireFormatLite::WIRETYPE_VARINT;
+      break;
+    case UnknownField::TYPE_FIXED32:
+      wire_type = WireFormatLite::WIRETYPE_FIXED32;
+      break;
+    case UnknownField::TYPE_FIXED64:
+      wire_type = WireFormatLite::WIRETYPE_FIXED64;
+      break;
+    case UnknownField::TYPE_LENGTH_DELIMITED:
+      wire_type = WireFormatLite::WIRETYPE_LENGTH_DELIMITED;
+      break;
+    case UnknownField::TYPE_GROUP:
+      wire_type = WireFormatLite::WIRETYPE_START_GROUP;
+      break;
+  }
+  return PyLong_FromLong(wire_type);
+}
+
+static PyObject* GetData(PyUnknownFieldRef* self, void *closure) {
+  const UnknownField* field = GetUnknownField(self);
+  if (field == nullptr) {
+    return nullptr;
+  }
+  PyObject* data = nullptr;
+  switch (field->type()) {
+    case UnknownField::TYPE_VARINT:
+      data = PyLong_FromUnsignedLongLong(field->varint());
+      break;
+    case UnknownField::TYPE_FIXED32:
+      data = PyLong_FromUnsignedLong(field->fixed32());
+      break;
+    case UnknownField::TYPE_FIXED64:
+      data = PyLong_FromUnsignedLongLong(field->fixed64());
+      break;
+    case UnknownField::TYPE_LENGTH_DELIMITED:
+      data = PyBytes_FromStringAndSize(field->length_delimited().data(),
+                                       field->GetLengthDelimitedSize());
+      break;
+    case UnknownField::TYPE_GROUP:
+      data = PyUnknownFields_FromUnknownFieldSet(
+          self->parent, field->group());
+      break;
+  }
+  return data;
+}
+
+static void Dealloc(PyObject* pself) {
+  PyUnknownFieldRef* self =
+      reinterpret_cast<PyUnknownFieldRef*>(pself);
+  Py_CLEAR(self->parent);
+}
+
+static PyGetSetDef Getters[] = {
+    {"field_number", (getter)GetFieldNumber, nullptr},
+    {"wire_type", (getter)GetWireType, nullptr},
+    {"data", (getter)GetData, nullptr},
+    {nullptr},
+};
+
+}  // namespace unknown_field
+
+PyTypeObject PyUnknownFieldRef_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
+    ".PyUnknownFieldRef",       // tp_name
+    sizeof(PyUnknownFieldRef),  //  tp_basicsize
+    0,                          //  tp_itemsize
+    unknown_field::Dealloc,     //  tp_dealloc
+#if PY_VERSION_HEX < 0x03080000
+    nullptr,  // tp_print
+#else
+    0,  // tp_vectorcall_offset
+#endif
+    nullptr,                      //  tp_getattr
+    nullptr,                      //  tp_setattr
+    nullptr,                      //  tp_compare
+    nullptr,                      //  tp_repr
+    nullptr,                      //  tp_as_number
+    nullptr,                      //  tp_as_sequence
+    nullptr,                      //  tp_as_mapping
+    PyObject_HashNotImplemented,  //  tp_hash
+    nullptr,                      //  tp_call
+    nullptr,                      //  tp_str
+    nullptr,                      //  tp_getattro
+    nullptr,                      //  tp_setattro
+    nullptr,                      //  tp_as_buffer
+    Py_TPFLAGS_DEFAULT,           //  tp_flags
+    "unknown field",              //  tp_doc
+    nullptr,                      //  tp_traverse
+    nullptr,                      //  tp_clear
+    nullptr,                      //  tp_richcompare
+    0,                            //  tp_weaklistoffset
+    nullptr,                      //  tp_iter
+    nullptr,                      //  tp_iternext
+    nullptr,                      //  tp_methods
+    nullptr,                      //  tp_members
+    unknown_field::Getters,       //  tp_getset
+    nullptr,                      //  tp_base
+    nullptr,                      //  tp_dict
+    nullptr,                      //  tp_descr_get
+    nullptr,                      //  tp_descr_set
+    0,                            //  tp_dictoffset
+    nullptr,                      //  tp_init
+};
+
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google

python/google/protobuf/pyext/unknown_fields.h

@@ -0,0 +1,91 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__
+#define GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__
+
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <memory>
+#include <set>
+
+#include "google/protobuf/pyext/message.h"
+
+namespace google {
+namespace protobuf {
+
+class UnknownField;
+class UnknownFieldSet;
+
+namespace python {
+struct CMessage;
+
+typedef struct PyUnknownFields {
+  PyObject_HEAD;
+  // Strong pointer to the parent CMessage or PyUnknownFields.
+  // The top PyUnknownFields holds a reference to its parent CMessage
+  // object before release.
+  // Sub PyUnknownFields holds reference to parent PyUnknownFields.
+  PyObject* parent;
+
+  // Pointer to the C++ UnknownFieldSet.
+  // PyUnknownFields does not own this pointer.
+  const UnknownFieldSet* fields;
+
+  // Weak references to child unknown fields.
+  std::set<PyUnknownFields*> sub_unknown_fields;
+} PyUnknownFields;
+
+typedef struct PyUnknownFieldRef {
+  PyObject_HEAD;
+  // Every Python PyUnknownFieldRef holds a reference to its parent
+  // PyUnknownFields in order to keep it alive.
+  PyUnknownFields* parent;
+
+  // The UnknownField index in UnknownFields.
+  Py_ssize_t index;
+} UknownFieldRef;
+
+extern PyTypeObject PyUnknownFields_Type;
+extern PyTypeObject PyUnknownFieldRef_Type;
+
+namespace unknown_fields {
+
+// Builds an PyUnknownFields for a specific message.
+PyObject* NewPyUnknownFields(CMessage *parent);
+void Clear(PyUnknownFields* self);
+
+}  // namespace unknown_fields
+}  // namespace python
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__