Merge commit '36bca61764984ff5395653cf8377ec5daa71b709' as 'libs/protobuf'

libs/protobuf/kokoro/linux/aarch64/cpp_crosscompile_and_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator.
+# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image.
+
+set -ex
+
+# the build commands are expected to run under dockcross docker image
+# where the CC, CXX and other toolchain variables already point to the crosscompiler
+cmake .
+make -j8
+
+# check that the resulting test binary is indeed an aarch64 ELF
+(file ./tests | grep -q "ELF 64-bit LSB executable, ARM aarch64") || (echo "Test binary in not an aarch64 binary"; exit 1)
+
+# run the basic set of C++ tests under QEMU
+# there are other tests we could run (e.g. ./lite-test), but this is sufficient as a smoketest
+qemu-aarch64 ./tests

libs/protobuf/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_linux_aarch64.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# Pin the dockcross image since newer versions of the image break the build
+PINNED_DOCKCROSS_IMAGE_VERSION=dockcross/linux-arm64:20210625-795dd4d
+
+# running dockcross image without any arguments generates a wrapper
+# scripts that can be used to run commands under the dockcross image
+# easily.
+# See https://github.com/dockcross/dockcross#usage for details
+docker run $DOCKER_TTY_ARGS --rm $PINNED_DOCKCROSS_IMAGE_VERSION >dockcross-linux-arm64.sh
+chmod +x dockcross-linux-arm64.sh
+
+# the wrapper script has CRLF line endings and bash doesn't like that
+# so we change CRLF line endings into LF.
+sed -i 's/\r//g' dockcross-linux-arm64.sh
+
+# The dockcross wrapper script runs arbitrary commands under the selected dockcross
+# image with the following properties which make its use very convenient:
+# * the current working directory is mounted under /work so the container can easily
+#   access the current workspace
+# * the processes in the container run under the same UID and GID as the host process so unlike
+#   vanilla "docker run" invocations, the workspace doesn't get polluted with files
+#   owned by root.
+./dockcross-linux-arm64.sh --image $PINNED_DOCKCROSS_IMAGE_VERSION -- "$@"

libs/protobuf/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# Pin the dockcross image since newer versions of the image break the build
+# We use an older version of dockcross image that has gcc4.9.4 because it was built
+# before https://github.com/dockcross/dockcross/pull/449
+# Thanks to that, wheel build with this image aren't actually
+# compliant with manylinux2014, but only with manylinux_2_24
+PINNED_DOCKCROSS_IMAGE_VERSION=dockcross/manylinux2014-aarch64:20200929-608e6ac
+
+# running dockcross image without any arguments generates a wrapper
+# scripts that can be used to run commands under the dockcross image
+# easily.
+# See https://github.com/dockcross/dockcross#usage for details
+docker run $DOCKER_TTY_ARGS --rm $PINNED_DOCKCROSS_IMAGE_VERSION >dockcross-manylinux2014-aarch64.sh
+chmod +x dockcross-manylinux2014-aarch64.sh
+
+# the wrapper script has CRLF line endings and bash doesn't like that
+# so we change CRLF line endings into LF.
+sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh
+
+# The dockcross wrapper script runs arbitrary commands under the selected dockcross
+# image with the following properties which make its use very convenient:
+# * the current working directory is mounted under /work so the container can easily
+#   access the current workspace
+# * the processes in the container run under the same UID and GID as the host process so unlike
+#   vanilla "docker run" invocations, the workspace doesn't get polluted with files
+#   owned by root.
+./dockcross-manylinux2014-aarch64.sh --image $PINNED_DOCKCROSS_IMAGE_VERSION -- "$@"

libs/protobuf/kokoro/linux/aarch64/php_build_and_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -ex
+
+# Install composer
+curl -sS https://getcomposer.org/installer | php
+mkdir -p "$HOME/bin"
+mv composer.phar "$HOME/bin/composer"
+PATH="$HOME/bin:$PATH"
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+cd php
+
+composer install
+composer test
+composer test_c

libs/protobuf/kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Builds protobuf C++ with aarch64 crosscompiler.
+
+set -ex
+
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_WITH_ZLIB=0 -Dprotobuf_BUILD_TESTS=OFF .
+make -j8

libs/protobuf/kokoro/linux/aarch64/python_crosscompile_aarch64.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Builds protobuf python including the C++ extension with aarch64 crosscompiler.
+# The outputs of this script are laid out so that we can later test them under an aarch64 emulator.
+# NOTE: This script is expected to run under the dockcross/manylinux2014-aarch64 docker image.
+
+set -ex
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+# Build protoc and libprotobuf
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_WITH_ZLIB=0 -Dprotobuf_BUILD_TESTS=OFF .
+make -j8
+
+# create a simple shell wrapper that runs crosscompiled protoc under qemu
+echo '#!/bin/bash' >protoc_qemu_wrapper.sh
+echo 'exec qemu-aarch64 "../protoc" "$@"' >>protoc_qemu_wrapper.sh
+chmod ugo+x protoc_qemu_wrapper.sh
+
+# PROTOC variable is by build_py step that runs under ./python directory
+export PROTOC=../protoc_qemu_wrapper.sh
+
+pushd python
+
+# NOTE: this step will use protoc_qemu_wrapper.sh to generate protobuf files.
+${PYTHON} setup.py build_py
+
+# when crosscompiling for aarch64, --plat-name needs to be set explicitly
+# to end up with correctly named wheel file
+# the value should be manylinuxABC_ARCH and dockcross docker image
+# conveniently provides the value in the AUDITWHEEL_PLAT env
+plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
+
+# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
+export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
+
+# Build the python extension inplace to be able to python unittests later
+${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace
+
+# Build the binary wheel (to check it with auditwheel)
+${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag

libs/protobuf/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+cd python
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+${PYTHON} -m pip install --user pytest auditwheel numpy
+
+# check that we are really using aarch64 python
+(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1)
+
+# step 1: run all python unittests
+# we've built the python extension previously with --inplace option
+# so we can just discover all the unittests and run them directly under 
+# the python/ directory.
+LD_LIBRARY_PATH=. PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf
+
+# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible.
+# auditwheel needs to run on wheel's target platform (or under an emulator)
+${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl
+
+# step 3: smoketest that the wheel can be installed and run a smokecheck
+${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl
+# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfully loaded the python cpp extension!")'

libs/protobuf/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running
+# inside docker containers.
+
+set -ex
+
+# show pre-existing qemu registration
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+
+# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that:
+# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278")
+# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below)
+#
+# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static
+# docker image to provide a new enough version of qemu-user-static and register it with
+# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
+
+# Print current qemu reqistration to make sure everything is setup correctly.
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64

libs/protobuf/kokoro/linux/aarch64/ruby_build_and_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+gem install bundler
+
+cd ruby
+
+bundle
+rake
+rake clobber_package gem
+
+# run all the tests
+rake test

libs/protobuf/kokoro/linux/aarch64/test_cpp_aarch64.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Crosscompiles protobuf C++ under dockcross docker image and runs the tests under an emulator.
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+# run the C++ build and test script under dockcross/linux-arm64 image
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_linux_aarch64.sh kokoro/linux/aarch64/cpp_crosscompile_and_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/test_csharp_aarch64.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# First, build protobuf C# tests under x86_64 docker image
+# Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory
+# (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run)
+CSHARP_BUILD_COMMAND="dotnet publish -c Release -f net60 csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj"
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim bash -c "$CSHARP_BUILD_COMMAND"
+
+# Use an actual aarch64 docker image to run protobuf C# tests with an emulator. "dotnet vstest" allows
+# running tests from a pre-built project.
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+CSHARP_TEST_COMMAND="dotnet vstest csharp/src/Google.Protobuf.Test/bin/Release/net60/publish/Google.Protobuf.Test.dll"
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8 bash -c "$CSHARP_TEST_COMMAND"

libs/protobuf/kokoro/linux/aarch64/test_java_aarch64.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile protoc as we will later need it for the java build.
+# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
+# with a wide range of linux distros (including any docker images we will use later to build and test java)
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
+
+# the command that will be used to build and test java under an emulator
+# * IsValidUtf8Test and DecodeUtf8Test tests are skipped because they take very long under an emulator.
+TEST_JAVA_COMMAND="mvn --batch-mode -DskipTests install && mvn --batch-mode -Dtest='**/*Test, !**/*IsValidUtf8Test, !**/*DecodeUtf8Test' -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false surefire:test"
+
+# use an actual aarch64 docker image (with a real aarch64 java and maven) to run build & test protobuf java under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+# * the JAVA_OPTS and MAVEN_CONFIG variables are being set mostly to silence warnings about non-existent home directory
+#   and to avoid polluting the workspace.
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "JAVA_OPTS=-Duser.home=/home/fake-user" -e "MAVEN_CONFIG=/home/fake-user/.m2" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/maven:3.8-openjdk-11 bash -c "cd java && $TEST_JAVA_COMMAND"

libs/protobuf/kokoro/linux/aarch64/test_php_aarch64.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+# there is no php testing docker image readily available, so we build
+# our own. It's a aarch64 image, but that's fine since qemu will
+# automatically be used to run the commands in the dockerfile.
+docker build -t testimage_protobuf_php_arm64v8 kokoro/linux/aarch64/testimage_protobuf_php_arm64v8
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile protoc as we will later need it for the php build.
+# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
+# with a wide range of linux distros (including any docker images we will use later to build and test php)
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
+
+# use an actual aarch64 docker image (with a real aarch64 php) to run build & test protobuf php under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work testimage_protobuf_php_arm64v8 kokoro/linux/aarch64/php_build_and_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/test_python_aarch64.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh
+
+# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution
+#   (manylinux_2_24 is debian-based while manylinux2014 is centos-based)
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/test_ruby_aarch64.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile protoc as we will later need it for the ruby build.
+# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
+# with a wide range of linux distros (including any docker images we will use later to build and test ruby)
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
+
+# use an actual aarch64 docker image (with a real aarch64 ruby) to run build & test protobuf ruby under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "PROTOC=/work/protoc" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/ruby:2.7.3-buster kokoro/linux/aarch64/ruby_build_and_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/testimage_protobuf_php_arm64v8/Dockerfile

@@ -0,0 +1,3 @@
+FROM arm64v8/debian:buster
+
+RUN apt-get update && apt-get install -y php7.3-cli php7.3-dev php7.3-bcmath composer phpunit curl git valgrind && apt-get clean