Merge commit '36bca61764984ff5395653cf8377ec5daa71b709' as 'libs/protobuf'

libs/protobuf/kokoro/README.md

@@ -0,0 +1,12 @@
+
+Kokoro Infrastructure
+----------------------
+
+The files in this directory serve as plumbing for running Protobuf
+tests under Kokoro, our internal CI.
+
+We have shared this part of our CI configuration in hopes that it is
+helpful to contributors who want to better understand the details of
+our test and release processes. If there are changes, please file an
+issue; unfortunately, we may not be able to accept PRs (but feel free
+to send one if it helps to explain the issue).

libs/protobuf/kokoro/common/bazel_flags.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Helper for setting up common bazel flags in Kokoro.
+#
+# This script prints extra flags to a bazel invocation when it is run from
+# Kokoro.  When the special environment variables are not present (e.g., if you
+# run Kokoro build scripts locally), this script only flips some debug settings.
+#
+# Example of running directly:
+#   bazel test $(path/to/bazel_flags.sh) //...
+
+function bazel_flags::gen_invocation_id() {
+  # Create a new invocation ID and store in the artifacts dir.
+  local _invocation_id=$(uuidgen | tr A-Z a-z)
+
+  # Put the new invocation ID at the start of the output IDs file. Some
+  # Google-internal tools only look at the first entry, so this ensures the most
+  # recent entry is first.
+  local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids
+  local _temp_ids=$(mktemp)
+  echo ${_invocation_id} > ${_temp_ids}
+  [[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids}
+  mv -f ${_temp_ids} ${_ids_file}
+
+  echo -n ${_invocation_id}
+}
+
+# Prints flags to use on Kokoro.
+function bazel_flags::kokoro_flags() {
+  [[ -n ${KOKORO_JOB_NAME:-} ]] || return
+
+  local -a _flags
+  _flags+=(
+    --invocation_id=$(bazel_flags::gen_invocation_id)
+    --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/${KOKORO_JOB_NAME}
+  )
+  if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then
+    _flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} )
+  else
+    _flags+=( --google_default_credentials=true )
+  fi
+
+  echo "${_flags[@]}"
+}
+
+echo "$(bazel_flags::kokoro_flags) --keep_going --test_output=errors"

libs/protobuf/kokoro/common/caplog.sh

@@ -0,0 +1,92 @@
+# Log capturing for the Kokoro runtime environment.
+#
+# This script should be `source`d from Kokoro build scripts to configure log
+# capturing when running under Kokoro.
+#
+# When not running under Kokoro, no logs will be collected. If you want to run
+# locally and collect logs anyway, set the KOKORO_ARTIFACTS_DIR environment
+# variable to a directory where the logs should go.
+#
+# The job `.cfg` file needs the following stanzas to declare the captured logs
+# as outputs (yes, these are globs, not regexes):
+#
+#   action: {
+#     define_artifacts: {
+#       regex: "**/*sponge_log.log"
+#       regex: "**/*sponge_log.xml"
+#     }
+#   }
+#
+# Use the provided functions below as build/test fixtures, e.g.:
+#
+#   source kokoro/common/capture_logs.sh
+#   caplog build/step1 <build command>
+#   caplog tests/step2 <test command>
+#
+# If log capturing is enabled, this script will set some variables that can be
+# used if necessary:
+#
+#   CAPLOG_DIR         is used for logs
+#   CAPLOG_CMAKE_ARGS  contains extra cmake args to enable test XML output
+#   CAPLOG_CTEST_ARGS  contains extra ctest args to capture combined test logs
+#
+# For example:
+#
+#   if [[ -v CAPLOG_DIR_BUILD ]]; then
+#     cp extra_diagnostics.log "${CAPLOG_DIR_BUILD}/diagnostics.log"
+#   fi
+#
+#   # Use ${...:-} form under `set -u`:
+#   caplog build/01_configure cmake -G Ninja ${CAPLOG_CMAKE_ARGS:-}
+#   caplog build/02_build     cmake --build
+#   caplog test/03_test       ctest ${CAPLOG_CTEST_ARGS:-}
+
+if [[ -z ${KOKORO_ARTIFACTS_DIR:-} ]]; then
+  function caplog() { shift; "$@"; }
+else
+
+  CAPLOG_DIR="$(mktemp -d)"
+  CAPLOG_CMAKE_ARGS="-Dprotobuf_TEST_XML_OUTDIR=${CAPLOG_DIR}/tests/"
+  CAPLOG_CTEST_ARGS="--verbose"
+
+  # Captures the stdout/stderr of a command to a named log file.
+  # Usage: caplog NAME COMMAND [ARGS...]
+  function caplog() {
+    _name="${CAPLOG_DIR}/${1%.log}.log"; shift
+    mkdir -p "${_name%/*}"
+    date
+    time ( "$@" 2>&1 | tee "${_name}" )
+    if [[ $? != 0 ]] ; then
+      cat "${_name}"
+      return 1
+    fi
+  }
+
+  # Trap handler: renames logs on script exit so they will be found by Kokoro.
+  function _caplog_onexit() {
+    _rc=$?
+    set +x
+    echo "Collecting logs [${BASH_SOURCE}]"
+
+    find "${CAPLOG_DIR}" -type f -name '*.log' \
+      | while read _textlog; do
+      # Ensure an XML file exists for each .log file.
+      touch ${_textlog%.log}.xml
+    done
+
+    find "${CAPLOG_DIR}" -type f \( -name '*.xml' -or -name '*.log' \) \
+      | while read _src; do
+      # Move to artifacts dir, preserving the path relative to CAPLOG_DIR.
+      # The filename changes from foo/bar.log to foo/bar/sponge_log.log.
+      _logfile=${_src/${CAPLOG_DIR}\//}
+      _stem=${KOKORO_ARTIFACTS_DIR}/${_logfile%.*}
+      _ext=${_logfile##*.}
+      mkdir -p ${_stem}
+      mv -v "${_src}" "${_stem}/sponge_log.${_ext}"
+    done
+    rm -rv "${CAPLOG_DIR}"
+    exit ${_rc}
+  }
+  trap _caplog_onexit EXIT
+
+fi

libs/protobuf/kokoro/common/cmake.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# Build tests under CMake.
+#
+# This script is used from macos and linux builds. It runs cmake and ctest in
+# the current directory. Any additional setup should be done before running this
+# script.
+#
+# This script uses `caplog` to save logfiles. See caplog.sh for details.
+
+set -eu -o pipefail
+: ${SCRIPT_ROOT:=$(cd $(dirname $0)/../..; pwd)}
+
+################################################################################
+# If you are using this script to run tests, you can set some environment
+# variables to control behavior:
+#
+# By default, find the sources based on this script's path.
+: ${SOURCE_DIR:=${SCRIPT_ROOT}}
+#
+# By default, put outputs under <git root>/cmake/build.
+: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build}
+#
+# CMAKE_BUILD_TYPE is supported in cmake 3.22+. If set, we pass the value of this
+# variable explicitly for compatibility with older versions of cmake. See:
+#   https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
+# (N.B.: not to be confused with CMAKE_CONFIG_TYPE.)
+if [[ -n ${CMAKE_BUILD_TYPE:-} ]]; then
+  CMAKE_BUILD_TYPE_FLAG="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+else
+  CMAKE_BUILD_TYPE_FLAG=
+fi
+#
+# For several other CMake options, see docs here:
+#   https://cmake.org/cmake/help/latest/manual/cmake-env-variables.7.html
+#
+# Some variables you may want to override (see cmake docs for details):
+#   CMAKE_BUILD_PARALLEL_LEVEL
+#   CMAKE_CONFIG_TYPE (N.B.: not to be confused with CMAKE_BUILD_TYPE)
+#   CMAKE_GENERATOR
+#   CTEST_PARALLEL_LEVEL
+################################################################################
+
+echo "Building using..."
+echo "       Sources: ${SOURCE_DIR}"
+echo "  Build output: ${BUILD_DIR}"
+if [[ ${SOURCE_DIR} != ${SCRIPT_ROOT} ]]; then
+  echo " Build scripts: ${SCRIPT_ROOT}"
+fi
+set -x
+source ${SCRIPT_ROOT}/kokoro/common/caplog.sh
+
+#
+# Configure under $BUILD_DIR:
+#
+mkdir -p "${BUILD_DIR}"
+
+(
+  cd "${BUILD_DIR}"
+  caplog 01_configure \
+    cmake -S "${SOURCE_DIR}" \
+    ${CMAKE_BUILD_TYPE_FLAG} \
+    ${CAPLOG_CMAKE_ARGS:-}
+)
+if [[ -n ${CAPLOG_DIR:-} ]]; then
+  # Save configuration logs.
+  mkdir -p "${CAPLOG_DIR}/CMakeFiles"
+  cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles"
+fi
+
+#
+# Build:
+#
+caplog 02_build \
+  cmake --build "${BUILD_DIR}"
+
+#
+# Run tests
+#
+(
+  cd "${BUILD_DIR}"
+  caplog 03_combined_testlog \
+    ctest ${CAPLOG_CTEST_ARGS:-}
+)

libs/protobuf/kokoro/docs/common.cfg

@@ -0,0 +1,44 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "protobuf/kokoro/docs/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+
+env_vars: {
+    key: "STAGING_BUCKET"
+    value: "docs-staging"
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "yoshi-automation-github-key"
+    }
+  }
+}
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "docuploader_service_account"
+    }
+  }
+}

libs/protobuf/kokoro/docs/publish-python.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Adapted from https://github.com/googleapis/google-cloud-python/blob/main/.kokoro/publish-docs.sh
+
+set -eo pipefail
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+cd github/protobuf/python
+
+# install package
+sudo apt-get update
+sudo apt-get -y install software-properties-common
+sudo add-apt-repository universe
+sudo apt-get update
+sudo apt-get -y install unzip
+wget https://github.com/protocolbuffers/protobuf/releases/download/v21.1/protoc-21.1-linux-x86_64.zip
+unzip protoc-21.1-linux-x86_64.zip bin/protoc
+mv bin/protoc ../protoc
+python3 -m venv venv
+source venv/bin/activate
+python setup.py install
+
+# install docs dependencies
+python -m pip install -r docs/requirements.txt
+
+# build docs
+cd docs
+make html
+cd ..
+deactivate
+
+python3 -m pip install protobuf==4.21.1 gcp-docuploader==0.6.3
+
+# install a json parser
+sudo apt-get -y install jq
+
+# create metadata
+python3 -m docuploader create-metadata \
+  --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
+  --version=$(python3 setup.py --version) \
+  --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
+  --distribution-name=$(python3 setup.py --name) \
+  --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
+  --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
+  --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
+
+cat docs.metadata
+
+# upload docs
+python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging

libs/protobuf/kokoro/docs/python.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Tell the trampoline which build file to use.
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/protobuf/kokoro/docs/publish-python.sh"
+}

libs/protobuf/kokoro/docs/trampoline.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Copied from https://github.com/googleapis/google-cloud-python/blob/main/.kokoro/trampoline.sh
+
+set -eo pipefail
+
+python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"  || ret_code=$?
+
+chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
+${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true
+
+exit ${ret_code}

libs/protobuf/kokoro/linux/32-bit/build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request 32" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.
+
+set -ex
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=$(pwd)
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/php/32bit@sha256:824cbdff02ee543eb69ee4b02c8c58cc7887f70f49e41725a35765d92a898b4f
+
+git submodule update --init --recursive
+
+docker run \
+  "$@" \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  bash -l "/workspace/kokoro/linux/32-bit/test_php.sh"

libs/protobuf/kokoro/linux/32-bit/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/32-bit/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

libs/protobuf/kokoro/linux/32-bit/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/32-bit/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/32-bit/test_php.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+use_php() {
+  VERSION=$1
+  export PATH=/usr/local/php-${VERSION}/bin:$PATH
+}
+
+build_php() {
+  use_php $1
+  pushd php
+  rm -rf vendor
+  php -v
+  php -m
+  composer update
+  composer test
+  popd
+}
+
+test_php_c() {
+  pushd php
+  rm -rf vendor
+  php -v
+  php -m
+  composer update
+  composer test_c
+  popd
+}
+
+build_php_c() {
+  use_php $1
+  test_php_c
+}
+
+mkdir -p build
+pushd build
+cmake ..
+cmake --build . -- -j20
+ctest --verbose --parallel 20
+export PROTOC=$(pwd)/protoc
+popd
+
+build_php 7.0
+build_php 7.1
+build_php 7.4
+build_php_c 7.0
+build_php_c 7.1
+build_php_c 7.4
+build_php_c 7.1-zts
+build_php_c 7.2-zts
+build_php_c 7.5-zts
+
+# Cleanup after CMake build
+rm -rf build

libs/protobuf/kokoro/linux/aarch64/cpp_crosscompile_and_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator.
+# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image.
+
+set -ex
+
+# the build commands are expected to run under dockcross docker image
+# where the CC, CXX and other toolchain variables already point to the crosscompiler
+cmake .
+make -j8
+
+# check that the resulting test binary is indeed an aarch64 ELF
+(file ./tests | grep -q "ELF 64-bit LSB executable, ARM aarch64") || (echo "Test binary in not an aarch64 binary"; exit 1)
+
+# run the basic set of C++ tests under QEMU
+# there are other tests we could run (e.g. ./lite-test), but this is sufficient as a smoketest
+qemu-aarch64 ./tests

libs/protobuf/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_linux_aarch64.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# Pin the dockcross image since newer versions of the image break the build
+PINNED_DOCKCROSS_IMAGE_VERSION=dockcross/linux-arm64:20210625-795dd4d
+
+# running dockcross image without any arguments generates a wrapper
+# scripts that can be used to run commands under the dockcross image
+# easily.
+# See https://github.com/dockcross/dockcross#usage for details
+docker run $DOCKER_TTY_ARGS --rm $PINNED_DOCKCROSS_IMAGE_VERSION >dockcross-linux-arm64.sh
+chmod +x dockcross-linux-arm64.sh
+
+# the wrapper script has CRLF line endings and bash doesn't like that
+# so we change CRLF line endings into LF.
+sed -i 's/\r//g' dockcross-linux-arm64.sh
+
+# The dockcross wrapper script runs arbitrary commands under the selected dockcross
+# image with the following properties which make its use very convenient:
+# * the current working directory is mounted under /work so the container can easily
+#   access the current workspace
+# * the processes in the container run under the same UID and GID as the host process so unlike
+#   vanilla "docker run" invocations, the workspace doesn't get polluted with files
+#   owned by root.
+./dockcross-linux-arm64.sh --image $PINNED_DOCKCROSS_IMAGE_VERSION -- "$@"

libs/protobuf/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# Pin the dockcross image since newer versions of the image break the build
+# We use an older version of dockcross image that has gcc4.9.4 because it was built
+# before https://github.com/dockcross/dockcross/pull/449
+# Thanks to that, wheel build with this image aren't actually
+# compliant with manylinux2014, but only with manylinux_2_24
+PINNED_DOCKCROSS_IMAGE_VERSION=dockcross/manylinux2014-aarch64:20200929-608e6ac
+
+# running dockcross image without any arguments generates a wrapper
+# scripts that can be used to run commands under the dockcross image
+# easily.
+# See https://github.com/dockcross/dockcross#usage for details
+docker run $DOCKER_TTY_ARGS --rm $PINNED_DOCKCROSS_IMAGE_VERSION >dockcross-manylinux2014-aarch64.sh
+chmod +x dockcross-manylinux2014-aarch64.sh
+
+# the wrapper script has CRLF line endings and bash doesn't like that
+# so we change CRLF line endings into LF.
+sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh
+
+# The dockcross wrapper script runs arbitrary commands under the selected dockcross
+# image with the following properties which make its use very convenient:
+# * the current working directory is mounted under /work so the container can easily
+#   access the current workspace
+# * the processes in the container run under the same UID and GID as the host process so unlike
+#   vanilla "docker run" invocations, the workspace doesn't get polluted with files
+#   owned by root.
+./dockcross-manylinux2014-aarch64.sh --image $PINNED_DOCKCROSS_IMAGE_VERSION -- "$@"

libs/protobuf/kokoro/linux/aarch64/php_build_and_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -ex
+
+# Install composer
+curl -sS https://getcomposer.org/installer | php
+mkdir -p "$HOME/bin"
+mv composer.phar "$HOME/bin/composer"
+PATH="$HOME/bin:$PATH"
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+cd php
+
+composer install
+composer test
+composer test_c

libs/protobuf/kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Builds protobuf C++ with aarch64 crosscompiler.
+
+set -ex
+
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_WITH_ZLIB=0 -Dprotobuf_BUILD_TESTS=OFF .
+make -j8

libs/protobuf/kokoro/linux/aarch64/python_crosscompile_aarch64.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Builds protobuf python including the C++ extension with aarch64 crosscompiler.
+# The outputs of this script are laid out so that we can later test them under an aarch64 emulator.
+# NOTE: This script is expected to run under the dockcross/manylinux2014-aarch64 docker image.
+
+set -ex
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+# Build protoc and libprotobuf
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_WITH_ZLIB=0 -Dprotobuf_BUILD_TESTS=OFF .
+make -j8
+
+# create a simple shell wrapper that runs crosscompiled protoc under qemu
+echo '#!/bin/bash' >protoc_qemu_wrapper.sh
+echo 'exec qemu-aarch64 "../protoc" "$@"' >>protoc_qemu_wrapper.sh
+chmod ugo+x protoc_qemu_wrapper.sh
+
+# PROTOC variable is by build_py step that runs under ./python directory
+export PROTOC=../protoc_qemu_wrapper.sh
+
+pushd python
+
+# NOTE: this step will use protoc_qemu_wrapper.sh to generate protobuf files.
+${PYTHON} setup.py build_py
+
+# when crosscompiling for aarch64, --plat-name needs to be set explicitly
+# to end up with correctly named wheel file
+# the value should be manylinuxABC_ARCH and dockcross docker image
+# conveniently provides the value in the AUDITWHEEL_PLAT env
+plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
+
+# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
+export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
+
+# Build the python extension inplace to be able to python unittests later
+${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace
+
+# Build the binary wheel (to check it with auditwheel)
+${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag

libs/protobuf/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+cd python
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+${PYTHON} -m pip install --user pytest auditwheel numpy
+
+# check that we are really using aarch64 python
+(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1)
+
+# step 1: run all python unittests
+# we've built the python extension previously with --inplace option
+# so we can just discover all the unittests and run them directly under 
+# the python/ directory.
+LD_LIBRARY_PATH=. PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf
+
+# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible.
+# auditwheel needs to run on wheel's target platform (or under an emulator)
+${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl
+
+# step 3: smoketest that the wheel can be installed and run a smokecheck
+${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl
+# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfully loaded the python cpp extension!")'

libs/protobuf/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running
+# inside docker containers.
+
+set -ex
+
+# show pre-existing qemu registration
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+
+# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that:
+# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278")
+# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below)
+#
+# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static
+# docker image to provide a new enough version of qemu-user-static and register it with
+# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
+
+# Print current qemu reqistration to make sure everything is setup correctly.
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64

libs/protobuf/kokoro/linux/aarch64/ruby_build_and_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+gem install bundler
+
+cd ruby
+
+bundle
+rake
+rake clobber_package gem
+
+# run all the tests
+rake test

libs/protobuf/kokoro/linux/aarch64/test_cpp_aarch64.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Crosscompiles protobuf C++ under dockcross docker image and runs the tests under an emulator.
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+# run the C++ build and test script under dockcross/linux-arm64 image
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_linux_aarch64.sh kokoro/linux/aarch64/cpp_crosscompile_and_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/test_csharp_aarch64.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# First, build protobuf C# tests under x86_64 docker image
+# Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory
+# (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run)
+CSHARP_BUILD_COMMAND="dotnet publish -c Release -f net60 csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj"
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim bash -c "$CSHARP_BUILD_COMMAND"
+
+# Use an actual aarch64 docker image to run protobuf C# tests with an emulator. "dotnet vstest" allows
+# running tests from a pre-built project.
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+CSHARP_TEST_COMMAND="dotnet vstest csharp/src/Google.Protobuf.Test/bin/Release/net60/publish/Google.Protobuf.Test.dll"
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8 bash -c "$CSHARP_TEST_COMMAND"

libs/protobuf/kokoro/linux/aarch64/test_java_aarch64.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile protoc as we will later need it for the java build.
+# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
+# with a wide range of linux distros (including any docker images we will use later to build and test java)
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
+
+# the command that will be used to build and test java under an emulator
+# * IsValidUtf8Test and DecodeUtf8Test tests are skipped because they take very long under an emulator.
+TEST_JAVA_COMMAND="mvn --batch-mode -DskipTests install && mvn --batch-mode -Dtest='**/*Test, !**/*IsValidUtf8Test, !**/*DecodeUtf8Test' -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false surefire:test"
+
+# use an actual aarch64 docker image (with a real aarch64 java and maven) to run build & test protobuf java under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+# * the JAVA_OPTS and MAVEN_CONFIG variables are being set mostly to silence warnings about non-existent home directory
+#   and to avoid polluting the workspace.
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "JAVA_OPTS=-Duser.home=/home/fake-user" -e "MAVEN_CONFIG=/home/fake-user/.m2" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/maven:3.8-openjdk-11 bash -c "cd java && $TEST_JAVA_COMMAND"

libs/protobuf/kokoro/linux/aarch64/test_php_aarch64.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+# there is no php testing docker image readily available, so we build
+# our own. It's a aarch64 image, but that's fine since qemu will
+# automatically be used to run the commands in the dockerfile.
+docker build -t testimage_protobuf_php_arm64v8 kokoro/linux/aarch64/testimage_protobuf_php_arm64v8
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile protoc as we will later need it for the php build.
+# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
+# with a wide range of linux distros (including any docker images we will use later to build and test php)
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
+
+# use an actual aarch64 docker image (with a real aarch64 php) to run build & test protobuf php under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work testimage_protobuf_php_arm64v8 kokoro/linux/aarch64/php_build_and_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/test_python_aarch64.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh
+
+# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution
+#   (manylinux_2_24 is debian-based while manylinux2014 is centos-based)
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/test_ruby_aarch64.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
+# crosscompile protoc as we will later need it for the ruby build.
+# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
+# with a wide range of linux distros (including any docker images we will use later to build and test ruby)
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
+
+# use an actual aarch64 docker image (with a real aarch64 ruby) to run build & test protobuf ruby under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "PROTOC=/work/protoc" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/ruby:2.7.3-buster kokoro/linux/aarch64/ruby_build_and_run_tests_with_qemu_aarch64.sh

libs/protobuf/kokoro/linux/aarch64/testimage_protobuf_php_arm64v8/Dockerfile

@@ -0,0 +1,3 @@
+FROM arm64v8/debian:buster
+
+RUN apt-get update && apt-get install -y php7.3-cli php7.3-dev php7.3-bcmath composer phpunit curl git valgrind && apt-get clean

libs/protobuf/kokoro/linux/bazel.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -ex
+
+if [[ -z "${CONTAINER_IMAGE}" ]]; then
+  CONTAINER_IMAGE=gcr.io/protobuf-build/bazel/linux@sha256:2bfd061284eff8234f2fcca16d71d43c69ccf3a22206628b54c204a6a9aac277
+fi
+
+cd $(dirname $0)/../..
+GIT_REPO_ROOT=`pwd`
+
+ENVS=()
+
+# Check for specific versions pinned to the docker image.  In these cases we
+# want to forward the environment variable to tests, so that they can verify
+# that the correct version is being picked up by Bazel.
+ENVS+=("--test_env=KOKORO_JAVA_VERSION")
+ENVS+=("--test_env=KOKORO_PYTHON_VERSION")
+ENVS+=("--test_env=KOKORO_RUBY_VERSION")
+
+if [ -n "$BAZEL_ENV" ]; then
+  for env in $BAZEL_ENV; do
+    ENVS+="--action_env=${env}"
+  done
+fi
+
+function run {
+  local CONFIG=$1
+  local BAZEL_CONFIG=$2
+
+  tmpfile=$(mktemp -u)
+
+  rm -rf $GIT_REPO_ROOT/bazel-out $GIT_REPO_ROOT/bazel-bin
+  rm -rf $GIT_REPO_ROOT/logs
+
+  docker run \
+    --cidfile $tmpfile \
+    --cap-add=SYS_PTRACE \
+    -v $GIT_REPO_ROOT:/workspace \
+    $CONTAINER_IMAGE \
+    test \
+    $(${GIT_REPO_ROOT}/kokoro/common/bazel_flags.sh) \
+    ${ENVS[@]} \
+    $PLATFORM_CONFIG \
+    $BAZEL_CONFIG \
+    $BAZEL_EXTRA_FLAGS \
+    $BAZEL_TARGETS
+
+  # Save logs for Kokoro
+  docker cp \
+    `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR/$CONFIG
+}
+
+if [ -n "$BAZEL_CONFIGS" ]; then
+  for config in $BAZEL_CONFIGS; do
+    run $config "--config=$config"
+  done
+else
+    run
+fi

libs/protobuf/kokoro/linux/bazel/common.cfg

@@ -0,0 +1,31 @@
+# Default setup for the all of our Kokoro builds.
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/bazel/linux-san:b6bfa3bb505e83f062af0cb0ed23abf1e89b9edb"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//src/... @com_google_protobuf_examples//..."
+}
+
+env_vars {
+  key: "BAZEL_CONFIGS"
+  value: "opt dbg asan kokoro-msan tsan ubsan"
+}
+
+env_vars {
+  key: "BAZEL_EXTRA_FLAGS"
+  value: "--distinct_host_configuration=false"
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/bazel/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/bazel/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Build file to set up and run tests using CMake
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=`pwd`
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
+
+# Update git submodules
+git submodule update --init --recursive
+
+tmpfile=$(mktemp -u)
+
+docker run \
+  --cidfile $tmpfile \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  /test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_EXAMPLES=ON
+
+# Save logs for Kokoro
+docker cp \
+  `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

libs/protobuf/kokoro/linux/cmake/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/cmake/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake_install/build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Build file to build, install, and test using CMake.
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=`pwd`
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
+
+# Update git submodules
+git submodule update --init --recursive
+
+tmpfile=$(mktemp -u)
+
+docker run \
+  --cidfile $tmpfile \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  "/install.sh && /test.sh \
+  -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \
+  -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \
+  -Dprotobuf_BUILD_CONFORMANCE=ON"
+
+
+# Save logs for Kokoro
+docker cp \
+  `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

libs/protobuf/kokoro/linux/cmake_install/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_install/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/cmake_install/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake_install/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake_ninja/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Build file to set up and run tests using CMake with the Ninja generator.
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=`pwd`
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
+
+# Update git submodules
+git submodule update --init --recursive
+
+tmpfile=$(mktemp -u)
+
+docker run \
+  --cidfile $tmpfile \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  /test.sh -G Ninja -Dprotobuf_BUILD_CONFORMANCE=ON
+
+# Save logs for Kokoro
+docker cp \
+  `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

libs/protobuf/kokoro/linux/cmake_ninja/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_ninja/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/cmake_ninja/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake_ninja/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake_shared/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Build file to set up and run tests via CMake using shared libraries
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=`pwd`
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
+
+# Update git submodules
+git submodule update --init --recursive
+
+tmpfile=$(mktemp -u)
+
+docker run \
+  --cidfile $tmpfile \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  /test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_SHARED_LIBS=ON
+
+# Save logs for Kokoro
+docker cp \
+  `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

libs/protobuf/kokoro/linux/cmake_shared/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/cmake_shared/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cmake_shared/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cpp_aarch64/common.cfg

@@ -0,0 +1,21 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/emulation/linux:aarch64-4e847d7a01c1792471b6dd985ab0bf2677332e6f"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//src/..."
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/cpp_aarch64/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cpp_aarch64/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cpp_tcmalloc/common.cfg

@@ -0,0 +1,21 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 1440
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/tcmalloc/linux:64e8944e4f18d7d6c9649112a8a93be57e693cd8"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//src/..."
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/cpp_tcmalloc/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/cpp_tcmalloc/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/csharp/common.cfg

@@ -0,0 +1,27 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 1440
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/csharp/linux:3.1.415-6.0.100-6bbe70439ba5b0404bb12662cebc0296909389fa"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//csharp/..."
+}
+
+env_vars {
+  key: "BAZEL_EXTRA_FLAGS"
+  value: "--action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 "
+         "--test_env=DOTNET_CLI_HOME=/home/bazel"
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/csharp/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/csharp/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/csharp_aarch64/build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "continuous" and "presubmit" jobs.
+
+set -ex
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
+
+kokoro/linux/aarch64/test_csharp_aarch64.sh

libs/protobuf/kokoro/linux/csharp_aarch64/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/csharp_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

libs/protobuf/kokoro/linux/csharp_aarch64/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/csharp_aarch64/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_aarch64/common.cfg

@@ -0,0 +1,21 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/emulation/linux:aarch64-4e847d7a01c1792471b6dd985ab0bf2677332e6f"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//java/..."
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/java_aarch64/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_aarch64/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_jdk11/common.cfg

@@ -0,0 +1,16 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//java/..."
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/java_jdk11/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_jdk11/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_jdk17/common.cfg

@@ -0,0 +1,21 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/java/linux:17-64e8944e4f18d7d6c9649112a8a93be57e693cd8"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//java/..."
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/java_jdk17/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_jdk17/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_linkage_monitor/build.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.
+
+use_bazel.sh 4.2.2
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+bazel build //:protoc
+
+# The java build setup expects protoc in the root directory.
+cp bazel-bin/protoc .
+
+cd java
+# Installs the snapshot version locally
+mvn -e -B -Dhttps.protocols=TLSv1.2 install -Dmaven.test.skip=true
+
+# Linkage Monitor uses the snapshot versions installed in $HOME/.m2 to verify compatibility
+JAR=linkage-monitor-latest-all-deps.jar
+curl -v -O "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/${JAR}"
+# Fails if there's new linkage errors compared with baseline
+java -jar $JAR com.google.cloud:libraries-bom

libs/protobuf/kokoro/linux/java_linkage_monitor/common.cfg

@@ -0,0 +1,12 @@
+# Config file for running Linkage Monitor in Kokoro
+# https://github.com/GoogleCloudPlatform/cloud-opensource-java/tree/master/linkage-monitor
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/java_linkage_monitor/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

libs/protobuf/kokoro/linux/java_linkage_monitor/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/java_linkage_monitor/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/jruby92/common.cfg

@@ -0,0 +1,26 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/ruby/linux:jruby-9.2.20.1-64e8944e4f18d7d6c9649112a8a93be57e693cd8"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//ruby/..."
+}
+
+env_vars {
+  key: "BAZEL_EXTRA_FLAGS"
+  value: "--define=ruby_platform=java"
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/jruby92/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/jruby92/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/jruby93/common.cfg

@@ -0,0 +1,26 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/ruby/linux:jruby-9.3.4.0-64e8944e4f18d7d6c9649112a8a93be57e693cd8"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//ruby/..."
+}
+
+env_vars {
+  key: "BAZEL_EXTRA_FLAGS"
+  value: "--define=ruby_platform=java"
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/jruby93/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/jruby93/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/php_aarch64/build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "continuous" and "presubmit" jobs.
+
+set -ex
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
+
+kokoro/linux/aarch64/test_php_aarch64.sh

libs/protobuf/kokoro/linux/php_aarch64/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/php_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

libs/protobuf/kokoro/linux/php_aarch64/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/php_aarch64/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/php_all/build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# This is the entry point for kicking off a Kokoro job. This path is referenced
+# from the .cfg files in this directory.
+
+set -ex
+
+# Change to repo base.
+cd $(dirname $0)/../../..
+
+docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test_valgrind"
+
+docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:7.0.33-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
+docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:7.3.28-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
+docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:7.4.18-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
+docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
+
+# Run specialized memory leak & multirequest tests.
+docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test_c && tests/multirequest.sh && tests/memory_leak_test.sh"
+
+# Most of our tests use a debug build of PHP, but we do one build against an opt
+# php just in case that surfaces anything unexpected.
+docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"

libs/protobuf/kokoro/linux/php_all/common.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/php_all/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

libs/protobuf/kokoro/linux/php_all/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/php_all/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/python310/common.cfg

@@ -0,0 +1,21 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/python/linux:3.10-2f706fd1ab49f4e97af769388be486069b63efee"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//python/... @upb//python/..."
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/python310/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/python310/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/python310_cpp/common.cfg

@@ -0,0 +1,27 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/python/linux:3.10-2f706fd1ab49f4e97af769388be486069b63efee"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  # Note: upb tests don't work since the C++ extension takes precedence here.
+  value: "//python/..."
+}
+
+env_vars {
+  key: "BAZEL_EXTRA_FLAGS"
+  value: "--define=use_fast_cpp_protos=true"
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/python310_cpp/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/python310_cpp/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/python37/common.cfg

@@ -0,0 +1,21 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/python/linux:3.7-2f706fd1ab49f4e97af769388be486069b63efee"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  value: "//python/... @upb//python/..."
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/python37/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/python37/presubmit.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.

libs/protobuf/kokoro/linux/python37_cpp/common.cfg

@@ -0,0 +1,27 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel.sh"
+timeout_mins: 120
+
+env_vars {
+  key: "CONTAINER_IMAGE"
+  value: "gcr.io/protobuf-build/python/linux:3.7-2f706fd1ab49f4e97af769388be486069b63efee"
+}
+
+env_vars {
+  key: "BAZEL_TARGETS"
+  # Note: upb tests don't work since the C++ extension takes precedence here.
+  value: "//python/..."
+}
+
+env_vars {
+  key: "BAZEL_EXTRA_FLAGS"
+  value: "--define=use_fast_cpp_protos=true"
+}
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

libs/protobuf/kokoro/linux/python37_cpp/continuous.cfg

@@ -0,0 +1 @@
+# Keep this file empty! Use common.cfg instead.