Squashed 'libs/protobuf/' content from commit fcd3b9a85

git-subtree-dir: libs/protobuf
git-subtree-split: fcd3b9a85ef36e46643dc30176cea1a7ad62e02b
This commit is contained in:
Henry Winkel
2022-10-22 14:46:58 +02:00
commit 36bca61764
2186 changed files with 838730 additions and 0 deletions

23
kokoro/linux/32-bit/build.sh Executable file
View File

@@ -0,0 +1,23 @@
#!/bin/bash
#
# This is the top-level script we give to Kokoro as the entry point for
# running the "pull request 32" project:
#
# This script selects a specific Dockerfile (for building a Docker image) and
# a script to run inside that image.
set -ex
# Change to repo root
cd $(dirname $0)/../../..
GIT_REPO_ROOT=$(pwd)
CONTAINER_IMAGE=gcr.io/protobuf-build/php/32bit@sha256:824cbdff02ee543eb69ee4b02c8c58cc7887f70f49e41725a35765d92a898b4f
git submodule update --init --recursive
docker run \
"$@" \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
bash -l "/workspace/kokoro/linux/32-bit/test_php.sh"

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/32-bit/build.sh"
timeout_mins: 120
action {
define_artifacts {
regex: "**/sponge_log.xml"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,58 @@
#!/bin/bash
set -eux
# Change to repo root
cd $(dirname $0)/../../..
use_php() {
VERSION=$1
export PATH=/usr/local/php-${VERSION}/bin:$PATH
}
build_php() {
use_php $1
pushd php
rm -rf vendor
php -v
php -m
composer update
composer test
popd
}
test_php_c() {
pushd php
rm -rf vendor
php -v
php -m
composer update
composer test_c
popd
}
build_php_c() {
use_php $1
test_php_c
}
mkdir -p build
pushd build
cmake ..
cmake --build . -- -j20
ctest --verbose --parallel 20
export PROTOC=$(pwd)/protoc
popd
build_php 7.0
build_php 7.1
build_php 7.4
build_php_c 7.0
build_php_c 7.1
build_php_c 7.4
build_php_c 7.1-zts
build_php_c 7.2-zts
build_php_c 7.5-zts
# Cleanup after CMake build
rm -rf build

View File

@@ -0,0 +1,18 @@
#!/bin/bash
#
# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator.
# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image.
set -ex
# the build commands are expected to run under dockcross docker image
# where the CC, CXX and other toolchain variables already point to the crosscompiler
cmake .
make -j8
# check that the resulting test binary is indeed an aarch64 ELF
(file ./tests | grep -q "ELF 64-bit LSB executable, ARM aarch64") || (echo "Test binary in not an aarch64 binary"; exit 1)
# run the basic set of C++ tests under QEMU
# there are other tests we could run (e.g. ./lite-test), but this is sufficient as a smoketest
qemu-aarch64 ./tests

View File

@@ -0,0 +1,36 @@
#!/bin/bash
set -e
# go to the repo root
cd $(dirname $0)/../../../..
if [[ -t 0 ]]; then
DOCKER_TTY_ARGS="-it"
else
# The input device on kokoro is not a TTY, so -it does not work.
DOCKER_TTY_ARGS=
fi
# Pin the dockcross image since newer versions of the image break the build
PINNED_DOCKCROSS_IMAGE_VERSION=dockcross/linux-arm64:20210625-795dd4d
# running dockcross image without any arguments generates a wrapper
# scripts that can be used to run commands under the dockcross image
# easily.
# See https://github.com/dockcross/dockcross#usage for details
docker run $DOCKER_TTY_ARGS --rm $PINNED_DOCKCROSS_IMAGE_VERSION >dockcross-linux-arm64.sh
chmod +x dockcross-linux-arm64.sh
# the wrapper script has CRLF line endings and bash doesn't like that
# so we change CRLF line endings into LF.
sed -i 's/\r//g' dockcross-linux-arm64.sh
# The dockcross wrapper script runs arbitrary commands under the selected dockcross
# image with the following properties which make its use very convenient:
# * the current working directory is mounted under /work so the container can easily
# access the current workspace
# * the processes in the container run under the same UID and GID as the host process so unlike
# vanilla "docker run" invocations, the workspace doesn't get polluted with files
# owned by root.
./dockcross-linux-arm64.sh --image $PINNED_DOCKCROSS_IMAGE_VERSION -- "$@"

View File

@@ -0,0 +1,40 @@
#!/bin/bash
set -e
# go to the repo root
cd $(dirname $0)/../../../..
if [[ -t 0 ]]; then
DOCKER_TTY_ARGS="-it"
else
# The input device on kokoro is not a TTY, so -it does not work.
DOCKER_TTY_ARGS=
fi
# Pin the dockcross image since newer versions of the image break the build
# We use an older version of dockcross image that has gcc4.9.4 because it was built
# before https://github.com/dockcross/dockcross/pull/449
# Thanks to that, wheel build with this image aren't actually
# compliant with manylinux2014, but only with manylinux_2_24
PINNED_DOCKCROSS_IMAGE_VERSION=dockcross/manylinux2014-aarch64:20200929-608e6ac
# running dockcross image without any arguments generates a wrapper
# scripts that can be used to run commands under the dockcross image
# easily.
# See https://github.com/dockcross/dockcross#usage for details
docker run $DOCKER_TTY_ARGS --rm $PINNED_DOCKCROSS_IMAGE_VERSION >dockcross-manylinux2014-aarch64.sh
chmod +x dockcross-manylinux2014-aarch64.sh
# the wrapper script has CRLF line endings and bash doesn't like that
# so we change CRLF line endings into LF.
sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh
# The dockcross wrapper script runs arbitrary commands under the selected dockcross
# image with the following properties which make its use very convenient:
# * the current working directory is mounted under /work so the container can easily
# access the current workspace
# * the processes in the container run under the same UID and GID as the host process so unlike
# vanilla "docker run" invocations, the workspace doesn't get polluted with files
# owned by root.
./dockcross-manylinux2014-aarch64.sh --image $PINNED_DOCKCROSS_IMAGE_VERSION -- "$@"

View File

@@ -0,0 +1,18 @@
#!/bin/bash
set -ex
# Install composer
curl -sS https://getcomposer.org/installer | php
mkdir -p "$HOME/bin"
mv composer.phar "$HOME/bin/composer"
PATH="$HOME/bin:$PATH"
# go to the repo root
cd $(dirname $0)/../../..
cd php
composer install
composer test
composer test_c

View File

@@ -0,0 +1,8 @@
#!/bin/bash
#
# Builds protobuf C++ with aarch64 crosscompiler.
set -ex
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_WITH_ZLIB=0 -Dprotobuf_BUILD_TESTS=OFF .
make -j8

View File

@@ -0,0 +1,44 @@
#!/bin/bash
#
# Builds protobuf python including the C++ extension with aarch64 crosscompiler.
# The outputs of this script are laid out so that we can later test them under an aarch64 emulator.
# NOTE: This script is expected to run under the dockcross/manylinux2014-aarch64 docker image.
set -ex
PYTHON="/opt/python/cp38-cp38/bin/python"
# Initialize any submodules.
git submodule update --init --recursive
# Build protoc and libprotobuf
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_WITH_ZLIB=0 -Dprotobuf_BUILD_TESTS=OFF .
make -j8
# create a simple shell wrapper that runs crosscompiled protoc under qemu
echo '#!/bin/bash' >protoc_qemu_wrapper.sh
echo 'exec qemu-aarch64 "../protoc" "$@"' >>protoc_qemu_wrapper.sh
chmod ugo+x protoc_qemu_wrapper.sh
# PROTOC variable is by build_py step that runs under ./python directory
export PROTOC=../protoc_qemu_wrapper.sh
pushd python
# NOTE: this step will use protoc_qemu_wrapper.sh to generate protobuf files.
${PYTHON} setup.py build_py
# when crosscompiling for aarch64, --plat-name needs to be set explicitly
# to end up with correctly named wheel file
# the value should be manylinuxABC_ARCH and dockcross docker image
# conveniently provides the value in the AUDITWHEEL_PLAT env
plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
# Build the python extension inplace to be able to python unittests later
${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace
# Build the binary wheel (to check it with auditwheel)
${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag

View File

@@ -0,0 +1,28 @@
#!/bin/bash
set -ex
# go to the repo root
cd $(dirname $0)/../../..
cd python
PYTHON="/opt/python/cp38-cp38/bin/python"
${PYTHON} -m pip install --user pytest auditwheel numpy
# check that we are really using aarch64 python
(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1)
# step 1: run all python unittests
# we've built the python extension previously with --inplace option
# so we can just discover all the unittests and run them directly under
# the python/ directory.
LD_LIBRARY_PATH=. PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf
# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible.
# auditwheel needs to run on wheel's target platform (or under an emulator)
${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl
# step 3: smoketest that the wheel can be installed and run a smokecheck
${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl
# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfully loaded the python cpp extension!")'

View File

@@ -0,0 +1,26 @@
#!/bin/bash
#
# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running
# inside docker containers.
set -ex
# show pre-existing qemu registration
cat /proc/sys/fs/binfmt_misc/qemu-aarch64
# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that:
# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278")
# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below)
#
# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static
# docker image to provide a new enough version of qemu-user-static and register it with
# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"),
# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
# binary to be accessible from the docker image we're emulating.
# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
# to install qemu-user-static with the right flags.
docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
# Print current qemu reqistration to make sure everything is setup correctly.
cat /proc/sys/fs/binfmt_misc/qemu-aarch64

View File

@@ -0,0 +1,17 @@
#!/bin/bash
set -ex
# go to the repo root
cd $(dirname $0)/../../..
gem install bundler
cd ruby
bundle
rake
rake clobber_package gem
# run all the tests
rake test

View File

@@ -0,0 +1,14 @@
#!/bin/bash
#
# Crosscompiles protobuf C++ under dockcross docker image and runs the tests under an emulator.
set -e
# go to the repo root
cd $(dirname $0)/../../..
# Initialize any submodules.
git submodule update --init --recursive
# run the C++ build and test script under dockcross/linux-arm64 image
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_linux_aarch64.sh kokoro/linux/aarch64/cpp_crosscompile_and_run_tests_with_qemu_aarch64.sh

View File

@@ -0,0 +1,29 @@
#!/bin/bash
set -ex
# go to the repo root
cd $(dirname $0)/../../..
if [[ -t 0 ]]; then
DOCKER_TTY_ARGS="-it"
else
# The input device on kokoro is not a TTY, so -it does not work.
DOCKER_TTY_ARGS=
fi
# First, build protobuf C# tests under x86_64 docker image
# Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory
# (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run)
CSHARP_BUILD_COMMAND="dotnet publish -c Release -f net60 csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj"
docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim bash -c "$CSHARP_BUILD_COMMAND"
# Use an actual aarch64 docker image to run protobuf C# tests with an emulator. "dotnet vstest" allows
# running tests from a pre-built project.
# * mount the protobuf root as /work to be able to access the crosscompiled files
# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
CSHARP_TEST_COMMAND="dotnet vstest csharp/src/Google.Protobuf.Test/bin/Release/net60/publish/Google.Protobuf.Test.dll"
docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8 bash -c "$CSHARP_TEST_COMMAND"

View File

@@ -0,0 +1,32 @@
#!/bin/bash
set -ex
# go to the repo root
cd $(dirname $0)/../../..
if [[ -t 0 ]]; then
DOCKER_TTY_ARGS="-it"
else
# The input device on kokoro is not a TTY, so -it does not work.
DOCKER_TTY_ARGS=
fi
# crosscompile protoc as we will later need it for the java build.
# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
# with a wide range of linux distros (including any docker images we will use later to build and test java)
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
# the command that will be used to build and test java under an emulator
# * IsValidUtf8Test and DecodeUtf8Test tests are skipped because they take very long under an emulator.
TEST_JAVA_COMMAND="mvn --batch-mode -DskipTests install && mvn --batch-mode -Dtest='**/*Test, !**/*IsValidUtf8Test, !**/*DecodeUtf8Test' -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false surefire:test"
# use an actual aarch64 docker image (with a real aarch64 java and maven) to run build & test protobuf java under an emulator
# * mount the protobuf root as /work to be able to access the crosscompiled files
# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
# * the JAVA_OPTS and MAVEN_CONFIG variables are being set mostly to silence warnings about non-existent home directory
# and to avoid polluting the workspace.
docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "JAVA_OPTS=-Duser.home=/home/fake-user" -e "MAVEN_CONFIG=/home/fake-user/.m2" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/maven:3.8-openjdk-11 bash -c "cd java && $TEST_JAVA_COMMAND"

View File

@@ -0,0 +1,31 @@
#!/bin/bash
set -ex
# go to the repo root
cd $(dirname $0)/../../..
# there is no php testing docker image readily available, so we build
# our own. It's a aarch64 image, but that's fine since qemu will
# automatically be used to run the commands in the dockerfile.
docker build -t testimage_protobuf_php_arm64v8 kokoro/linux/aarch64/testimage_protobuf_php_arm64v8
if [[ -t 0 ]]; then
DOCKER_TTY_ARGS="-it"
else
# The input device on kokoro is not a TTY, so -it does not work.
DOCKER_TTY_ARGS=
fi
# crosscompile protoc as we will later need it for the php build.
# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
# with a wide range of linux distros (including any docker images we will use later to build and test php)
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
# use an actual aarch64 docker image (with a real aarch64 php) to run build & test protobuf php under an emulator
# * mount the protobuf root as /work to be able to access the crosscompiled files
# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work testimage_protobuf_php_arm64v8 kokoro/linux/aarch64/php_build_and_run_tests_with_qemu_aarch64.sh

View File

@@ -0,0 +1,26 @@
#!/bin/bash
set -e
# go to the repo root
cd $(dirname $0)/../../..
if [[ -t 0 ]]; then
DOCKER_TTY_ARGS="-it"
else
# The input device on kokoro is not a TTY, so -it does not work.
DOCKER_TTY_ARGS=
fi
# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh
# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator
# * mount the protobuf root as /work to be able to access the crosscompiled files
# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution
# (manylinux_2_24 is debian-based while manylinux2014 is centos-based)
# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

View File

@@ -0,0 +1,27 @@
#!/bin/bash
set -ex
# go to the repo root
cd $(dirname $0)/../../..
if [[ -t 0 ]]; then
DOCKER_TTY_ARGS="-it"
else
# The input device on kokoro is not a TTY, so -it does not work.
DOCKER_TTY_ARGS=
fi
# crosscompile protoc as we will later need it for the ruby build.
# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
# with a wide range of linux distros (including any docker images we will use later to build and test ruby)
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
# use an actual aarch64 docker image (with a real aarch64 ruby) to run build & test protobuf ruby under an emulator
# * mount the protobuf root as /work to be able to access the crosscompiled files
# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "PROTOC=/work/protoc" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/ruby:2.7.3-buster kokoro/linux/aarch64/ruby_build_and_run_tests_with_qemu_aarch64.sh

View File

@@ -0,0 +1,3 @@
FROM arm64v8/debian:buster
RUN apt-get update && apt-get install -y php7.3-cli php7.3-dev php7.3-bcmath composer phpunit curl git valgrind && apt-get clean

60
kokoro/linux/bazel.sh Executable file
View File

@@ -0,0 +1,60 @@
#!/bin/bash
set -ex
if [[ -z "${CONTAINER_IMAGE}" ]]; then
CONTAINER_IMAGE=gcr.io/protobuf-build/bazel/linux@sha256:2bfd061284eff8234f2fcca16d71d43c69ccf3a22206628b54c204a6a9aac277
fi
cd $(dirname $0)/../..
GIT_REPO_ROOT=`pwd`
ENVS=()
# Check for specific versions pinned to the docker image. In these cases we
# want to forward the environment variable to tests, so that they can verify
# that the correct version is being picked up by Bazel.
ENVS+=("--test_env=KOKORO_JAVA_VERSION")
ENVS+=("--test_env=KOKORO_PYTHON_VERSION")
ENVS+=("--test_env=KOKORO_RUBY_VERSION")
if [ -n "$BAZEL_ENV" ]; then
for env in $BAZEL_ENV; do
ENVS+="--action_env=${env}"
done
fi
function run {
local CONFIG=$1
local BAZEL_CONFIG=$2
tmpfile=$(mktemp -u)
rm -rf $GIT_REPO_ROOT/bazel-out $GIT_REPO_ROOT/bazel-bin
rm -rf $GIT_REPO_ROOT/logs
docker run \
--cidfile $tmpfile \
--cap-add=SYS_PTRACE \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
test \
$(${GIT_REPO_ROOT}/kokoro/common/bazel_flags.sh) \
${ENVS[@]} \
$PLATFORM_CONFIG \
$BAZEL_CONFIG \
$BAZEL_EXTRA_FLAGS \
$BAZEL_TARGETS
# Save logs for Kokoro
docker cp \
`cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR/$CONFIG
}
if [ -n "$BAZEL_CONFIGS" ]; then
for config in $BAZEL_CONFIGS; do
run $config "--config=$config"
done
else
run
fi

View File

@@ -0,0 +1,31 @@
# Default setup for the all of our Kokoro builds.
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/bazel/linux-san:b6bfa3bb505e83f062af0cb0ed23abf1e89b9edb"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//src/... @com_google_protobuf_examples//..."
}
env_vars {
key: "BAZEL_CONFIGS"
value: "opt dbg asan kokoro-msan tsan ubsan"
}
env_vars {
key: "BAZEL_EXTRA_FLAGS"
value: "--distinct_host_configuration=false"
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

26
kokoro/linux/cmake/build.sh Executable file
View File

@@ -0,0 +1,26 @@
#!/bin/bash
#
# Build file to set up and run tests using CMake
set -eux
# Change to repo root
cd $(dirname $0)/../../..
GIT_REPO_ROOT=`pwd`
CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
# Update git submodules
git submodule update --init --recursive
tmpfile=$(mktemp -u)
docker run \
--cidfile $tmpfile \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
/test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_EXAMPLES=ON
# Save logs for Kokoro
docker cp \
`cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/cmake/build.sh"
timeout_mins: 1440
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,30 @@
#!/bin/bash
#
# Build file to build, install, and test using CMake.
set -eux
# Change to repo root
cd $(dirname $0)/../../..
GIT_REPO_ROOT=`pwd`
CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
# Update git submodules
git submodule update --init --recursive
tmpfile=$(mktemp -u)
docker run \
--cidfile $tmpfile \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
"/install.sh && /test.sh \
-Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \
-Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \
-Dprotobuf_BUILD_CONFORMANCE=ON"
# Save logs for Kokoro
docker cp \
`cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/cmake_install/build.sh"
timeout_mins: 1440
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,26 @@
#!/bin/bash
#
# Build file to set up and run tests using CMake with the Ninja generator.
set -eux
# Change to repo root
cd $(dirname $0)/../../..
GIT_REPO_ROOT=`pwd`
CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
# Update git submodules
git submodule update --init --recursive
tmpfile=$(mktemp -u)
docker run \
--cidfile $tmpfile \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
/test.sh -G Ninja -Dprotobuf_BUILD_CONFORMANCE=ON
# Save logs for Kokoro
docker cp \
`cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/cmake_ninja/build.sh"
timeout_mins: 1440
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,26 @@
#!/bin/bash
#
# Build file to set up and run tests via CMake using shared libraries
set -eux
# Change to repo root
cd $(dirname $0)/../../..
GIT_REPO_ROOT=`pwd`
CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
# Update git submodules
git submodule update --init --recursive
tmpfile=$(mktemp -u)
docker run \
--cidfile $tmpfile \
-v $GIT_REPO_ROOT:/workspace \
$CONTAINER_IMAGE \
/test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_SHARED_LIBS=ON
# Save logs for Kokoro
docker cp \
`cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/cmake/build.sh"
timeout_mins: 1440
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/emulation/linux:aarch64-4e847d7a01c1792471b6dd985ab0bf2677332e6f"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//src/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 1440
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/tcmalloc/linux:64e8944e4f18d7d6c9649112a8a93be57e693cd8"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//src/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,27 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 1440
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/csharp/linux:3.1.415-6.0.100-6bbe70439ba5b0404bb12662cebc0296909389fa"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//csharp/..."
}
env_vars {
key: "BAZEL_EXTRA_FLAGS"
value: "--action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 "
"--test_env=DOTNET_CLI_HOME=/home/bazel"
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,16 @@
#!/bin/bash
#
# This is the top-level script we give to Kokoro as the entry point for
# running the "continuous" and "presubmit" jobs.
set -ex
# Change to repo root
cd $(dirname $0)/../../..
# Initialize any submodules.
git submodule update --init --recursive
kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
kokoro/linux/aarch64/test_csharp_aarch64.sh

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/csharp_aarch64/build.sh"
timeout_mins: 120
action {
define_artifacts {
regex: "**/sponge_log.xml"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/emulation/linux:aarch64-4e847d7a01c1792471b6dd985ab0bf2677332e6f"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//java/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,16 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "BAZEL_TARGETS"
value: "//java/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/java/linux:17-64e8944e4f18d7d6c9649112a8a93be57e693cd8"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//java/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,27 @@
#!/bin/bash
#
# This is the top-level script we give to Kokoro as the entry point for
# running the "pull request" project:
#
# This script selects a specific Dockerfile (for building a Docker image) and
# a script to run inside that image.
use_bazel.sh 4.2.2
# Change to repo root
cd $(dirname $0)/../../..
bazel build //:protoc
# The java build setup expects protoc in the root directory.
cp bazel-bin/protoc .
cd java
# Installs the snapshot version locally
mvn -e -B -Dhttps.protocols=TLSv1.2 install -Dmaven.test.skip=true
# Linkage Monitor uses the snapshot versions installed in $HOME/.m2 to verify compatibility
JAR=linkage-monitor-latest-all-deps.jar
curl -v -O "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/${JAR}"
# Fails if there's new linkage errors compared with baseline
java -jar $JAR com.google.cloud:libraries-bom

View File

@@ -0,0 +1,12 @@
# Config file for running Linkage Monitor in Kokoro
# https://github.com/GoogleCloudPlatform/cloud-opensource-java/tree/master/linkage-monitor
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/java_linkage_monitor/build.sh"
timeout_mins: 120
action {
define_artifacts {
regex: "**/sponge_log.xml"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,26 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/ruby/linux:jruby-9.2.20.1-64e8944e4f18d7d6c9649112a8a93be57e693cd8"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//ruby/..."
}
env_vars {
key: "BAZEL_EXTRA_FLAGS"
value: "--define=ruby_platform=java"
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,26 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/ruby/linux:jruby-9.3.4.0-64e8944e4f18d7d6c9649112a8a93be57e693cd8"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//ruby/..."
}
env_vars {
key: "BAZEL_EXTRA_FLAGS"
value: "--define=ruby_platform=java"
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,16 @@
#!/bin/bash
#
# This is the top-level script we give to Kokoro as the entry point for
# running the "continuous" and "presubmit" jobs.
set -ex
# Change to repo root
cd $(dirname $0)/../../..
# Initialize any submodules.
git submodule update --init --recursive
kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
kokoro/linux/aarch64/test_php_aarch64.sh

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/php_aarch64/build.sh"
timeout_mins: 120
action {
define_artifacts {
regex: "**/sponge_log.xml"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

23
kokoro/linux/php_all/build.sh Executable file
View File

@@ -0,0 +1,23 @@
#!/bin/bash
#
# This is the entry point for kicking off a Kokoro job. This path is referenced
# from the .cfg files in this directory.
set -ex
# Change to repo base.
cd $(dirname $0)/../../..
docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test_valgrind"
docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:7.0.33-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:7.3.28-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:7.4.18-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"
# Run specialized memory leak & multirequest tests.
docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-dbg-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test_c && tests/multirequest.sh && tests/memory_leak_test.sh"
# Most of our tests use a debug build of PHP, but we do one build against an opt
# php just in case that surfaces anything unexpected.
docker run $(test -t 0 && echo "-it") -v$PWD:/workspace gcr.io/protobuf-build/php/linux:8.0.5-fd99596d4c4c9b78f984ee667a9b26b91a28eb8d "composer test && composer test_c"

View File

@@ -0,0 +1,11 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/php_all/build.sh"
timeout_mins: 120
action {
define_artifacts {
regex: "**/sponge_log.xml"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/python/linux:3.10-2f706fd1ab49f4e97af769388be486069b63efee"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//python/... @upb//python/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,27 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/python/linux:3.10-2f706fd1ab49f4e97af769388be486069b63efee"
}
env_vars {
key: "BAZEL_TARGETS"
# Note: upb tests don't work since the C++ extension takes precedence here.
value: "//python/..."
}
env_vars {
key: "BAZEL_EXTRA_FLAGS"
value: "--define=use_fast_cpp_protos=true"
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/python/linux:3.7-2f706fd1ab49f4e97af769388be486069b63efee"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//python/... @upb//python/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,27 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/python/linux:3.7-2f706fd1ab49f4e97af769388be486069b63efee"
}
env_vars {
key: "BAZEL_TARGETS"
# Note: upb tests don't work since the C++ extension takes precedence here.
value: "//python/..."
}
env_vars {
key: "BAZEL_EXTRA_FLAGS"
value: "--define=use_fast_cpp_protos=true"
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/python/linux:3.8-2f706fd1ab49f4e97af769388be486069b63efee"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//python/... @upb//python/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,27 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/python/linux:3.8-2f706fd1ab49f4e97af769388be486069b63efee"
}
env_vars {
key: "BAZEL_TARGETS"
# Note: upb tests don't work since the C++ extension takes precedence here.
value: "//python/..."
}
env_vars {
key: "BAZEL_EXTRA_FLAGS"
value: "--define=use_fast_cpp_protos=true"
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1 @@
# Keep this file empty! Use common.cfg instead.

View File

@@ -0,0 +1,21 @@
# Config file for running tests in Kokoro
# Location of the build script in repository
build_file: "protobuf/kokoro/linux/bazel.sh"
timeout_mins: 120
env_vars {
key: "CONTAINER_IMAGE"
value: "gcr.io/protobuf-build/python/linux:3.9-2f706fd1ab49f4e97af769388be486069b63efee"
}
env_vars {
key: "BAZEL_TARGETS"
value: "//python/... @upb//python/..."
}
action {
define_artifacts {
regex: "**/sponge_log.*"
}
}

Some files were not shown because too many files have changed in this diff Show More