From 6fd12a076d213255e37c00ca8955baa5f1cb77a1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 5 Jul 2018 16:43:26 +0200 Subject: [PATCH 01/21] add scripts for automated building of C# nuget on Kokoro --- kokoro/release/csharp/windows/build_nuget.bat | 5 +++++ kokoro/release/csharp/windows/release.cfg | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 kokoro/release/csharp/windows/build_nuget.bat create mode 100644 kokoro/release/csharp/windows/release.cfg diff --git a/kokoro/release/csharp/windows/build_nuget.bat b/kokoro/release/csharp/windows/build_nuget.bat new file mode 100644 index 0000000000000..bc15bd57ef02f --- /dev/null +++ b/kokoro/release/csharp/windows/build_nuget.bat @@ -0,0 +1,5 @@ +@rem enter repo root +cd /d %~dp0\..\..\..\.. + +cd csharp\src +call build_packages.bat diff --git a/kokoro/release/csharp/windows/release.cfg b/kokoro/release/csharp/windows/release.cfg new file mode 100644 index 0000000000000..f508c65bda0fe --- /dev/null +++ b/kokoro/release/csharp/windows/release.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/release/csharp/windows/build_nuget.bat" +timeout_mins: 60 + +action { + define_artifacts { + regex: "**/*.nupkg" + } +} From 9d0b9e4ea01fd1d895e312ee15a01866ad29085d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 13 Jul 2018 11:45:28 +0200 Subject: [PATCH 02/21] fix build_nuget.bat --- kokoro/release/csharp/windows/build_nuget.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kokoro/release/csharp/windows/build_nuget.bat b/kokoro/release/csharp/windows/build_nuget.bat index bc15bd57ef02f..5da1e13602689 100644 --- a/kokoro/release/csharp/windows/build_nuget.bat +++ b/kokoro/release/csharp/windows/build_nuget.bat @@ -1,5 +1,5 @@ @rem enter repo root cd /d %~dp0\..\..\..\.. -cd csharp\src +cd csharp call build_packages.bat From 937884476e8a5873af4e2a44762ed76865ffd877 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 13 Jul 2018 13:26:56 +0200 Subject: [PATCH 03/21] overcome the global.json constraint with a hack --- kokoro/release/csharp/windows/build_nuget.bat | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kokoro/release/csharp/windows/build_nuget.bat b/kokoro/release/csharp/windows/build_nuget.bat index 5da1e13602689..7552bec3a9ff9 100644 --- a/kokoro/release/csharp/windows/build_nuget.bat +++ b/kokoro/release/csharp/windows/build_nuget.bat @@ -2,4 +2,13 @@ cd /d %~dp0\..\..\..\.. cd csharp + +@rem see what dotnet version is available +dotnet --version + +@rem TODO(jtattermusch): Kokoro workers currently only have dotnet SDK 2.1.3 +@rem so we just overwrite the SDK requirement in global.json as the results +@rem should be fully compatible. +echo { "sdk": { "version": "2.1.3" } } >global.json + call build_packages.bat From 2f6443edcc86f5f3465bf37f5a5f9348d513f0c4 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 16 Jul 2018 19:17:42 +0200 Subject: [PATCH 04/21] use SDK version available on Kokoro --- csharp/global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/global.json b/csharp/global.json index 5ab775b9b4717..080dcb94038b6 100644 --- a/csharp/global.json +++ b/csharp/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "2.0.3" + "version": "2.1.3" } } From d754a0c478db3632d1dd490f62f33542c554fe5f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 16 Jul 2018 19:19:02 +0200 Subject: [PATCH 05/21] Revert "overcome the global.json constraint with a hack" This reverts commit c02346c72aafd1826b0397dbbce4ce04bf252f50. --- kokoro/release/csharp/windows/build_nuget.bat | 9 --------- 1 file changed, 9 deletions(-) diff --git a/kokoro/release/csharp/windows/build_nuget.bat b/kokoro/release/csharp/windows/build_nuget.bat index 7552bec3a9ff9..5da1e13602689 100644 --- a/kokoro/release/csharp/windows/build_nuget.bat +++ b/kokoro/release/csharp/windows/build_nuget.bat @@ -2,13 +2,4 @@ cd /d %~dp0\..\..\..\.. cd csharp - -@rem see what dotnet version is available -dotnet --version - -@rem TODO(jtattermusch): Kokoro workers currently only have dotnet SDK 2.1.3 -@rem so we just overwrite the SDK requirement in global.json as the results -@rem should be fully compatible. -echo { "sdk": { "version": "2.1.3" } } >global.json - call build_packages.bat From 149e31a6c046236c2d7d2fc8e2997270ae10fc3d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 17 Jul 2018 10:21:29 +0200 Subject: [PATCH 06/21] upgrade dotnet SDK to 2.1.3 on kokoro linux --- kokoro/linux/prepare_build_linux_rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kokoro/linux/prepare_build_linux_rc b/kokoro/linux/prepare_build_linux_rc index f64ea95238593..d9b3bec3aafd1 100644 --- a/kokoro/linux/prepare_build_linux_rc +++ b/kokoro/linux/prepare_build_linux_rc @@ -7,7 +7,7 @@ sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsof sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EB3E94ADBE1229CF sudo apt-get update # We use the .NET Core SDK 2.x to build... -sudo apt-get install -y dotnet-sdk-2.0.3 +sudo apt-get install -y dotnet-sdk-2.1.3 # But we also need the 1.x framework to test against, as we # target netstandard1.x sudo apt-get install -y dotnet-sharedframework-microsoft.netcore.app-1.0.5 From 770dde7d314e4fb06ab12244dbd1972157c648ad Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 3 Aug 2018 16:26:18 -0700 Subject: [PATCH 07/21] Update php package.xml to 3.6.1 (#5000) --- php/ext/google/protobuf/package.xml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/php/ext/google/protobuf/package.xml b/php/ext/google/protobuf/package.xml index 28d713f344198..a3bf2752e7791 100644 --- a/php/ext/google/protobuf/package.xml +++ b/php/ext/google/protobuf/package.xml @@ -10,11 +10,11 @@ protobuf-opensource@google.com yes - 2018-06-06 + 2018-08-03 - 3.6.0 - 3.6.0 + 3.6.1 + 3.6.1 stable @@ -245,6 +245,22 @@ G A release. 3-Clause BSD License +G A release. + + + + + 3.6.1 + 3.6.1 + + + stable + stable + + 2018-08-03 + + 3-Clause BSD License + G A release. From fba5ef410ad4a851fb4200b7380e3320ee048266 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Mon, 6 Aug 2018 22:07:52 -0700 Subject: [PATCH 08/21] Python wheel kokoro dev (#5002) * Add config for building python wheel * Update submodules * Fix setup.py to use c++11 * Fix syntax error * Fix syntax error * Add dir for artifact * Update artifact dirctory * Remove python 3.3 * Clean up virtual env * Clean up env * Test environment * Test env variable * Automatic update version number * Remove dependency on python-wheel branch --- .../release/python/linux/build_artifacts.sh | 45 +++++++++++++++++ kokoro/release/python/linux/config.sh | 48 +++++++++++++++++++ kokoro/release/python/linux/release.cfg | 8 ++++ 3 files changed, 101 insertions(+) create mode 100755 kokoro/release/python/linux/build_artifacts.sh create mode 100644 kokoro/release/python/linux/config.sh create mode 100644 kokoro/release/python/linux/release.cfg diff --git a/kokoro/release/python/linux/build_artifacts.sh b/kokoro/release/python/linux/build_artifacts.sh new file mode 100755 index 0000000000000..032e30d566304 --- /dev/null +++ b/kokoro/release/python/linux/build_artifacts.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -ex + +# change to repo root +pushd $(dirname $0)/../../../.. + +export REPO_DIR=protobuf +export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"` +export BUILD_COMMIT=v$BUILD_VERSION +export PLAT=x86_64 +export UNICODE_WIDTH=32 +export MACOSX_DEPLOYMENT_TARGET=10.9 + +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + +git clone https://github.com/matthew-brett/multibuild.git +cp kokoro/release/python/linux/config.sh config.sh + +build_artifact_version() { + MB_PYTHON_VERSION=$1 + + # Clean up env + rm -rf venv + sudo rm -rf protobuf + git clone https://github.com/google/protobuf.git + + source multibuild/common_utils.sh + source multibuild/travis_steps.sh + before_install + + clean_code $REPO_DIR $BUILD_COMMIT + sed -i '/Wno-sign-compare/a \ \ \ \ \ \ \ \ extra_compile_args.append("-std=c++11")' $REPO_DIR/python/setup.py + cat $REPO_DIR/python/setup.py + + build_wheel $REPO_DIR/python $PLAT + + mv wheelhouse/* $ARTIFACT_DIR +} + +build_artifact_version 2.7 +build_artifact_version 3.4 +build_artifact_version 3.5 +build_artifact_version 3.6 diff --git a/kokoro/release/python/linux/config.sh b/kokoro/release/python/linux/config.sh new file mode 100644 index 0000000000000..bfa8a0098b963 --- /dev/null +++ b/kokoro/release/python/linux/config.sh @@ -0,0 +1,48 @@ +# Define custom utilities +# Test for OSX with [ -n "$IS_OSX" ] + +function pre_build { + # Any stuff that you need to do before you start building the wheels + # Runs in the root directory of this repository. + pushd protobuf + + # Build protoc + ./autogen.sh + ./configure + + CXXFLAGS="-fPIC -g -O2" ./configure + make -j8 + + # Generate python dependencies. + pushd python + python setup.py build_py + popd + + popd +} + +function bdist_wheel_cmd { + # Builds wheel with bdist_wheel, puts into wheelhouse + # + # It may sometimes be useful to use bdist_wheel for the wheel building + # process. For example, versioneer has problems with versions which are + # fixed with bdist_wheel: + # https://github.com/warner/python-versioneer/issues/121 + local abs_wheelhouse=$1 + + # Modify build version + pwd + ls + python setup.py bdist_wheel --cpp_implementation --compile_static_extension + cp dist/*.whl $abs_wheelhouse +} + +function build_wheel { + build_wheel_cmd "bdist_wheel_cmd" $@ +} + +function run_tests { + # Runs tests on installed distribution from an empty directory + python --version + python -c "from google.protobuf.pyext import _message;" +} diff --git a/kokoro/release/python/linux/release.cfg b/kokoro/release/python/linux/release.cfg new file mode 100644 index 0000000000000..b31b3f11929b4 --- /dev/null +++ b/kokoro/release/python/linux/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Linux release builds +build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} From af3b2ab9159ab59003c539d48d8920a1d7dcd5ac Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Wed, 8 Aug 2018 15:52:16 -0700 Subject: [PATCH 09/21] Python wheel kokoro for mac (#5017) * Add config for building python wheel on mac * Specify os name * Clean up venv * Debug * Recover old path --- .../release/python/macos/build_artifacts.sh | 47 +++++++++++++++++++ kokoro/release/python/macos/release.cfg | 8 ++++ 2 files changed, 55 insertions(+) create mode 100755 kokoro/release/python/macos/build_artifacts.sh create mode 100644 kokoro/release/python/macos/release.cfg diff --git a/kokoro/release/python/macos/build_artifacts.sh b/kokoro/release/python/macos/build_artifacts.sh new file mode 100755 index 0000000000000..2aec5e64710d7 --- /dev/null +++ b/kokoro/release/python/macos/build_artifacts.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -ex + +# change to repo root +pushd $(dirname $0)/../../../.. + +export REPO_DIR=protobuf +export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"` +export BUILD_COMMIT=v$BUILD_VERSION +export PLAT=x86_64 +export UNICODE_WIDTH=32 +export MACOSX_DEPLOYMENT_TARGET=10.9 +export TRAVIS_OS_NAME="osx" + +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + +git clone https://github.com/matthew-brett/multibuild.git +cp kokoro/release/python/linux/config.sh config.sh + +OLD_PATH=$PATH + +build_artifact_version() { + MB_PYTHON_VERSION=$1 + + # Clean up env + rm -rf venv + sudo rm -rf protobuf + git clone https://github.com/google/protobuf.git + export PATH=$OLD_PATH + + source multibuild/common_utils.sh + source multibuild/travis_steps.sh + before_install + + clean_code $REPO_DIR $BUILD_COMMIT + + build_wheel $REPO_DIR/python $PLAT + + mv wheelhouse/* $ARTIFACT_DIR +} + +build_artifact_version 2.7 +build_artifact_version 3.4 +build_artifact_version 3.5 +build_artifact_version 3.6 diff --git a/kokoro/release/python/macos/release.cfg b/kokoro/release/python/macos/release.cfg new file mode 100644 index 0000000000000..85cf81b507f45 --- /dev/null +++ b/kokoro/release/python/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX release builds +build_file: "protobuf/kokoro/release/python/macos/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} From f2a1356966cef3889434368598ac888d131489fb Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Mon, 13 Aug 2018 10:33:56 -0700 Subject: [PATCH 10/21] Python wheel kokoro for windows (#5034) * Add kokoro config for building python wheels on windows * Move script to root * Debug * Debug * Change working directory * Add msbuild to path * Debug * Fix python dir name * Debug * Move artifacts to targeted directory * Update artifact directory * Debug * Debug * Automatic generation of version * Reduce file numbers --- .../python/windows/build_artifacts.bat | 53 ++++++++++++++++ .../python/windows/build_single_artifact.bat | 62 +++++++++++++++++++ kokoro/release/python/windows/release.cfg | 8 +++ 3 files changed, 123 insertions(+) create mode 100644 kokoro/release/python/windows/build_artifacts.bat create mode 100644 kokoro/release/python/windows/build_single_artifact.bat create mode 100644 kokoro/release/python/windows/release.cfg diff --git a/kokoro/release/python/windows/build_artifacts.bat b/kokoro/release/python/windows/build_artifacts.bat new file mode 100644 index 0000000000000..86270e6fb6e09 --- /dev/null +++ b/kokoro/release/python/windows/build_artifacts.bat @@ -0,0 +1,53 @@ +REM Move scripts to root +cd github\protobuf +copy kokoro\release\python\windows\build_single_artifact.bat build_single_artifact.bat + +REM Set environment variables +set REPO_DIR=protobuf +set PACKAGE_NAME=protobuf +set BUILD_DLL=OFF +set UNICODE=ON +set PB_TEST_DEP="six==1.9" +set OTHER_TEST_DEP="setuptools==38.5.1" +set OLD_PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH% +for /f "tokens=*" %%i in ( 'grep -i "version" python/google/protobuf/__init__.py ^| grep -o "'.*'"' ) do set BUILD_VERSION=%%i +set BUILD_COMMIT=v%BUILD_VERSION:'=% + +REM Fetch multibuild +git clone https://github.com/matthew-brett/multibuild.git + +REM Install zlib +mkdir zlib +curl -L -o zlib.zip http://www.winimage.com/zLibDll/zlib123dll.zip +curl -L -o zlib-src.zip http://www.winimage.com/zLibDll/zlib123.zip +7z x zlib.zip -ozlib +7z x zlib-src.zip -ozlib\include +SET ZLIB_ROOT=%cd%\zlib +del /Q zlib.zip +del /Q zlib-src.zip + +REM Create directory for artifacts +SET ARTIFACT_DIR=%cd%\artifacts +mkdir %ARTIFACT_DIR% + +REM Build wheel + +SET PYTHON=C:\python35_32bit +SET PYTHON_VERSION=3.5 +SET PYTHON_ARCH=32 +CALL build_single_artifact.bat + +SET PYTHON=C:\python35 +SET PYTHON_VERSION=3.5 +SET PYTHON_ARCH=64 +CALL build_single_artifact.bat + +SET PYTHON=C:\python36_32bit +SET PYTHON_VERSION=3.6 +SET PYTHON_ARCH=32 +CALL build_single_artifact.bat + +SET PYTHON=C:\python36 +SET PYTHON_VERSION=3.6 +SET PYTHON_ARCH=64 +CALL build_single_artifact.bat diff --git a/kokoro/release/python/windows/build_single_artifact.bat b/kokoro/release/python/windows/build_single_artifact.bat new file mode 100644 index 0000000000000..9d58199289855 --- /dev/null +++ b/kokoro/release/python/windows/build_single_artifact.bat @@ -0,0 +1,62 @@ +setlocal + +if %PYTHON%==C:\python35_32bit set generator=Visual Studio 14 +if %PYTHON%==C:\python35_32bit set vcplatform=Win32 + +if %PYTHON%==C:\python35 set generator=Visual Studio 14 Win64 +if %PYTHON%==C:\python35 set vcplatform=x64 + +if %PYTHON%==C:\python36_32bit set generator=Visual Studio 14 +if %PYTHON%==C:\python36_32bit set vcplatform=Win32 + +if %PYTHON%==C:\python36 set generator=Visual Studio 14 Win64 +if %PYTHON%==C:\python36 set vcplatform=x64 + +REM Prepend newly installed Python to the PATH of this build (this cannot be +REM done from inside the powershell script as it would require to restart +REM the parent CMD process). +SET PATH=%PYTHON%;%PYTHON%\Scripts;%OLD_PATH% +python -m pip install -U pip +pip install wheel + +REM Check that we have the expected version and architecture for Python +python --version +python -c "import struct; print(struct.calcsize('P') * 8)" + +rmdir /s/q protobuf +git clone https://github.com/google/protobuf.git + +REM Checkout release commit +cd %REPO_DIR% +git checkout %BUILD_COMMIT% + +REM ====================== +REM Build Protobuf Library +REM ====================== + +mkdir src\.libs + +mkdir vcprojects +pushd vcprojects +cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=OFF ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform% /p:Configuration=Release +dir /s /b +popd +copy vcprojects\Release\libprotobuf.lib src\.libs\libprotobuf.a +copy vcprojects\Release\libprotobuf-lite.lib src\.libs\libprotobuf-lite.a +SET PATH=%cd%\vcprojects\Release;%PATH% +dir vcprojects\Release + +REM ====================== +REM Build python library +REM ====================== + +cd python + +REM sed -i 's/\ extra_compile_args\ =\ \[\]/\ extra_compile_args\ =\ \[\'\/MT\'\]/g' setup.py + +python setup.py bdist_wheel --cpp_implementation --compile_static_extension +dir dist +copy dist\* %ARTIFACT_DIR% +dir %ARTIFACT_DIR% +cd ..\.. diff --git a/kokoro/release/python/windows/release.cfg b/kokoro/release/python/windows/release.cfg new file mode 100644 index 0000000000000..5261606477ab1 --- /dev/null +++ b/kokoro/release/python/windows/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Windows protoc release builds +build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} From d2c35ce4cb6ff767712a879f6bfad9d50ada577d Mon Sep 17 00:00:00 2001 From: Jozef Izso Date: Wed, 15 Aug 2018 15:50:45 +0200 Subject: [PATCH 11/21] Backport the PR #4579 to 3.6.x branch Implements #3380 --- Makefile.am | 1 + cmake/CMakeLists.txt | 11 +++++++++ cmake/libprotobuf-lite.cmake | 8 ++++++- cmake/libprotobuf.cmake | 8 ++++++- cmake/libprotoc.cmake | 2 +- cmake/protoc.cmake | 9 +++++++- cmake/version.rc.in | 45 ++++++++++++++++++++++++++++++++++++ 7 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 cmake/version.rc.in diff --git a/Makefile.am b/Makefile.am index ae4ac858cd291..8334db6095490 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1036,6 +1036,7 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ cmake/protobuf.pc.cmake \ cmake/protoc.cmake \ cmake/tests.cmake \ + cmake/version.rc.in \ editors/README.txt \ editors/proto.vim \ editors/protobuf-mode.el \ diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 02174e96aa3a0..ada9dc25533c6 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -166,12 +166,23 @@ if (MSVC) add_definitions(/bigobj) string(REPLACE "/" "\\" PROTOBUF_SOURCE_WIN32_PATH ${protobuf_SOURCE_DIR}) string(REPLACE "/" "\\" PROTOBUF_BINARY_WIN32_PATH ${protobuf_BINARY_DIR}) + string(REPLACE "." "," protobuf_RC_FILEVERSION "${protobuf_VERSION}") configure_file(extract_includes.bat.in extract_includes.bat) # Suppress linker warnings about files with no symbols defined. set(CMAKE_STATIC_LINKER_FLAGS /ignore:4221) + + # Configure Resource Compiler + enable_language(RC) + # use English language (0x409) in resource compiler + set(rc_flags "/l0x409") + # fix rc.exe invocations because of usage of add_definitions() + set(CMAKE_RC_COMPILE_OBJECT " ${rc_flags} /fo ") + + configure_file(version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY) endif (MSVC) + get_filename_component(protobuf_source_dir ${protobuf_SOURCE_DIR} PATH) include_directories( diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake index 911fce5e5d348..82b954a1dd9f5 100644 --- a/cmake/libprotobuf-lite.cmake +++ b/cmake/libprotobuf-lite.cmake @@ -48,8 +48,14 @@ set(libprotobuf_lite_includes ${protobuf_source_dir}/src/google/protobuf/wire_format_lite.h ) +if (MSVC) +set(libprotoc_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc +) +endif() + add_library(libprotobuf-lite ${protobuf_SHARED_OR_STATIC} - ${libprotobuf_lite_files} ${libprotobuf_lite_includes}) + ${libprotobuf_lite_files} ${libprotobuf_lite_includes} ${libprotobuf_lite_rc_files}) target_link_libraries(libprotobuf-lite ${CMAKE_THREAD_LIBS_INIT}) target_include_directories(libprotobuf-lite PUBLIC ${protobuf_source_dir}/src) if(MSVC AND protobuf_BUILD_SHARED_LIBS) diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index 65d05c194453b..463d65a382fb1 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -112,8 +112,14 @@ set(libprotobuf_includes ${protobuf_source_dir}/src/google/protobuf/wrappers.pb.h ) +if (MSVC) +set(libprotoc_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc +) +endif() + add_library(libprotobuf ${protobuf_SHARED_OR_STATIC} - ${libprotobuf_lite_files} ${libprotobuf_files} ${libprotobuf_includes}) + ${libprotobuf_lite_files} ${libprotobuf_files} ${libprotobuf_includes} ${libprotobuf_rc_files}) target_link_libraries(libprotobuf ${CMAKE_THREAD_LIBS_INIT}) if(protobuf_WITH_ZLIB) target_link_libraries(libprotobuf ${ZLIB_LIBRARIES}) diff --git a/cmake/libprotoc.cmake b/cmake/libprotoc.cmake index 58568bb2fc009..92dfd306e3cd6 100644 --- a/cmake/libprotoc.cmake +++ b/cmake/libprotoc.cmake @@ -168,7 +168,7 @@ set(libprotoc_rc_files endif() add_library(libprotoc ${protobuf_SHARED_OR_STATIC} - ${libprotoc_files} ${libprotoc_headers}) + ${libprotoc_files} ${libprotoc_headers} ${libprotoc_rc_files}) target_link_libraries(libprotoc libprotobuf) if(MSVC AND protobuf_BUILD_SHARED_LIBS) target_compile_definitions(libprotoc diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake index 5777b16ed21d2..9bf6f5a9298dc 100644 --- a/cmake/protoc.cmake +++ b/cmake/protoc.cmake @@ -2,6 +2,13 @@ set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc ) -add_executable(protoc ${protoc_files}) +set(protoc_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc +) + +add_executable(protoc ${protoc_files} ${protoc_rc_files}) target_link_libraries(protoc libprotobuf libprotoc) add_executable(protobuf::protoc ALIAS protoc) + +set_target_properties(protoc PROPERTIES + VERSION ${protobuf_VERSION}) diff --git a/cmake/version.rc.in b/cmake/version.rc.in new file mode 100644 index 0000000000000..cbce1e53891c9 --- /dev/null +++ b/cmake/version.rc.in @@ -0,0 +1,45 @@ +#define VS_FF_DEBUG 0x1L +#define VS_VERSION_INFO 0x1L +#define VS_FFI_FILEFLAGSMASK 0x17L +#define VER_PRIVATEBUILD 0x0L +#define VER_PRERELEASE 0x0L +#define VOS__WINDOWS32 0x4L +#define VFT_DLL 0x2L +#define VFT2_UNKNOWN 0x0L + +#ifndef DEBUG +#define VER_DEBUG 0 +#else +#define VER_DEBUG VS_FF_DEBUG +#endif + + +VS_VERSION_INFO VERSIONINFO + FILEVERSION @protobuf_RC_FILEVERSION@,0 + PRODUCTVERSION @protobuf_RC_FILEVERSION@,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK + FILEFLAGS VER_DEBUG + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL +BEGIN + BLOCK "VarFileInfo" + BEGIN + // English language (0x409) and the Windows Unicode codepage (1200) + VALUE "Translation", 0x409, 1200 + END + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "Compiled with @CMAKE_CXX_COMPILER_ID@ @CMAKE_CXX_COMPILER_VERSION@\0" + VALUE "ProductVersion", "@protobuf_VERSION@\0" + VALUE "FileVersion", "@protobuf_VERSION@\0" + VALUE "InternalName", "protobuf\0" + VALUE "ProductName", "Protocol Buffers - Google's Data Interchange Format\0" + VALUE "CompanyName", "Google Inc.\0" + VALUE "LegalCopyright", "Copyright 2008 Google Inc. All rights reserved.\0" + VALUE "Licence", "BSD\0" + VALUE "Info", "https://developers.google.com/protocol-buffers/\0" + END + END +END From 33d41628c2d06f3d51764c08caaf7e9ecc305065 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 20 Jan 2017 15:39:34 -0800 Subject: [PATCH 12/21] Set LIBPROTOBUF_EXPORT on GzipOutputStream::Options This fixes issue #2610. This was originally fixed in a101fa52895fc2ad83d8b5d610243531b1608a08 , but got reverted by d36c0c538a545fac5d9db6ba65c525246d4efa95 --- src/google/protobuf/io/gzip_stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/io/gzip_stream.h b/src/google/protobuf/io/gzip_stream.h index 15b02fe3d7dc7..df1a446ebc16f 100644 --- a/src/google/protobuf/io/gzip_stream.h +++ b/src/google/protobuf/io/gzip_stream.h @@ -118,7 +118,7 @@ class LIBPROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream { ZLIB = 2, }; - struct Options { + struct LIBPROTOBUF_EXPORT Options { // Defaults to GZIP. Format format; From fd25c5617b2a1f1ee875bf6c1e7518473409a211 Mon Sep 17 00:00:00 2001 From: Nehal J Wani Date: Sun, 1 Jul 2018 14:52:25 -0500 Subject: [PATCH 13/21] Set LIBPROTOBUF_EXPORT on GenericTypeHandler This should fix https://github.com/google/protobuf/issues/4717 --- src/google/protobuf/repeated_field.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index b47ea99484e0c..fbfbf94282433 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -705,13 +705,13 @@ void GenericTypeHandler::Merge(const string& from, // Declarations of the specialization as we cannot define them here, as the // header that defines ProtocolMessage depends on types defined in this header. #define DECLARE_SPECIALIZATIONS_FOR_BASE_PROTO_TYPES(TypeName) \ - template<> \ + template<> LIBPROTOBUF_EXPORT \ TypeName* GenericTypeHandler::NewFromPrototype( \ const TypeName* prototype, google::protobuf::Arena* arena); \ - template<> \ + template<> LIBPROTOBUF_EXPORT \ google::protobuf::Arena* GenericTypeHandler::GetArena( \ TypeName* value); \ - template<> \ + template<> LIBPROTOBUF_EXPORT \ void* GenericTypeHandler::GetMaybeArenaPointer( \ TypeName* value); From 740ff022e2dc2ad7618380e4751dacbcb18ad4be Mon Sep 17 00:00:00 2001 From: Frank Gasdorf Date: Tue, 13 Nov 2018 01:56:22 +0100 Subject: [PATCH 14/21] Mark sun.misc dependency as optional (#5249) --- java/core/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/core/pom.xml b/java/core/pom.xml index 067749c670958..0c0fd8ff0ffec 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -135,6 +135,7 @@ https://developers.google.com/protocol-buffers/ com.google.protobuf com.google.protobuf;version=${project.version} + sun.misc;resolution:=optional,* From 046e8fb7483df4e4fba028b8e85f68241a08f7f4 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 21 Nov 2018 11:00:45 -0800 Subject: [PATCH 15/21] Cherry-picked Bazel fix from PR #5085 (#5382) * use flag_values = {"@bazel_tools//tools/cpp:compiler": "x"} for compiler config_settings * Create compiler_config_setting.bzl --- BUILD | 7 +++---- compiler_config_setting.bzl | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 compiler_config_setting.bzl diff --git a/BUILD b/BUILD index 19f07c548d970..002ac5fe674a6 100644 --- a/BUILD +++ b/BUILD @@ -39,10 +39,9 @@ COPTS = select({ ], }) -config_setting( - name = "msvc", - values = { "compiler": "msvc-cl" }, -) +load(":compiler_config_setting.bzl", "create_compiler_config_setting") + +create_compiler_config_setting(name = "msvc", value = "msvc-cl") config_setting( name = "android", diff --git a/compiler_config_setting.bzl b/compiler_config_setting.bzl new file mode 100644 index 0000000000000..5e52a6524ffe5 --- /dev/null +++ b/compiler_config_setting.bzl @@ -0,0 +1,21 @@ +"""Creates config_setting that allows selecting based on 'compiler' value.""" + +def create_compiler_config_setting(name, value): + # The "do_not_use_tools_cpp_compiler_present" attribute exists to + # distinguish between older versions of Bazel that do not support + # "@bazel_tools//tools/cpp:compiler" flag_value, and newer ones that do. + # In the future, the only way to select on the compiler will be through + # flag_values{"@bazel_tools//tools/cpp:compiler"} and the else branch can + # be removed. + if hasattr(cc_common, "do_not_use_tools_cpp_compiler_present"): + native.config_setting( + name = name, + flag_values = { + "@bazel_tools//tools/cpp:compiler": value, + }, + ) + else: + native.config_setting( + name = name, + values = {"compiler": value}, + ) From af4810663bf1fb2388e88ad9038b80551aefd6da Mon Sep 17 00:00:00 2001 From: Ittai Zeidman Date: Mon, 21 May 2018 23:48:10 +0300 Subject: [PATCH 16/21] remove PACKAGE_NAME and REPOSITORY_NAME deprecated usage (#4650) --- protobuf.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf.bzl b/protobuf.bzl index 78f19c621a364..4226a1424c1b1 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -266,8 +266,8 @@ def internal_gen_well_known_protos_java(srcs): Args: srcs: the well known protos """ - root = Label("%s//protobuf_java" % (REPOSITORY_NAME)).workspace_root - pkg = PACKAGE_NAME + "/" if PACKAGE_NAME else "" + root = Label("%s//protobuf_java" % (native.repository_name())).workspace_root + pkg = native.package_name() + "/" if native.package_name() else "" if root == "": include = " -I%ssrc " % pkg else: From 539bc017a62f91bdf7c547b58948cb5a2f59d918 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Thu, 12 Jul 2018 10:58:10 -0700 Subject: [PATCH 17/21] Add Python 3.7 compatibility (#4862) Compilation of Python wrappers fails with Python 3.7 because the Python folks changed their C API such that PyUnicode_AsUTF8AndSize() now returns a const char* rather than a char*. Add a patch to work around. Relates #4086. --- python/google/protobuf/pyext/descriptor.cc | 2 +- python/google/protobuf/pyext/descriptor_containers.cc | 2 +- python/google/protobuf/pyext/descriptor_pool.cc | 2 +- python/google/protobuf/pyext/extension_dict.cc | 2 +- python/google/protobuf/pyext/message.cc | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 8af0cb1289f86..19a1c38a62e12 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -56,7 +56,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc index bc007f7efa551..0153664f504f2 100644 --- a/python/google/protobuf/pyext/descriptor_containers.cc +++ b/python/google/protobuf/pyext/descriptor_containers.cc @@ -66,7 +66,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index 95882aeb35a9a..962accc6e9b4d 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -48,7 +48,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index 018b5c2c49777..174c5470c271f 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -53,7 +53,7 @@ #endif #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 5893533adf5d9..31094b7e1003d 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -79,7 +79,7 @@ (PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AsString(ob)) #define PyString_AsStringAndSize(ob, charpp, sizep) \ (PyUnicode_Check(ob)? \ - ((*(charpp) = PyUnicode_AsUTF8AndSize(ob, (sizep))) == NULL? -1: 0): \ + ((*(charpp) = const_cast(PyUnicode_AsUTF8AndSize(ob, (sizep)))) == NULL? -1: 0): \ PyBytes_AsStringAndSize(ob, (charpp), (sizep))) #endif #endif @@ -1529,7 +1529,7 @@ PyObject* HasField(CMessage* self, PyObject* arg) { return NULL; } #else - field_name = PyUnicode_AsUTF8AndSize(arg, &size); + field_name = const_cast(PyUnicode_AsUTF8AndSize(arg, &size)); if (!field_name) { return NULL; } From 4bb130e7e65c3c5208f59419cef831274db2de9c Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Wed, 28 Nov 2018 16:45:16 -0800 Subject: [PATCH 18/21] Change MACOSX_DEPLOYMENT_TARGET to 10.9 (#5406) * Add kokoro build for python source package * Use libc++ for xcode 10 (#5303) The xcode 10 removes the deprecated libstdc++ library. We could set "MACOSX_DEPLOYMENT_TARGET" to "10.9" to use libc++ instead. * Add python 3.7 build * Add build for python 3.7 on linux and windows * Remove unused source build * Add comment * Fix $MACOSX_DEPLOYMENT_TARGET mismatch * Fix MACOSX_DEPLOYMENT_TARGET mismatch * Add missing import for sysconfig * Add missing imports --- kokoro/release/python/linux/build_artifacts.sh | 1 + kokoro/release/python/macos/build_artifacts.sh | 1 + .../release/python/windows/build_artifacts.bat | 10 ++++++++++ python/setup.py | 16 ++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/kokoro/release/python/linux/build_artifacts.sh b/kokoro/release/python/linux/build_artifacts.sh index 032e30d566304..0b30b271a0f26 100755 --- a/kokoro/release/python/linux/build_artifacts.sh +++ b/kokoro/release/python/linux/build_artifacts.sh @@ -43,3 +43,4 @@ build_artifact_version 2.7 build_artifact_version 3.4 build_artifact_version 3.5 build_artifact_version 3.6 +build_artifact_version 3.7 diff --git a/kokoro/release/python/macos/build_artifacts.sh b/kokoro/release/python/macos/build_artifacts.sh index 2aec5e64710d7..15d7091242ddc 100755 --- a/kokoro/release/python/macos/build_artifacts.sh +++ b/kokoro/release/python/macos/build_artifacts.sh @@ -45,3 +45,4 @@ build_artifact_version 2.7 build_artifact_version 3.4 build_artifact_version 3.5 build_artifact_version 3.6 +build_artifact_version 3.7 diff --git a/kokoro/release/python/windows/build_artifacts.bat b/kokoro/release/python/windows/build_artifacts.bat index 86270e6fb6e09..c2f6a2c1ea9a6 100644 --- a/kokoro/release/python/windows/build_artifacts.bat +++ b/kokoro/release/python/windows/build_artifacts.bat @@ -51,3 +51,13 @@ SET PYTHON=C:\python36 SET PYTHON_VERSION=3.6 SET PYTHON_ARCH=64 CALL build_single_artifact.bat + +SET PYTHON=C:\python37_32bit +SET PYTHON_VERSION=3.7 +SET PYTHON_ARCH=32 +CALL build_single_artifact.bat + +SET PYTHON=C:\python37 +SET PYTHON_VERSION=3.7 +SET PYTHON_ARCH=64 +CALL build_single_artifact.bat diff --git a/python/setup.py b/python/setup.py index a9df075e7becd..d6d3e1989883d 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,10 +1,14 @@ #! /usr/bin/env python # # See README for usage instructions. +from distutils import util import glob import os +import pkg_resources +import re import subprocess import sys +import sysconfig import platform # We must use setuptools, not distutils, because we need to use the @@ -186,6 +190,18 @@ def get_option_from_sys_argv(option_str): extra_compile_args.append('-Wno-invalid-offsetof') extra_compile_args.append('-Wno-sign-compare') + # https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes + # C++ projects must now migrate to libc++ and are recommended to set a + # deployment target of macOS 10.9 or later, or iOS 7 or later. + if sys.platform == 'darwin': + mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if mac_target and (pkg_resources.parse_version(mac_target) < + pkg_resources.parse_version('10.9.0')): + os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' + os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( + r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.9-\1', + util.get_platform()) + # https://github.com/Theano/Theano/issues/4926 if sys.platform == 'win32': extra_compile_args.append('-D_hypot=hypot') From 365e540d055636ae458f00a7d80959f9e9798e56 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 2 May 2017 17:13:27 -0700 Subject: [PATCH 19/21] Removed mention of Buffer in byteSourceToUint8Array The Closure compiler complains about Buffer since that class exists only in Node. That logic does not seem to be needed (unit tests and conformance tests pass without it), so let's just remove it to solve the problem. --- js/binary/utils.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/js/binary/utils.js b/js/binary/utils.js index 55a9ccd4ceff0..87570ff81c3f7 100644 --- a/js/binary/utils.js +++ b/js/binary/utils.js @@ -971,10 +971,6 @@ jspb.utils.byteSourceToUint8Array = function(data) { return /** @type {!Uint8Array} */(new Uint8Array(data)); } - if (data.constructor === Buffer) { - return /** @type {!Uint8Array} */(new Uint8Array(data)); - } - if (data.constructor === Array) { data = /** @type {!Array} */(data); return /** @type {!Uint8Array} */(new Uint8Array(data)); From 3cb0aeca93f0afd5aa05fa83eaf85f4820f465e5 Mon Sep 17 00:00:00 2001 From: David Ostrovsky Date: Fri, 7 Dec 2018 21:53:18 +0100 Subject: [PATCH 20/21] Revert "Removed mention of Buffer in byteSourceToUint8Array" This reverts commit 365e540d055636ae458f00a7d80959f9e9798e56. --- js/binary/utils.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/js/binary/utils.js b/js/binary/utils.js index 87570ff81c3f7..55a9ccd4ceff0 100644 --- a/js/binary/utils.js +++ b/js/binary/utils.js @@ -971,6 +971,10 @@ jspb.utils.byteSourceToUint8Array = function(data) { return /** @type {!Uint8Array} */(new Uint8Array(data)); } + if (data.constructor === Buffer) { + return /** @type {!Uint8Array} */(new Uint8Array(data)); + } + if (data.constructor === Array) { data = /** @type {!Array} */(data); return /** @type {!Uint8Array} */(new Uint8Array(data)); From da775bcb3825f3c8a41f10a332cfece5830eb881 Mon Sep 17 00:00:00 2001 From: David Ostrovsky Date: Fri, 7 Dec 2018 21:56:56 +0100 Subject: [PATCH 21/21] utils.js: Fix closure compiler error Apply changes from this commit: "Integrated internal changes from Google" 13fd045dbb2b4dacea32be162a41d5a4b0d1802f --- js/binary/utils.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/js/binary/utils.js b/js/binary/utils.js index 55a9ccd4ceff0..99283c2c9634c 100644 --- a/js/binary/utils.js +++ b/js/binary/utils.js @@ -971,9 +971,10 @@ jspb.utils.byteSourceToUint8Array = function(data) { return /** @type {!Uint8Array} */(new Uint8Array(data)); } - if (data.constructor === Buffer) { - return /** @type {!Uint8Array} */(new Uint8Array(data)); - } + if (typeof Buffer != 'undefined' && data.constructor === Buffer) { + return /** @type {!Uint8Array} */ ( + new Uint8Array(/** @type {?} */ (data))); + } if (data.constructor === Array) { data = /** @type {!Array} */(data);