Skip to content

Pass native build args from top-level scripts #2071

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
janvorli merged 7 commits into from
Jan 30, 2020
Merged

Pass native build args from top-level scripts #2071

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a6c70cc
Pass native build args from top-level scripts
am11 Jan 23, 2020
9b4625c
Remove obsolete message; switch platform/arch order
am11 Jan 23, 2020
fc0743e
Move export to a separate line
am11 Jan 23, 2020
0b1f7ab
Remove unnecessary brace
am11 Jan 23, 2020
8da5aca
Purge debug line
am11 Jan 23, 2020
593e300
Use URL-encoding to escape msbuild CMakeArgs property
am11 Jan 30, 2020
c645da2
Revert changes in pal_networkchange.c
am11 Jan 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion eng/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ usage()
echo " --configuration <value> Build configuration: Debug or Release (short: -c)"
echo " --verbosity <value> MSBuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
echo " --binaryLog Output binary log (short: -bl)"
echo " --cross Optional argument to signify cross compilation"
echo " --help Print help and exit (short: -h)"
echo ""

Expand All @@ -45,11 +46,20 @@ usage()
echo " --testscope Test scope, allowed values: innerloop, outerloop, all"
echo " --allconfigurations Build packages for all build configurations"
echo ""

echo "Native build settings:"
echo " --clang Optional argument to build using clang in PATH (default)"
echo " --clangx.y Optional argument to build using clang version x.y"
echo " --cmakeargs User-settable additional arguments passed to CMake."
echo " --gcc Optional argument to build using gcc in PATH (default)"
echo " --gccx.y Optional argument to build using gcc version x.y"

echo "Command line arguments starting with '/p:' are passed through to MSBuild."
echo "Arguments can also be passed in with a single hyphen."
}

arguments=''
cmakeargs=''
extraargs=''
build=false
buildtests=false
Expand Down Expand Up @@ -134,6 +144,22 @@ while [[ $# > 0 ]]; do
arguments="$arguments /p:LibrariesConfiguration=$2"
shift 2
;;
-cross)
arguments="$arguments /p:CrossBuild=True"
shift 1
;;
-clang*)
arguments="$arguments /p:Compiler=$opt"
shift 1
;;
-cmakeargs)
cmakeargs="${cmakeargs} ${opt} $2"
shift 2
;;
-gcc*)
arguments="$arguments /p:Compiler=$opt"
shift 1
;;
*)
ea=$1

Expand Down Expand Up @@ -165,5 +191,8 @@ if [ ${#actInt[@]} -eq 0 ]; then
arguments="-restore -build $arguments"
fi

arguments="$arguments $extraargs"
# URL-encode space (%20) to avoid quoting issues until the msbuild call in /eng/common/tools.sh.
# In *proj files (XML docs), URL-encoded string are rendered in their decoded form.
cmakeargs="${cmakeargs// /%20}"
arguments="$arguments /p:CMakeArgs=\"$cmakeargs\" $extraargs"
"$scriptroot/common/build.sh" $arguments
12 changes: 7 additions & 5 deletions eng/native/build-commons.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ isMSBuildOnNETCoreSupported()

if [[ ( "$__HostOS" == "Linux" ) && ( "$__HostArch" == "x64" || "$__HostArch" == "arm" || "$__HostArch" == "arm64" ) ]]; then
__IsMSBuildOnNETCoreSupported=1
elif [[ "$__HostArch" == "x64" && ( "$__HostOS" == "OSX" || "$__HostOS" == "FreeBSD" ) ]]; then
elif [[ ( "$__HostOS" == "OSX" || "$__HostOS" == "FreeBSD" ) && "$__HostArch" == "x64" ]]; then
__IsMSBuildOnNETCoreSupported=1
fi
}
Expand Down Expand Up @@ -213,7 +213,6 @@ case "$CPUName" in
;;

armv7l)
echo "Unsupported CPU $CPUName detected, build might not succeed!"
__BuildArch=arm
__HostArch=arm
;;
Expand Down Expand Up @@ -456,7 +455,8 @@ __CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:

# Configure environment if we are doing a verbose build
if [[ "$__VerboseBuild" == 1 ]]; then
export VERBOSE=1
VERBOSE=1
export VERBOSE
__CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed"
fi

Expand All @@ -466,9 +466,11 @@ fi

# Configure environment if we are doing a cross compile.
if [[ "$__CrossBuild" == 1 ]]; then
export CROSSCOMPILE=1
CROSSCOMPILE=1
export CROSSCOMPILE
if [[ ! -n "$ROOTFS_DIR" ]]; then
export ROOTFS_DIR="$__RepoRootDir/.tools/rootfs/$__BuildArch"
ROOTFS_DIR="$__RepoRootDir/.tools/rootfs/$__BuildArch"
export ROOTFS_DIR
fi
fi

Expand Down
18 changes: 12 additions & 6 deletions eng/native/init-distro-rid.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,12 @@ initNonPortableDistroRid()
fi

if [ -n "${nonPortableBuildID}" ]; then
export __DistroRid="${nonPortableBuildID}"
__DistroRid="${nonPortableBuildID}"

# We are using a non-portable build rid. Force __PortableBuild to false.
export __PortableBuild=0
__PortableBuild=0

export __DistroRid __PortableBuild
fi
}

Expand Down Expand Up @@ -125,7 +127,8 @@ initDistroRidGlobal()

if [ "$buildArch" = "armel" ]; then
# Armel cross build is Tizen specific and does not support Portable RID build
export __PortableBuild=0
__PortableBuild=0
export __PortableBuild
isPortable=0
fi

Expand All @@ -134,7 +137,8 @@ initDistroRidGlobal()
if [ -z "${__DistroRid}" ]; then
# The non-portable build rid was not set. Set the portable rid.

export __PortableBuild=1
__PortableBuild=1
export __PortableBuild
local distroRid=""

# Check for musl-based distros (e.g Alpine Linux, Void Linux).
Expand All @@ -152,7 +156,8 @@ initDistroRidGlobal()
fi
fi

export __DistroRid="${distroRid}"
__DistroRid="${distroRid}"
export __DistroRid
fi

if [ -z "$__DistroRid" ]; then
Expand All @@ -163,6 +168,7 @@ initDistroRidGlobal()
echo "__DistroRid: ${__DistroRid}"
echo "__RuntimeId: ${__DistroRid}"

export __RuntimeId="${__DistroRid}"
__RuntimeId="${__DistroRid}"
export __RuntimeId
fi
}
43 changes: 29 additions & 14 deletions src/coreclr/build-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ build_test_wrappers()
if [[ "$__BuildTestWrappers" -ne -0 ]]; then
echo "${__MsgPrefix}Creating test wrappers..."

export __Exclude="${__ProjectDir}/tests/issues.targets"
export __BuildLogRootName="Tests_XunitWrapper"
__Exclude="${__ProjectDir}/tests/issues.targets"
__BuildLogRootName="Tests_XunitWrapper"

export __Exclude __BuildLogRootName

buildVerbosity="Summary"

Expand Down Expand Up @@ -79,7 +81,8 @@ generate_layout()
fi

# Set up the directory for MSBuild debug logs.
export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
export MSBUILDDEBUGPATH

__BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"

Expand All @@ -101,7 +104,8 @@ generate_layout()
xUnitTestBinBase="$__TestWorkingDir"
fi

export CORE_ROOT="$xUnitTestBinBase"/Tests/Core_Root
CORE_ROOT="$xUnitTestBinBase"/Tests/Core_Root
export CORE_ROOT

if [[ -d "${CORE_ROOT}" ]]; then
rm -rf "$CORE_ROOT"
Expand Down Expand Up @@ -255,7 +259,9 @@ build_Tests()
fi
fi

export __CMakeBinDir="$__TestBinDir"
__CMakeBinDir="$__TestBinDir"
export __CMakeBinDir

if [[ ! -d "$__TestIntermediatesDir" ]]; then
mkdir -p "$__TestIntermediatesDir"
fi
Expand Down Expand Up @@ -291,7 +297,8 @@ build_Tests()
fi

# Set up the directory for MSBuild debug logs.
export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
export MSBUILDDEBUGPATH

__BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"

Expand Down Expand Up @@ -384,23 +391,26 @@ build_MSBuild_projects()
# See https://github.com/Microsoft/msbuild/issues/2993

# __SkipPackageRestore and __SkipTargetingPackBuild used to control build by tests/src/dirs.proj
export __SkipPackageRestore=false
export __SkipTargetingPackBuild=false
export __NumberOfTestGroups=3
__SkipPackageRestore=false
__SkipTargetingPackBuild=false
__NumberOfTestGroups=3

__AppendToLog=false

if [[ -n "$__priority1" ]]; then
export __NumberOfTestGroups=10
__NumberOfTestGroups=10
fi

export __SkipPackageRestore __SkipTargetingPackBuild __NumberOfTestGroups

for (( testGroupToBuild=1 ; testGroupToBuild <= __NumberOfTestGroups; testGroupToBuild = testGroupToBuild + 1 ))
do
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""

export __TestGroupToBuild="$testGroupToBuild"
__TestGroupToBuild="$testGroupToBuild"
export __TestGroupToBuild

# Generate build command
buildArgs=("$projectName")
Expand All @@ -427,8 +437,11 @@ build_MSBuild_projects()
echo " $__BuildErr"
exit 1
fi
export __SkipPackageRestore=true
export __SkipTargetingPackBuild=true

__SkipPackageRestore=true
__SkipTargetingPackBuild=true
export __SkipPackageRestore __SkipTargetingPackBuild

__AppendToLog=true
done
else
Expand Down Expand Up @@ -538,7 +551,9 @@ __CodeCoverage=
__IncludeTests=INCLUDE_TESTS

# Set the various build properties here so that CMake and MSBuild can pick them up
export __ProjectDir="$__ProjectRoot"
__ProjectDir="$__ProjectRoot"
export __ProjectDir

__BuildTestWrappers=1
__BuildTestWrappersOnly=
__Compiler=clang
Expand Down
26 changes: 17 additions & 9 deletions src/coreclr/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,13 +131,15 @@ build_cross_architecture_components()
return
fi

export __CMakeBinDir="$crossArchBinDir"
export CROSSCOMPILE=0
__CMakeBinDir="$crossArchBinDir"
CROSSCOMPILE=0
export __CMakeBinDir CROSSCOMPILE

__CMakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CROSS_COMPONENTS_BUILD=1 $__CMakeArgs"
build_native "$__CrossArch" "$__ProjectRoot" "$__ProjectRoot" "$intermediatesForBuild" "cross-architecture components"

export CROSSCOMPILE=1
CROSSCOMPILE=1
export CROSSCOMPILE
}

build_CoreLib_ni()
Expand All @@ -146,7 +148,8 @@ build_CoreLib_ni()
local __CoreLibILDir=$2

if [[ "$__PartialNgen" == 1 ]]; then
export COMPlus_PartialNGen=1
COMPlus_PartialNGen=1
export COMPlus_PartialNGen
fi

if [[ -e "$__CrossGenCoreLibLog" ]]; then
Expand Down Expand Up @@ -469,8 +472,10 @@ __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
# Set the remaining variables based upon the determined build configuration
__BinDir="$__RootBinDir/bin/coreclr/$__BuildOS.$__BuildArch.$__BuildType"
__PackagesBinDir="$__BinDir/.nuget"
export __IntermediatesDir="$__RootBinDir/obj/coreclr/$__BuildOS.$__BuildArch.$__BuildType"
export __ArtifactsIntermediatesDir="$__RepoRootDir/artifacts/obj/coreclr"
__IntermediatesDir="$__RootBinDir/obj/coreclr/$__BuildOS.$__BuildArch.$__BuildType"
__ArtifactsIntermediatesDir="$__RepoRootDir/artifacts/obj/coreclr"
export __IntermediatesDir __ArtifactsIntermediatesDir

__CrossComponentBinDir="$__BinDir"

__CrossArch="$__HostArch"
Expand All @@ -485,19 +490,22 @@ if [[ -z "$HOME" ]]; then
if [[ ! -d "$__ProjectDir/temp_home" ]]; then
mkdir temp_home
fi
export HOME="$__ProjectDir"/temp_home
HOME="$__ProjectDir"/temp_home
export HOME
echo "HOME not defined; setting it to $HOME"
fi

# Specify path to be set for CMAKE_INSTALL_PREFIX.
# This is where all built CoreClr libraries will copied to.
export __CMakeBinDir="$__BinDir"
__CMakeBinDir="$__BinDir"
export __CMakeBinDir

# Make the directories necessary for build if they don't exist
setup_dirs_local

# Set up the directory for MSBuild debug logs.
export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
export MSBUILDDEBUGPATH

# Check prereqs.
check_prereqs
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/coreclr.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@
<ItemGroup>
<_CoreClrBuildArg Condition="$([MSBuild]::IsOsPlatform(Windows))" Include="-skiptests" />
<_CoreClrBuildArg Condition="'$(TargetArchitecture)' != ''" Include="-$(TargetArchitecture)" />
<_CoreClrBuildArg Include="$(CMakeArgs)" />
<_CoreClrBuildArg Include="-$(Configuration.ToLower())" />
<_CoreClrBuildArg Include="$(Compiler)" />
<_CoreClrBuildArg Condition="'$(ContinuousIntegrationBuild)' == 'true'" Include="-ci" />
<_CoreClrBuildArg Condition="'$(CrossBuild)' == 'true'" Include="-cross" />
Copy link
Member Author

@am11 am11 Jan 23, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[IMO]
Now that all native build scripts share common arguments via:

runtime/eng/native/build-commons.sh

Lines 160 to 194 in e2325f3

usage()
{
echo "Usage: $0 <options>"
echo ""
echo "Common Options:"
echo ""
echo "BuildArch can be: -arm, -armel, -arm64, -armel, x64, x86, -wasm"
echo "BuildType can be: -debug, -checked, -release"
echo "-bindir: output directory (defaults to $__ProjectRoot/artifacts)"
echo "-ci: indicates if this is a CI build."
echo "-clang: optional argument to build using clang in PATH (default)."
echo "-clangx.y: optional argument to build using clang version x.y."
echo "-cmakeargs: user-settable additional arguments passed to CMake."
echo "-configureonly: do not perform any builds; just configure the build."
echo "-cross: optional argument to signify cross compilation,"
echo " will use ROOTFS_DIR environment variable if set."
echo "-gcc: optional argument to build using gcc in PATH."
echo "-gccx.y: optional argument to build using gcc version x.y."
echo "-msbuildonunsupportedplatform: build managed binaries even if distro is not officially supported."
echo "-ninja: target ninja instead of GNU make"
echo "-numproc: set the number of build processes."
echo "-portablebuild: pass -portablebuild=false to force a non-portable build."
echo "-skipconfigure: skip build configuration."
echo "-skipgenerateversion: disable version generation even if MSBuild is supported."
echo "-stripsymbols: skip native image generation."
echo "-verbose: optional argument to enable verbose build output."
echo ""
echo "Additional Options:"
echo ""
for i in "${!usage_list[@]}"; do
echo "${usage_list[${i}]}"
done
echo ""
exit 1
}
we can extract these argument construction lines from coreclr/coreclr.proj, installer/corehost/build.proj and libraries/Native/build-native.proj into a common .props file, to avoid three different approaches of doing exactly the same thing (coreclr: ItemGroup, installer: property concatenation and libraries: separate properties followed by concatenation).

Copy link
Member

@janvorli janvorli Jan 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc: @jkoritzinsky, @ViktorHofer

<_CoreClrBuildArg Condition="$([MSBuild]::IsOsPlatform(Windows)) and ('$(TargetArchitecture)' == 'x86' or '$(TargetArchitecture)' == 'x64') and '$(Configuration)' == 'Release'" Include="-enforcepgo" />
<_CoreClrBuildArg Condition="!$([MSBuild]::IsOsPlatform(Windows)) and '$(Configuration)' == 'Release'" Include="-stripsymbols" />
<_CoreClrBuildArg Condition="'$(OfficialBuildId)' != ''" Include="-officialbuildid=$(OfficialBuildId)" />
Expand Down
2 changes: 2 additions & 0 deletions src/installer/corehost/build.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
<BuildArgs>$(ConfigurationGroup) $(TargetArchitecture) -apphostver "$(AppHostVersion)" -hostver "$(HostVersion)" -fxrver "$(HostResolverVersion)" -policyver "$(HostPolicyVersion)" -commithash "$(LatestCommit)"</BuildArgs>
<BuildArgs Condition="'$(PortableBuild)' != 'true'">$(BuildArgs) -portablebuild=false</BuildArgs>
<BuildArgs Condition="'$(CrossBuild)' == 'true'">$(BuildArgs) -cross</BuildArgs>
<BuildArgs Condition="'$(Compiler)' != ''">$(BuildArgs) $(Compiler)</BuildArgs>
<BuildArgs Condition="'$(CMakeArgs)' != ''">$(BuildArgs) $(CMakeArgs)</BuildArgs>
<BuildArgs Condition="'$(StripSymbols)' == 'true'">$(BuildArgs) -stripsymbols</BuildArgs>
</PropertyGroup>

Expand Down
2 changes: 1 addition & 1 deletion src/installer/corehost/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ __IntermediatesDir="$__RootBinDir/obj/$__DistroRidLower.$__BuildType"

export __BinDir __IntermediatesDir

__CMakeArgs="-DCLI_CMAKE_HOST_VER=\"$__host_ver\" -DCLI_CMAKE_COMMON_HOST_VER=\"$__apphost_ver\" -DCLI_CMAKE_HOST_FXR_VER=\"$__fxr_ver\""
__CMakeArgs="-DCLI_CMAKE_HOST_VER=\"$__host_ver\" -DCLI_CMAKE_COMMON_HOST_VER=\"$__apphost_ver\" -DCLI_CMAKE_HOST_FXR_VER=\"$__fxr_ver\" $__CMakeArgs"
__CMakeArgs="-DCLI_CMAKE_HOST_POLICY_VER=\"$__policy_ver\" -DCLI_CMAKE_PKG_RID=\"$__DistroRid\" -DCLI_CMAKE_COMMIT_HASH=\"$__commit_hash\" $__CMakeArgs"

if [[ "$__PortableBuild" == 1 ]]; then
Expand Down
3 changes: 2 additions & 1 deletion src/libraries/Native/build-native.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,14 @@
<_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true' and '$(OSGroup)' != 'WebAssembly'"> -stripsymbols</_StripSymbolsArg>
<_PortableBuildArg Condition="'$(PortableBuild)' != 'true'"> -portablebuild=false</_PortableBuildArg>
<_CrossBuildArg Condition="'$(CrossBuild)' == 'true'"> -cross</_CrossBuildArg>
<_CMakeArgs Condition="'$(CMakeArgs)' != ''"> $(CMakeArgs)</_CMakeArgs>

<!--
BuildNativeCompiler is a pass-through argument, to pass an argument to build-native.sh. It is intended to be
used to force a specific compiler toolset.
-->
<_BuildNativeCompilerArg Condition="'$(BuildNativeCompiler)' != ''"> $(BuildNativeCompiler)</_BuildNativeCompilerArg>
<_BuildNativeUnixArgs>$(_BuildNativeArgs)$(_ProcessCountArg)$(_StripSymbolsArg)$(_PortableBuildArg)$(_CrossBuildArg)$(_BuildNativeCompilerArg)</_BuildNativeUnixArgs>
<_BuildNativeUnixArgs>$(_BuildNativeArgs)$(_ProcessCountArg)$(_StripSymbolsArg)$(_PortableBuildArg)$(_CrossBuildArg)$(_BuildNativeCompilerArg)$(_CMakeArgs) $(Compiler)</_BuildNativeUnixArgs>
</PropertyGroup>

<Message Text="$(MSBuildProjectDirectory)/build-native.sh $(_BuildNativeUnixArgs)" Importance="High"/>
Expand Down