Switch Bootstrapping model to use Arcade

Summary of changes:
- Update to latest version of .NET SDK
- Delete all the root level wrapper scripts
- Switch bootstrapping model to pull in arcade scripts
- Update documentation
- Update CI scripts
- Update official build definitions

BuildToolsVersion.txt and DotnetCLIVersion.txt are no longer used
and instead the cli version is in global.json and the BuildTools
version is in dependencies.props for the time being until we eliminate
it completely and move everything to arcade.

Root build scripts now call into eng/common build scripts. With wrapper
scripts under eng to allow for custom argument alias given we have removed
run.exe.

Restructed Build.proj in the root to separte the different phases into
targets so we can call them individually through the arcade scripts or
our CI/official build definitions.

Wrapped native build into a build-native.proj file so we can trigger
it directly from msbuild in our normal build flow. We can still directly
call the native build-native.cmd/sh for bootstraping new platforms but
that isn't the default any longer for our builds.

CI and official builds no longer call sync and then build we just call
build directly and let it do all the phases.

Fixed issues with BuildTools were we need to define RootIntermediateOutputPath
explicitly now given in BuildTools it looks for init-tools.msbuild which no
longer exists.

Update the PackagesDir to point to NugetPackageRoot now so we can configure
the packages to use the machine wide cache or local repo cache.

Currently our docker build definitions don't presist the
user wide nuget cache so we need to pass --ci to get the packages
and tmp folder localized to the repo which is presisted across
commands.

Our manual shims were having issues so removed the default
references and reduced the amount of seeds passed to help
eliminate potential reference resolution issues.

Author: weshaggard

.gitignore

@@ -4,6 +4,8 @@ syntax: glob
 
 # Tool Runtime Dir
 /[Tt]ools/
+.dotnet/
+.packages/
 
 # User-specific files
 *.suo
@@ -12,6 +14,7 @@ syntax: glob
 *.sln.docstates
 
 # Build results
+artifacts/
 .idea/
 [Dd]ebug/
 [Dd]ebugPublic/

BuildToolsVersion.txt

@@ -36,6 +36,11 @@
     <IsPrerelease>true</IsPrerelease>
   </PropertyGroup>
 
+  <!-- Provides package dependency version properties and verification/auto-upgrade configuration -->
+  <Import Project="$(MSBuildThisFileDirectory)dependencies.props" />
+
+  <Import Project="$(ProjectDir)artifacts\toolset\Common\Tools.proj.nuget.g.props" Condition="Exists('$(ProjectDir)artifacts\toolset\Common\Tools.proj.nuget.g.props')" />
+
   <!-- Common repo directories -->
   <PropertyGroup>
     <ProjectDir>$(MSBuildThisFileDirectory)</ProjectDir>
@@ -46,24 +51,27 @@
 
     <!-- Output directories -->
     <BinDir Condition="'$(BinDir)'==''">$(ProjectDir)bin/</BinDir>
-    
+    <RootIntermediateOutputPath>$(MSBuildThisFileDirectory)bin/obj/</RootIntermediateOutputPath>
+
     <TestWorkingDir Condition="'$(TestWorkingDir)'==''">$(BinDir)tests/</TestWorkingDir>
     <PackageOutputRoot Condition="'$(PackageOutputRoot)'=='' and '$(NonShippingPackage)' == 'true'">$(BinDir)packages_noship/</PackageOutputRoot>
     <PackageOutputRoot Condition="'$(PackageOutputRoot)'==''">$(BinDir)packages/</PackageOutputRoot>
 
     <!-- Input Directories -->
     <PackagesDir>$(DotNetRestorePackagesPath)</PackagesDir>
+    <PackagesDir Condition="'$(PackagesDir)'==''">$(NuGetPackageRoot)/</PackagesDir>
     <PackagesDir Condition="'$(PackagesDir)'==''">$(ProjectDir)packages/</PackagesDir>
     <RestorePackagesPath Condition="'$(RestorePackagesPath)'==''">$(PackagesDir)</RestorePackagesPath>
-    <NuGetPackageRoot Condition="'$(NuGetPackageRoot)'==''">$(PackagesDir)</NuGetPackageRoot>
     <ToolsDir Condition="'$(ToolsDir)'==''">$(ProjectDir)Tools/</ToolsDir>
     <IlasmToolPath>$(ToolsDir)ilasm/ilasm</IlasmToolPath>
     <CodeAnalysisRuleset>$(MSBuildThisFileDirectory)CodeAnalysis.ruleset</CodeAnalysisRuleset>
-
-    <!-- Set up ToolHostCmd for Arcade targets -->
-    <ToolHostCmd Condition="'$(ToolHostCmd)'==''">"$(ToolsDir)dotnetcli/dotnet"</ToolHostCmd>
+    <!-- Respect environment variable for the .NET install directory if set; otherwise, use the current default location -->
+    <DotNetRoot Condition="'$(DotNetRoot)' == ''">$(DOTNET_INSTALL_DIR)</DotNetRoot>
+    <DotNetRoot Condition="'$(DotNetRoot)' == ''">$(ProjectDir).dotnet\</DotNetRoot>
+    <DotNetRoot Condition="!HasTrailingSlash('$(DotNetRoot)')">$(DotNetRoot)\</DotNetRoot>
+    <DotnetCliPath Condition="'$(DotnetCliPath)'==''">$(DotNetRoot)</DotnetCliPath>
+    <ToolHostCmd Condition="'$(ToolHostCmd)'==''">"$(DotNetRoot)dotnet"</ToolHostCmd>
   </PropertyGroup>
-
   <!-- Choose .targets files that come from Arcade rather than from buildtools -->
   <PropertyGroup>
     <ExcludeNotSupportedImport>true</ExcludeNotSupportedImport>
@@ -80,19 +88,17 @@
   </PropertyGroup>
 
   <!-- Default properties for CI builds -->
-  <!-- TODO: Remove obsolete condition when moved to arcade. -->
-  <PropertyGroup Condition="'$(IsCIBuild)' == 'true' OR '$(ContinuousIntegrationBuild)' == 'true'">
+  <PropertyGroup Condition="'$(IsTestProject)' == 'true' and '$(ContinuousIntegrationBuild)' == 'true' and '$(OfficialBuildId)' == ''">
     <WithoutCategories>IgnoreForCI</WithoutCategories>
     <EnableDumpling>true</EnableDumpling>
     <CrashDumpFolder Condition="'$(RunningOnUnix)' != 'true'">%TMP%\CoreRunCrashDumps</CrashDumpFolder>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(OfficialBuildId)' == ''">
     <!-- Disable F5 and test explorer support for CI builds. -->
     <GenerateLaunchSettings>false</GenerateLaunchSettings>
     <IncludeVSTestReferences>false</IncludeVSTestReferences>
   </PropertyGroup>
 
-  <!-- Provides package dependency version properties and verification/auto-upgrade configuration -->
-  <Import Project="$(MSBuildThisFileDirectory)dependencies.props" />
-
   <!-- list of nuget package sources passed to nuget restore -->
   <!-- Example to consume local CoreCLR package:
        /p:OverridePackageSource=C:\coreclr\bin\Product\Windows_NT.x64.Debug\.nuget\pkg
@@ -118,6 +124,7 @@
     <ApiCompatExcludeAttributeList>$(MSBuildThisFileDirectory)tools-local/DefaultGenApiDocIds.txt</ApiCompatExcludeAttributeList>
     <!-- Build as portable by default -->
     <PortableBuild Condition="'$(PortableBuild)' == ''">true</PortableBuild>
+    <Performance Condition="'$(Performance)' == ''">false</Performance>
     <!-- Used for launchSettings.json and runtime config files. -->
     <AppDesignerFolder>Properties</AppDesignerFolder>
   </PropertyGroup>
@@ -169,7 +176,7 @@
     <_runtimeOS Condition="'$(_runtimeOS)' == 'tizen.5.0.0'">linux</_runtimeOS>
     <_runtimeOS Condition="'$(PortableBuild)' == 'true'">$(_portableOS)</_runtimeOS>
     <ToolRuntimeRID>$(_runtimeOS)-x64</ToolRuntimeRID>
-    
+
     <!-- There are no WebAssembly tools, so treat them as Windows -->
     <ToolRuntimeRID Condition="'$(RuntimeOS)' == 'WebAssembly'">win-x64</ToolRuntimeRID>
     <MicrosoftNetCoreIlasmPackageRuntimeId Condition="'$(MicrosoftNetCoreIlasmPackageRuntimeId)' == ''">$(ToolRuntimeRID)</MicrosoftNetCoreIlasmPackageRuntimeId>
@@ -269,6 +276,7 @@
     <Features>strict</Features>
     <WarningLevel>4</WarningLevel>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <Deterministic>false</Deterministic>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(IsReferenceAssembly)' == 'true'">
@@ -284,9 +292,9 @@
 
   <!-- Set up the default output and intermediate paths -->
   <PropertyGroup>
-    <!-- 
-      Microsoft.NET.TargetFrameworkInference.targets appends the TargetFramework. 
-      We do this manually and use the TargetGroup instead. 
+    <!--
+      Microsoft.NET.TargetFrameworkInference.targets appends the TargetFramework.
+      We do this manually and use the TargetGroup instead.
     -->
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
 
@@ -302,7 +310,7 @@
     <OutputPath Condition="'$(OutputPath)'==''">$(BaseOutputPath)$(OSPlatformConfig)/$(MSBuildProjectName)/$(TargetOutputRelPath)$(OutputPathSubfolder)</OutputPath>
 
     <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'==''">$(RootIntermediateOutputPath)$(OSPlatformConfig)/$(MSBuildProjectName)/</BaseIntermediateOutputPath>
-    <IntermediateOutputPath Condition="'$(IntermediateOutputPath)' == ''">$(BaseIntermediateOutputPath)$(TargetOutputRelPath)</IntermediateOutputPath>    
+    <IntermediateOutputPath Condition="'$(IntermediateOutputPath)' == ''">$(BaseIntermediateOutputPath)$(TargetOutputRelPath)</IntermediateOutputPath>
 
     <RuntimePath Condition="'$(RuntimePath)' == ''">$(BinDir)runtime/$(BuildConfiguration)/</RuntimePath>
     <ShimsTargetRuntimeRoot>$(BinDir)shimsTargetRuntime/</ShimsTargetRuntimeRoot>
@@ -382,7 +390,7 @@
     <!-- Don't run tests if we're building another platform's binaries on Windows -->
     <SkipTests Condition="'$(SkipTests)'=='' and ('$(OsEnvironment)'=='Windows_NT' and '$(TargetsWindows)'!='true' and '$(OSGroup)'!='AnyOS')">true</SkipTests>
 
-    <!-- 
+    <!--
       Set these properties early as they are consumed by some test projects.
       TODO: Remove when https://github.com/dotnet/corefx/issues/32421 is fixed.
     -->

Directory.Build.props

@@ -125,10 +125,7 @@
 
   <Import Project="$(ToolsDir)/Build.Common.targets" Condition="Exists('$(ToolsDir)/Build.Common.targets')" />
 
-  <!-- Import Arcade targets -->
-  <Import Project="$(PackagesDir)/$(MicrosoftDotNetGenApiPackage.ToLower())/$(MicrosoftDotNetGenApiPackageVersion)/build/$(MicrosoftDotNetGenApiPackage).targets" />
-  <Import Project="$(PackagesDir)/$(MicrosoftDotNetGenFacadesPackage.ToLower())/$(MicrosoftDotNetGenFacadesPackageVersion)/build/$(MicrosoftDotNetGenFacadesPackage).targets" />
-  <Import Project="$(PackagesDir)/$(MicrosoftDotNetApiCompatPackage.ToLower())/$(MicrosoftDotNetApiCompatPackageVersion)/build/$(MicrosoftDotNetApiCompatPackage).targets" />
+  <Import Project="$(ProjectDir)artifacts\toolset\Common\Tools.proj.nuget.g.targets" Condition="Exists('$(ProjectDir)artifacts\toolset\Common\Tools.proj.nuget.g.targets')" />
 
   <!-- permit a wrapping build system to contribute targets to this build -->
   <Import Condition="Exists('$(MSBuildThisFileDirectory)..\open.targets')" Project="$(MSBuildThisFileDirectory)..\open.targets" />

Directory.Build.targets

@@ -42,13 +42,11 @@ You can navigate to this from your PR by clicking the "Details" link to the righ
 
 You can perform code coverage runs locally on your own machine.  Normally to build your entire CoreFX repo, from the root of your repo you'd run:
 
-    build
-    build-tests
+    build -includetests
 
-To include code coverage in this run, augment the `build-tests` call with the `coverage` argument:
+To include code coverage in this run, augment the `build -includetests` call with the `coverage` argument:
 
-    build
-    build-tests -coverage
+    build -includetests -coverage
 
 This will do the build and testing as with the normal ```build```, but it will run the tests using the OpenCover tool.  A resulting index.htm file providing the results of the run will be available at:
 
@@ -66,9 +64,9 @@ The results for this one library will then show up in the aforementioned index.h
 
     cd src\System.Diagnostics.Debug\tests\
     dotnet msbuild /t:BuildAndTest /p:Coverage=true
-    
+
 And then once the run completes:
-    
+
     ..\..\..\bin\tests\coverage\index.htm
 
 ## Code coverage with System.Private.CoreLib code
@@ -83,7 +81,7 @@ The build and test projects take care of copying assemblies and PDBs as needed f
 
 Note: as of 10/2017 OpenCover, the default coverage tool, requires PDBs to be Windows PDBs - the needed conversions are automatically performed by coverage runs. You can determine if it is a Windows PDB by doing 'more System.Private.CoreLib.pdb',  if it begins with 'Microsoft C/C++ MSF 7.00' it is a Windows PDB.  If you need a Windows PDB the Pdb2Pdb tool will convert (or you can do a dotnet msbuild /t:rebuild /p:DebugType=full).
 
-## Cross-platform Coverage 
+## Cross-platform Coverage
 As of 07/2018 CoreFx is only able to get coverage information on Windows. To correct this we are experimenting with [coverlet](https://github.com/tonerdo/coverlet).
 
 ### Know Issues ###
@@ -95,7 +93,7 @@ On Windows by default coverage runs will use OpenCover instead of coverlet, use
 
 ```
 dotnet msbuild /t:RebuildAndTest /p:Coverage=True /p:UseCoverlet=True
-``` 
+```
 
 ### Unix Instructions ###
 On Unix just specifying `/p:Coverage=True` triggers the usage of coverlet. However, in order to generate the html report a few setup steps are needed.

Documentation/building/code-coverage.md

@@ -78,7 +78,7 @@ The managed components of CoreFX are architecture-independent and thus do not re
 
 Many of the managed binaries are also OS-independent, e.g. System.Linq.dll, while some are OS-specific, e.g. System.IO.FileSystem.dll, with different builds for Windows and Linux.
 
-    lgs@ubuntu ~/git/corefx/ $ ./build-managed.sh -debug -verbose
+    lgs@ubuntu ~/git/corefx/ $ ./build.sh -debug /p:BuildNative=false
 
 The output is at `bin/[BuildConfiguration]` where `BuildConfiguration` looks something like `netcoreapp-<OSGroup>-Debug-<Architecture>`. Ex: `bin/netcoreapp-Linux-Debug-x64`. For more details on the build configurations see [project-guidelines](../coding-guidelines/project-guidelines.md)
 
@@ -112,10 +112,10 @@ When building for a new architecture you will need to build the native pieces se
 
 Example building for armel
 ```
-build-native.sh -buildArch=armel
+src/Native/build-native.sh armel
 --> Output goes to bin/runtime/netcoreapp-Linux-Debug-armel
 
-build-managed.sh -buildArch=x64
+build /p:ArchGroup=x64 /p:BuildNative=false
 --> Output goes to bin/runtime/netcoreapp-Linux-Debug-x64
 ```
 

Documentation/building/cross-building.md

@@ -38,7 +38,7 @@ The following are the minimum requirements:
   * .NET Framework 4.6 Targeting Pack
   * Windows Universal CRT SDK
   * VC++ 2015.3 v140 Toolset (x86, x64)
-  
+
 #### Visual Studio 2017 - Command line install
 
 If you've installed Visual Studio 2017 already, go to `C:\Program Files (x86)\Microsoft Visual Studio\Installer` and run
@@ -56,7 +56,6 @@ For the best possible experience make sure to have the latest version of Visual
 From a (non-admin) Command Prompt window:
 
 - `build.cmd` - Will cause basic tool initialization and build the default configuration for refs, libs, and packages.
-- `build-tests.cmd` - Will build and run tests for the default configuration.
 
 For information on different configurations see [project-guidelines](../coding-guidelines/project-guidelines.md).
 

Documentation/building/windows-instructions.md

@@ -19,11 +19,6 @@ The CoreFX repo can be built from a regular, non-admin command prompt. The build
 Developer Workflow
 ------------------
 The dev workflow describes the [development process](https://github.com/dotnet/buildtools/blob/master/Documentation/Dev-workflow.md) to follow. It is divided into specific tasks that are fast, transparent and easy to understand.
-The tasks are represented in scripts (cmd/sh) in the root of the repo:
-* clean - Cleans up the binary output and optionally the working directory (`-all`)
-* sync - Pulls down external dependencies needed to build (i.e. build tools, xunit, coreclr, etc)
-* build - Builds the shipping libraries in corefx.
-* build-tests - Builds and runs the corefx tests.
 
 For more information about the different options that each task has, use the argument `-?` when calling the script.  For example:
 ```
@@ -44,36 +39,41 @@ The build configurations are generally defaulted based on where you are building
 - `-debug|-release` controls the optimization level the compilers use for the build. It defaults to `Debug`. (msbuild property `ConfigurationGroup`)
 - `-buildArch` identifies the architecture for the build. It defaults to `x64` but possible values include `x64`, `x86`, `arm`, or `arm64`. (msbuild property `ArchGroup`)
 
-These options are common for build, build-managed, build-native, and build-tests scripts.
-
 For more details on the build configurations see [project-guidelines](../coding-guidelines/project-guidelines.md#build-pivots).
 
 **Note**: Before working on individual projects or test projects you **must** run `build` from the root once before beginning that work. It is also a good idea to run `build` whenever you pull a large set of unknown changes into your branch.
 
-**Common full clean build and test run**
+The most common workflow for developers is to call `build` from the root once (preceeded by a `clean -all` if you have built previously) and then go and work on the individual library that you are trying to make changes for. On windows folks will usually open up the solution file in the root of that library directory and work in VS.
+
+By default build only builds the product libraries and none of the tests if you want to build and run all the tests you can call `build -test` to build and run only the tests or `build -includetests` to build the project as well as the tests
+
+**Examples**
+- Clean and build the product libraries
 ```
 clean -all
 build
-build-tests
 ```
 
-**Examples**
-
 - Building in debug mode for platform x64
 ```
-build -debug -buildArch=x64
+build -debug /p:ArchGroup=x64
 ```
 
 - Building the src and then building and running the tests
 ```
-build -tests
+build -includetests
 ```
 
 - Building for different target frameworks
 ```
-build -framework=netcoreapp
-build -framework=netfx
-build -framework=uap
+build -framework netcoreapp
+build -framework netfx
+build -framework uap
+```
+
+- Build only managed components and skip the native build
+```
+build /p:BuildNative=false
 ```
 
 ### Build Native
@@ -87,50 +87,33 @@ The native component should be buildable on any system.
 
 - Building in debug mode for platform x64
 ```
-build-native -debug -buildArch=x64
+./src/Native/build-native debug x64
 ```
 
-- The following example shows the argument `--`. Everything that is after it is not going to be processed, and will be passed as-is.
+- The following example shows how you would do an arm cross-compile build.
 ```
-build-native -debug -buildArch=arm -- cross verbose
+./src/Native/build-native debug arm cross verbose
 ```
 
 For more information about extra parameters take a look at the scripts `build-native` under src/Native.
 
-### Build Managed
-Since the managed build uses the .NET Core CLI (which the build will download), managed components can only be built on a subset of distros.
-There are some additional prerequisites from the CLI which need to be installed. Both libicu and libunwind are used by CoreCLR to execute managed code, so they must be installed. Since CoreFX does not actually link against these packages, runtime versions are sufficient. We also require curl to be present, which we use to download the .NET Core CLI.
-
-**Examples**
-
-- Building in debug mode for platform x64
-```
-build-managed -debug -buildArch=x64
-```
-
-- Building in debug mode for platform x64 targeting OS Linux
-```
-build-managed -debug -buildArch=x64 -os=Linux
-```
-
 ### Build And Run Tests
-To build the tests and run them you can call the build-test script. The same parameters you pass to build should also be passed to build-tests script to ensure you are building and running the tests on the same configuration you have build the product on. However to run tests on the same machine you need to ensure the machine supports the configuration you are building.
+To build the tests and run them you can call the build script passing -tests option. The same parameters you pass to build should also be passed to ensure you are building and running the tests on the same configuration you have build the product on. However to run tests on the same machine you need to ensure the machine supports the configuration you are building.
 
 **Examples**
-- The following shows how to build tests but not run them
+- The following shows how to build only the tests but not run them
 ```
-build-tests -skiptests
+build -test -skiptests
 ```
 
 - The following builds and runs all tests for netcoreapp in release configuration.
 ```
-build-tests -release -framework=netcoreapp
+build -test -release -framework=netcoreapp
 ```
 
-- The following example shows the argument `--`. Everything that is after it is not going to be processed and it is going to be passed as it is.
-Use it to pass extra msbuild properties, in this case to ignore tests ignored in CI.
+- The following example shows how to pass extra msbuild properties to ignore tests ignored in CI.
 ```
-build-tests -- /p:WithoutCategories=IgnoreForCI
+build -test /p:WithoutCategories=IgnoreForCI
 ```
 
 ### Building individual libraries

Documentation/project-docs/developer-guide.md

@@ -7,5 +7,6 @@
   </fallbackPackageFolders>
   <packageSources>
     <clear/>
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
   </packageSources>
 </configuration>