Changes to support RBE in Windows builds (#820)

This PR does a few things:
1. It changes how the Windows toolchain and SDK are fetched and where
they are stored. Previously, the Windows toolchain and SDK were
downloaded under `depot_tools`, which typically exists outside of the
engine source root both locally and in CI. That setup makes the Windows
SDK inaccessible to RBE builds. Instead, this PR checks out
`depot_tools` under `flutter/third_party/depot_tools` by listing it in
the `DEPS` file. This location is under the engine source root, and is
therefore accessible to RBE builds. This change has the downside of
breaking how CI does caching of the `depot_tools` Windows toolchain and
SDK, which will add some time to builds.
2. It updates Windows GN toolchain setup logic to match what is in the
Chrome tree. Updates to `build/find_depot_tools.py`,
`build/toolchain/win/BUILD.gn`, and
`build/toolchain/win/setup_toolchain.py` are copied from there and
adapted to the Flutter engine repo.

Author: zanderso

build/.gitignore

@@ -1,3 +1,4 @@
 # Generated file containing information about the VS toolchain on Windows
 win_toolchain.json
+new_win_toolchain.json
 /linux/debian_*-sysroot/

build/config/BUILDCONFIG.gn

@@ -526,13 +526,9 @@ if (custom_toolchain != "") {
   host_toolchain = "//build/toolchain/linux:clang_$host_cpu"
   set_default_toolchain("//build/toolchain/custom")
 } else if (is_win) {
-  if (is_clang) {
-    host_toolchain = "//build/toolchain/win:clang_$host_cpu"
-    set_default_toolchain("//build/toolchain/win:clang_$current_cpu")
-  } else {
-    host_toolchain = "//build/toolchain/win:$host_cpu"
-    set_default_toolchain("//build/toolchain/win:$current_cpu")
-  }
+  assert(is_clang)
+  host_toolchain = "//build/toolchain/win:clang_$host_cpu"
+  set_default_toolchain("//build/toolchain/win:clang_$current_cpu")
 } else if (is_android) {
   if (host_os == "linux") {
     # Use clang for the x86/64 Linux host builds.
@@ -544,11 +540,8 @@ if (custom_toolchain != "") {
   } else if (host_os == "mac") {
     host_toolchain = "//build/toolchain/mac:clang_$host_cpu"
   } else if (host_os == "win") {
-    if (is_clang) {
-      host_toolchain = "//build/toolchain/win:clang_$host_cpu"
-    } else {
-      host_toolchain = "//build/toolchain/win:$host_cpu"
-    }
+    assert(is_clang)
+    host_toolchain = "//build/toolchain/win:clang_$host_cpu"
   } else {
     assert(false, "Unknown host for android cross compile")
   }

build/find_depot_tools.py

@@ -1,23 +1,41 @@
-# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+#!/usr/bin/env python3
+# Copyright 2011 The Chromium Authors
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Small utility function to find depot_tools and add it to the python path.
 
 Will throw an ImportError exception if depot_tools can't be found since it
 imports breakpad.
+
+This can also be used as a standalone script to print out the depot_tools
+directory location.
 """
 
+
 import os
 import sys
 
 
+# Path to //src
+SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+
+
 def IsRealDepotTools(path):
-  return os.path.isfile(os.path.join(path, 'gclient.py'))
+  expanded_path = os.path.expanduser(path)
+  return os.path.isfile(os.path.join(expanded_path, 'gclient.py'))
 
 
 def add_depot_tools_to_path():
   """Search for depot_tools and add it to sys.path."""
-  # First look if depot_tools is already in PYTHONPATH.
+  # First, check if we have a DEPS'd in "depot_tools".
+  deps_depot_tools = os.path.join(SRC, 'flutter', 'third_party', 'depot_tools')
+  if IsRealDepotTools(deps_depot_tools):
+    # Put the pinned version at the start of the sys.path, in case there
+    # are other non-pinned versions already on the sys.path.
+    sys.path.insert(0, deps_depot_tools)
+    return deps_depot_tools
+
+  # Then look if depot_tools is already in PYTHONPATH.
   for i in sys.path:
     if i.rstrip(os.sep).endswith('depot_tools') and IsRealDepotTools(i):
       return i
@@ -39,7 +57,18 @@ def add_depot_tools_to_path():
   print('Failed to find depot_tools', file=sys.stderr)
   return None
 
-add_depot_tools_to_path()
+DEPOT_TOOLS_PATH = add_depot_tools_to_path()
 
 # pylint: disable=W0611
 import breakpad
+
+
+def main():
+  if DEPOT_TOOLS_PATH is None:
+    return 1
+  print(DEPOT_TOOLS_PATH)
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

build/toolchain/win/BUILD.gn

@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import("//build/toolchain/rbe.gni")
+import("//build/toolchain/win/win_toolchain_data.gni")
+
 default_clang_base_path = "//buildtools/windows-x64/clang"
 
 declare_args() {
@@ -31,23 +34,18 @@ assert(is_win)
 # This tool will is used as a wrapper for various commands below.
 tool_wrapper_path = rebase_path("tool_wrapper.py", root_build_dir)
 
-toolchain_data = exec_script("setup_toolchain.py",
-                             [
-                               visual_studio_path,
-                               windows_sdk_path,
-                               visual_studio_runtime_dirs,
-                               current_cpu,
-                             ],
-                             "scope")
-
-if (vc_bin_dir == "") {
-  vc_bin_dir = toolchain_data.vc_bin_dir
-}
-
 if (use_goma) {
-  goma_prefix = "$goma_dir/gomacc.exe "
+  compiler_prefix = "$goma_dir/gomacc.exe "
+  asm_prefix = "$goma_dir/gomacc.exe "
+} else if (use_rbe) {
+  compiler_args = rewrapper_command + [
+    "--labels=type=compile,compiler=clang-cl,lang=cpp ",
+  ]
+  compiler_prefix = string_join(" ", compiler_args)
+  asm_prefix = " "
 } else {
-  goma_prefix = ""
+  compiler_prefix = ""
+  asm_prefix = ""
 }
 
 if (current_toolchain == default_toolchain) {
@@ -80,48 +78,73 @@ template("msvc_toolchain") {
       }
     }
 
+    # TODO(zanderso): Assert that clang is always used or remove this logic.
+    if (defined(toolchain_args.is_clang)) {
+      toolchain_is_clang = toolchain_args.is_clang
+    } else {
+      toolchain_is_clang = is_clang
+    }
+
     env = invoker.environment
+    env_path = "$root_out_dir/$env"
+    print(env_path)
+    env_file = read_file(env_path, "string")
+    print(env_file)
 
     cl = invoker.cl
 
     # Make these apply to all tools below.
     lib_switch = ""
     lib_dir_switch = "/LIBPATH:"
 
-    tool("cc") {
-      rspfile = "{{output}}.rsp"
+    # If possible, pass system includes as flags to the compiler.  When that's
+    # not possible, load a full environment file (containing %INCLUDE% and
+    # %PATH%) -- e.g. 32-bit MSVS builds require %PATH% to be set and just
+    # passing in a list of include directories isn't enough.
+    if (defined(invoker.sys_include_flags)) {
+      assert(toolchain_is_clang)
+      env_wrapper = ""
+      sys_include_flags =
+          "${invoker.sys_include_flags} "  # Note trailing space.
+    } else {
+      # clang-cl doesn't need this env hoop, so omit it there.
+      assert(!toolchain_is_clang)
+      env_wrapper = "ninja -t msvc -e $env -- "  # Note trailing space.
+      sys_include_flags = ""
+    }
 
-      # TODO(brettw) enable this when GN support in the binary has been rolled.
-      #precompiled_header_type = "msvc"
-      pdbname = "{{target_out_dir}}/{{target_output_name}}_c.pdb"
-      command = "ninja -t msvc -e $env -- $cl /nologo /showIncludes @$rspfile /c {{source}} /Fo{{output}} /Fd$pdbname"
+    tool("cc") {
+      precompiled_header_type = "msvc"
+      pdbname = "{{target_out_dir}}/{{label_name}}_c.pdb"
       depsformat = "msvc"
       description = "CC {{output}}"
-      outputs = [
-        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj",
-      ]
-      rspfile_content = "{{defines}} {{include_dirs}} {{cflags}} {{cflags_c}}"
+      outputs = [ "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj" ]
+
+      # Label names may have spaces in them so the pdbname must be quoted. The
+      # source and output don't need to be quoted because GN knows they're a
+      # full file name and will quote automatically when necessary.
+      command = "$env_wrapper$cl /c {{source}} /nologo /showIncludes:user $sys_include_flags{{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} /Fo{{output}} /Fd\"$pdbname\""
     }
 
     tool("cxx") {
-      rspfile = "{{output}}.rsp"
-
-      # TODO(brettw) enable this when GN support in the binary has been rolled.
-      #precompiled_header_type = "msvc"
+      precompiled_header_type = "msvc"
 
       # The PDB name needs to be different between C and C++ compiled files.
-      pdbname = "{{target_out_dir}}/{{target_output_name}}_cc.pdb"
+      pdbname = "{{target_out_dir}}/{{label_name}}_cc.pdb"
+
+      # TODO(zanderso): This logic should be moved to be with the other compiler
+      # flag logic in build/config/compiler/BUILD.gn or so.
       flags = ""
       if (is_clang && invoker.current_cpu == "x86") {
         flags = "-m32"
       }
-      command = "ninja -t msvc -e $env -- $cl $flags /nologo /showIncludes @$rspfile /c {{source}} /Fo{{output}} /Fd$pdbname"
+
       depsformat = "msvc"
       description = "CXX {{output}}"
-      outputs = [
-        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj",
-      ]
-      rspfile_content = "{{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}}"
+      outputs = [ "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.obj" ]
+
+      # See comment in CC tool about quoting.
+      command = "$env_wrapper$cl /c {{source}} /Fo{{output}} $flags /nologo /showIncludes:user $sys_include_flags{{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} /Fd\"$pdbname\""
     }
 
     tool("rc") {
@@ -141,7 +164,7 @@ template("msvc_toolchain") {
       } else if (toolchain_args.current_cpu == "arm64") {
         is_msvc_assembler = false
         prefix = rebase_path("$clang_base_path/bin", root_build_dir)
-        ml = "${goma_prefix}${prefix}/clang-cl.exe --target=arm64-windows"
+        ml = "${compiler_prefix}${prefix}/clang-cl.exe --target=arm64-windows"
         x64 = ""
       } else {
         ml = "ml.exe"
@@ -175,14 +198,11 @@ template("msvc_toolchain") {
     }
 
     tool("solink") {
-      dllname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}"  # e.g.
-                                                                               # foo.dll
-      libname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}.lib"  # e.g.
-                                                                                   # foo.dll.lib
-      expname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}.exp"  # e.g.
-                                                                                   # foo.dll.exp
-      pdbname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}.pdb"  # e.g.
-                                                                                   # foo.dll.pdb
+      # E.g. "foo.dll":
+      dllname = "{{root_out_dir}}/{{target_output_name}}{{output_extension}}"
+      libname = "${dllname}.lib"  # e.g. foo.dll.lib
+      expname = "${dllname}.exp"
+      pdbname = "${dllname}.pdb"
       rspfile = "${dllname}.rsp"
 
       link_command = "$python_path $tool_wrapper_path link-wrapper $env False link.exe /nologo /IMPLIB:$libname /DLL /OUT:$dllname /PDB:${dllname}.pdb @$rspfile"
@@ -253,21 +273,22 @@ template("win_toolchains") {
   assert(defined(invoker.toolchain_arch))
   toolchain_arch = invoker.toolchain_arch
 
-  msvc_toolchain(target_name) {
-    environment = "environment." + toolchain_arch
-    cl = "${goma_prefix}\"${vc_bin_dir}/cl.exe\""
-    toolchain_args = {
-      if (defined(invoker.toolchain_args)) {
-        forward_variables_from(invoker.toolchain_args, "*")
-      }
-      current_cpu = toolchain_arch
-      is_clang = false
-    }
+  # The toolchain data for `msvc_toolchain()`.
+  if (toolchain_arch == "x86") {
+    win_toolchain_data = win_toolchain_data_x86
+  } else if (toolchain_arch == "x64") {
+    win_toolchain_data = win_toolchain_data_x64
+  } else if (toolchain_arch == "arm64") {
+    win_toolchain_data = win_toolchain_data_arm64
+  } else {
+    error("Unsupported toolchain_arch, add it to win_toolchain_data.gni")
   }
+
   msvc_toolchain("clang_" + target_name) {
     environment = "environment." + toolchain_arch
     prefix = rebase_path("$clang_base_path/bin", root_build_dir)
-    cl = "${goma_prefix}$prefix/clang-cl.exe"
+    cl = "${compiler_prefix}$prefix/clang-cl.exe"
+    sys_include_flags = "${win_toolchain_data.include_flags_imsvc}"
     toolchain_args = {
       if (defined(invoker.toolchain_args)) {
         forward_variables_from(invoker.toolchain_args, "*")