Use external repository for protobuf dependency. (tensorflow#1289)

Currently, TensorFlow uses a Git submodule for its dependency on protobuf. This was due to the hack used in protobuf for building Python support with Bazel, which was required since Bazel's Python rules did not support adding directories to `PYTHONPATH`. Now that the new `imports` attribute has been added to the Python rules in Bazel 0.2 and the hack for Python support in protobuf has been removed, this change removes the `google/protobuf` Git submodule and adds an external repository for including protobuf. This patch also adds gmock.BUILD to the tensorflow repo. Fixes tensorflow#1069 Fixes tensorflow#2021
phd3 · May 26, 2016 · 14ac223 · 14ac223
1 parent 15e51e6
commit 14ac223
Show file tree

Hide file tree

Showing 17 changed files with 119 additions and 80 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +0,0 @@
-[submodule "google/protobuf"]
-	path = google/protobuf
-	url = https://github.com/google/protobuf.git

diff --git a/WORKSPACE b/WORKSPACE
@@ -20,7 +20,7 @@ tf_workspace()
 
 # Specify the minimum required bazel version.
 load("//tensorflow:tensorflow.bzl", "check_version")
-check_version("0.1.4")
+check_version("0.2.0")
 
 # TENSORBOARD_BOWER_AUTOGENERATED_BELOW_THIS_LINE_DO_NOT_EDIT
 

diff --git a/configure b/configure
@@ -2,17 +2,6 @@
 
 DO_NOT_SUBMIT_WARNING="Unofficial setting. DO NOT SUBMIT!!!"
 
-## Verify that the submodule google/protobuf is available
-# TODO(cais): Remove this check once protobuf is no longer depended upon
-if [[ ! -f "google/protobuf/protobuf.bzl" ]]; then
-  echo "ERROR: It appears that the required submodule google/protobuf is not "\
-"available in this TensorFlow git clone."
-  echo "Please be sure to use the --recurse-submodules flag when performing "\
-"git clone of TensorFlow."
-
-  exit 1
-fi
-
 ## Set up python-related environment settings
 while true; do
   fromuser=""

diff --git a/gmock.BUILD b/gmock.BUILD
@@ -0,0 +1,28 @@
+cc_library(
+    name = "gtest",
+    srcs = [
+        "gmock-1.7.0/gtest/src/gtest-all.cc",
+        "gmock-1.7.0/src/gmock-all.cc",
+    ],
+    hdrs = glob([
+        "gmock-1.7.0/**/*.h",
+        "gmock-1.7.0/gtest/src/*.cc",
+        "gmock-1.7.0/src/*.cc",
+    ]),
+    includes = [
+        "gmock-1.7.0",
+        "gmock-1.7.0/gtest",
+        "gmock-1.7.0/gtest/include",
+        "gmock-1.7.0/include",
+    ],
+    linkopts = ["-pthread"],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "gtest_main",
+    srcs = ["gmock-1.7.0/src/gmock_main.cc"],
+    linkopts = ["-pthread"],
+    visibility = ["//visibility:public"],
+    deps = [":gtest"],
+)
diff --git a/google/protobuf b/google/protobuf
diff --git a/tensorflow/contrib/ffmpeg/default/BUILD b/tensorflow/contrib/ffmpeg/default/BUILD
@@ -16,7 +16,7 @@ cc_library(
         "//tensorflow/contrib/ffmpeg:ffmpeg_lib.h",
     ],
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
     ],
 )

diff --git a/tensorflow/contrib/layers/kernels/BUILD b/tensorflow/contrib/layers/kernels/BUILD
@@ -11,7 +11,7 @@ cc_library(
     name = "bucketization_kernel",
     srcs = ["bucketization_kernel.cc"],
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
     ],
@@ -22,7 +22,7 @@ cc_library(
     name = "sparse_feature_cross_kernel",
     srcs = ["sparse_feature_cross_kernel.cc"],
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
     ],

diff --git a/tensorflow/contrib/linear_optimizer/kernels/BUILD b/tensorflow/contrib/linear_optimizer/kernels/BUILD
@@ -21,7 +21,7 @@ cc_library(
         "squared-loss.h",
     ],
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
     ],
 )
@@ -43,7 +43,7 @@ cc_library(
     srcs = ["resources.cc"],
     hdrs = ["resources.h"],
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
     ],
@@ -68,7 +68,7 @@ cc_library(
     deps = [
         ":loss_updaters",
         ":resources",
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core/kernels:bounds_check_lib",
         "//third_party/eigen3",

diff --git a/tensorflow/contrib/metrics/kernels/BUILD b/tensorflow/contrib/metrics/kernels/BUILD
@@ -11,7 +11,7 @@ cc_library(
     name = "set_kernels",
     srcs = ["set_kernels.cc"],
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
     ],

diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
@@ -25,7 +25,7 @@ cc_library(
         "core/ops/tree_utils.h",
     ],
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
     ],

diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
@@ -1,7 +1,7 @@
 # Platform-specific build configurations.
 
-load("//google/protobuf:protobuf.bzl", "cc_proto_library")
-load("//google/protobuf:protobuf.bzl", "py_proto_library")
+load("@protobuf//:protobuf.bzl", "cc_proto_library")
+load("@protobuf//:protobuf.bzl", "py_proto_library")
 
 # configure may change the following line to True
 WITH_GCP_SUPPORT = False
@@ -31,49 +31,63 @@ def tf_proto_library_cc(name, srcs = [], has_services = None,
                         cc_api_version = 2, go_api_version = 2,
                         java_api_version = 2,
                         py_api_version = 2):
-  native.filegroup(name=name + "_proto_srcs",
-                   srcs=srcs + tf_deps(deps, "_proto_srcs"),
-                   testonly=testonly,)
+  native.filegroup(
+      name = name + "_proto_srcs",
+      srcs = srcs + tf_deps(deps, "_proto_srcs"),
+      testonly = testonly,
+  )
 
   use_grpc_plugin = None
   if cc_grpc_version:
     use_grpc_plugin = True
-  cc_proto_library(name=name + "_cc",
-                   srcs=srcs + tf_deps(deps, "_proto_srcs"),
-                   deps=deps + ["//google/protobuf:cc_wkt_protos"],
-                   cc_libs = cc_libs + ["//google/protobuf:protobuf"],
-                   use_grpc_plugin = use_grpc_plugin,
-                   testonly=testonly,
-                   visibility=visibility,)
+  cc_proto_library(
+      name = name + "_cc",
+      srcs = srcs + tf_deps(deps, "_proto_srcs"),
+      deps = deps + ["@protobuf//:cc_wkt_protos"],
+      cc_libs = cc_libs + ["@protobuf//:protobuf"],
+      protoc = "@protobuf//:protoc",
+      default_runtime = "@protobuf//:protobuf",
+      use_grpc_plugin = use_grpc_plugin,
+      testonly = testonly,
+      visibility = visibility,
+  )
 
 def tf_proto_library_py(name, srcs=[], deps=[], visibility=[], testonly=0,
                         srcs_version="PY2AND3"):
-  py_proto_library(name = name + "_py",
-                   srcs = srcs,
-                   srcs_version = srcs_version,
-                   deps = deps,
-                   visibility = visibility,
-                   testonly = testonly)
+  py_proto_library(
+      name = name + "_py",
+      srcs = srcs,
+      srcs_version = srcs_version,
+      deps = deps,
+      protoc = "@protobuf//:protoc",
+      default_runtime = "@protobuf//:protobuf_python",
+      visibility = visibility,
+      testonly = testonly,
+  )
 
 def tf_proto_library(name, srcs = [], has_services = None,
                      deps = [], visibility = [], testonly = 0,
                      cc_libs = [],
                      cc_api_version = 2, go_api_version = 2,
                      java_api_version = 2,
                      py_api_version = 2):
-  tf_proto_library_cc(name=name,
-                      srcs=srcs + tf_deps(deps, "_proto_srcs"),
-                      deps=deps,
-                      cc_libs=cc_libs,
-                      testonly=testonly,
-                      visibility=visibility,)
-
-  tf_proto_library_py(name=name,
-                      srcs=srcs + tf_deps(deps, "_proto_srcs"),
-                      srcs_version="PY2AND3",
-                      deps=deps + ["//google/protobuf:protobuf_python"],
-                      testonly=testonly,
-                      visibility=visibility,)
+  tf_proto_library_cc(
+      name = name,
+      srcs = srcs + tf_deps(deps, "_proto_srcs"),
+      deps = deps,
+      cc_libs = cc_libs,
+      testonly = testonly,
+      visibility = visibility,
+  )
+
+  tf_proto_library_py(
+      name = name,
+      srcs = srcs + tf_deps(deps, "_proto_srcs"),
+      srcs_version = "PY2AND3",
+      deps = deps + ["@protobuf//:protobuf_python"],
+      testonly = testonly,
+      visibility = visibility,
+  )
 
 def tf_additional_lib_srcs():
   return [

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
@@ -134,7 +134,7 @@ cc_binary(
     }),
     linkshared = 1,
     deps = [
-        "//google/protobuf",
+        "@protobuf//:protobuf",
         "//tensorflow/core:framework_headers_lib",
     ],
 )
@@ -885,7 +885,7 @@ py_library(
     name = "util",
     srcs = glob(["util/**/*.py"]),
     srcs_version = "PY2AND3",
-    deps = ["//google/protobuf:protobuf_python"],
+    deps = ["@protobuf//:protobuf_python"],
 )
 
 tf_proto_library(

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
@@ -539,7 +539,7 @@ def cc_header_only_library(name, deps=[], **kwargs):
 
 def tf_custom_op_library_additional_deps():
   return [
-      "//google/protobuf",
+      "@protobuf//:protobuf",
       "//third_party/eigen3",
       "//tensorflow/core:framework_headers_lib",
   ]

diff --git a/tensorflow/tools/benchmark/BUILD b/tensorflow/tools/benchmark/BUILD
@@ -61,7 +61,7 @@ tf_cc_test(
 # --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
 #
 # NOTE: currently '-pthread' must be removed from the LINK_OPTS variable
-# in google/protobuf/BUILD to sucessfully build for Android. This is temporary
+# in @protobuf//:BUILD to sucessfully build for Android. This is temporary
 # pending an update of the version of the protobuf library that Tensorflow
 # uses.
 cc_binary(

diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
@@ -27,10 +27,10 @@ sh_binary(
         "MANIFEST.in",
         "README",
         "setup.py",
-        "//tensorflow/core:framework_headers",
         ":other_headers",
         ":simple_console",
         "//tensorflow:tensorflow_py",
+        "//tensorflow/core:framework_headers",
         "//tensorflow/examples/tutorials/mnist:package",
         "//tensorflow/models/embedding:package",
         "//tensorflow/models/image/alexnet:all_files",

diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
@@ -61,8 +61,9 @@ function main() {
 
   # protobuf pip package doesn't ship with header files. Copy the headers
   # over so user defined ops can be compiled.
+  mkdir -p ${TMPDIR}/google
   rsync --include "*/" --include "*.h" --exclude "*" --prune-empty-dirs -a \
-    $RUNFILES/google ${TMPDIR}
+    $RUNFILES/external/protobuf ${TMPDIR}/google
   rsync -a $RUNFILES/third_party/eigen3 ${TMPDIR}/third_party
 
   cp tensorflow/tools/pip_package/MANIFEST.in ${TMPDIR}

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
@@ -4,30 +4,13 @@
 # within the workspace (e.g. "tensorflow/"), and tf_repo_name is the name of the
 # local_repository rule (e.g. "@tf").
 def tf_workspace(path_prefix = "", tf_repo_name = ""):
-  native.new_http_archive(
-    name = "gmock_archive",
-    url = "https://archive.openswitch.net/gmock-1.7.0.zip",
-    sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
-    build_file = path_prefix + "google/protobuf/gmock.BUILD",
-  )
-
   native.new_http_archive(
     name = "eigen_archive",
     url = "https://bitbucket.org/eigen/eigen/get/f3a13643ac1f.tar.gz",
     sha256 = "a9266e60366cddb371a23d86b11a297eee86372a89ef4b38a3509012f9cc37ec",
     build_file = path_prefix + "eigen.BUILD",
   )
 
-  native.bind(
-    name = "gtest",
-    actual = "@gmock_archive//:gtest",
-  )
-
-  native.bind(
-    name = "gtest_main",
-    actual = "@gmock_archive//:gtest_main",
-  )
-
   native.git_repository(
     name = "re2",
     remote = "https://github.com/google/re2.git",
@@ -85,16 +68,44 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     actual = "@six_archive//:six",
   )
 
+  native.git_repository(
+    name = "protobuf",
+    remote = "https://github.com/google/protobuf",
+    commit = "ed87c1fe2c6e1633cadb62cf54b2723b2b25c280",
+  )
+
+  native.new_http_archive(
+    name = "gmock_archive",
+    url = "https://archive.openswitch.net/gmock-1.7.0.zip",
+    sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
+    build_file = path_prefix + "gmock.BUILD",
+  )
+
+  native.bind(
+    name = "gtest",
+    actual = "@gmock_archive//:gtest",
+  )
+
+  native.bind(
+    name = "gtest_main",
+    actual = "@gmock_archive//:gtest_main",
+  )
+
+  native.bind(
+      name = "python_headers",
+      actual = tf_repo_name + "//util/python:python_headers",
+  )
+
   # grpc expects //external:protobuf_clib and //external:protobuf_compiler
   # to point to the protobuf's compiler library.
   native.bind(
     name = "protobuf_clib",
-    actual = tf_repo_name + "//google/protobuf:protoc_lib",
+    actual = "@protobuf//:protoc_lib",
   )
 
   native.bind(
     name = "protobuf_compiler",
-    actual = tf_repo_name + "//google/protobuf:protoc_lib",
+    actual = "@protobuf//:protoc_lib",
   )
 
   native.git_repository(