upgrade bazel

1. upgrade bazel to 4.2.2 2. support clang-7.0.1 3. support gcc-8.3.0 4. test in debian10 5. dockerfile can be used to make an image of the build environment add patch: 1. apache/brpc@6efb0cf 2. apache/brpc@3a928dd
opencurve · Apr 6, 2022 · 8891afb · 8891afb
1 parent fccfc6c
commit 8891afb
Show file tree

Hide file tree

Showing 67 changed files with 1,109 additions and 66 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -0,0 +1,5 @@
+build --define=with_glog=true --define=libunwind=true
+build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
+build --cxxopt -Wno-error=format-security
+build:gcc7-later --cxxopt -faligned-new
+build --incompatible_blacklisted_protos_requires_proto_info=false
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-0.17.2
+4.2.2
diff --git a/Makefile b/Makefile
@@ -7,15 +7,16 @@ release?= 0
 only?= "*"
 tag?= "curvebs:unknown"
 case?= "*"
+os?= "debian9"
 
 list:
 	@bash util/build.sh --list
 
 build:
-	@bash util/build.sh --only=$(only) --release=$(release)
+	@bash util/build.sh --only=$(only) --release=$(release) --os=$(os)
 
 install:
 	@bash util/install.sh --prefix=$(prefix) --only=$(only)
 
 image:
-	@bash util/image.sh $(tag)
+	@bash util/image.sh $(tag) $(os)
diff --git a/WORKSPACE b/WORKSPACE
@@ -17,6 +17,20 @@
 workspace(name = "curve")
 
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# skylib
+http_archive(
+    name = "bazel_skylib",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
+    ],
+    sha256 = "af87959afe497dc8dfd4c6cb66e1279cb98ccc84284619ebfec27d9c09a903de",
+)
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+bazel_skylib_workspace()
 
 git_repository(
     name = "com_github_baidu_braft",
@@ -32,13 +46,30 @@ bind(
 # proto_library, cc_proto_library, and java_proto_library rules implicitly
 # depend on @com_google_protobuf for protoc and proto runtimes.
 # This statement defines the @com_google_protobuf repo.
+
+# zlib
 http_archive(
-    name = "com_google_protobuf",
-    sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30",
-    strip_prefix = "protobuf-3.5.0",
-    urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"],
+    name = "net_zlib",
+    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
+    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+    strip_prefix = "zlib-1.2.11",
+    urls = ["https://zlib.net/zlib-1.2.11.tar.gz"],
+)
+
+bind(
+    name = "zlib",
+    actual = "@net_zlib//:zlib",
 )
 
+http_archive(
+    name = "com_google_protobuf",
+    strip_prefix = "protobuf-3.6.1.3",
+    patch_args = ["-p1"],
+    patches = ["//:thirdparties/protobuf/protobuf.patch"],
+    sha256 = "9510dd2afc29e7245e9e884336f848c8a6600a14ae726adb6befdb4f786f0be2",
+    urls = ["https://github.com/google/protobuf/archive/v3.6.1.3.zip"],
+)
+
 bind(
     name = "protobuf",
     actual = "@com_google_protobuf//:protobuf",
@@ -76,22 +107,19 @@ bind(
 http_archive(
     name = "com_github_gflags_gflags",
     strip_prefix = "gflags-2.2.2",
-    urls = [
-        "https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz",
-        "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz",
-    ],
+    urls = ["https://github.com/gflags/gflags/archive/v2.2.2.tar.gz"],
 )
 
 bind(
     name = "gflags",
     actual = "@com_github_gflags_gflags//:gflags",
 )
 
-new_http_archive(
+http_archive(
     name = "com_github_google_leveldb",
     build_file = "bazel/leveldb.BUILD",
     strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
-    url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz",
+    urls = ["https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz"],
 )
 
 bind(
@@ -142,49 +170,37 @@ bind(
 
 new_local_repository(
     name = "etcdclient",
-    build_file = "bazel/etcdclient.BUILD",
+    build_file = "external/bazel/etcdclient.BUILD",
     path = "thirdparties/etcdclient",
 )
 
-new_http_archive(
+http_archive(
     name = "aws",
-    urls = [
-        "https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
-        "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
-    ],
+    urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
     sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
     strip_prefix = "aws-sdk-cpp-1.7.340",
     build_file = "//:thirdparties/aws/aws.BUILD",
 )
 
-new_http_archive(
+http_archive(
     name = "aws_c_common",
-    urls = [
-        "https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
-        "https://mirror.tensorflow.org/github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
-    ],
+    urls = ["https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz"],
     sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
     strip_prefix = "aws-c-common-0.4.29",
     build_file = "//:thirdparties/aws/aws-c-common.BUILD",
 )
 
-new_http_archive(
+http_archive(
     name = "aws_c_event_stream",
-    urls = [
-        "https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
-        "https://mirror.tensorflow.org/github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
-    ],
+    urls = ["https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz"],
     sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
     strip_prefix = "aws-c-event-stream-0.1.4",
     build_file = "//:thirdparties/aws/aws-c-event-stream.BUILD",
 )
 
-new_http_archive(
+http_archive(
     name = "aws_checksums",
-    urls = [
-        "https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
-        "https://mirror.tensorflow.org/github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
-    ],
+    urls = ["https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz"],
     sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
     strip_prefix = "aws-checksums-0.1.5",
     build_file = "//:thirdparties/aws/aws-checksums.BUILD",
@@ -205,3 +221,11 @@ http_archive(
   strip_prefix = "abseil-cpp-20210324.2",
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
+
+# Bazel platform rules.
+http_archive(
+    name = "platforms",
+    sha256 = "b601beaf841244de5c5a50d2b2eddd34839788000fa1be4260ce6603ca0d8eb7",
+    strip_prefix = "platforms-98939346da932eef0b54cf808622f5bb0928f00b",
+    urls = ["https://github.com/bazelbuild/platforms/archive/98939346da932eef0b54cf808622f5bb0928f00b.zip"],
+)
diff --git a/build.sh b/build.sh
@@ -51,16 +51,16 @@ curve_version=${tag_version}+${commit_id}${debug}
 
 
 #step3 执行编译
-# check bazel verion, bazel vesion must = 0.17.2
+# check bazel verion, bazel vesion must = 4.2.2
 bazel_version=`bazel version | grep "Build label" | awk '{print $3}'`
 if [ -z ${bazel_version} ]
 then
-    echo "please install bazel 0.17.2 first"
+    echo "please install bazel 4.2.2 first"
     exit
 fi
-if [ ${bazel_version} != "0.17.2" ]
+if [ ${bazel_version} != "4.2.2" ]
 then
-    echo "bazel version must 0.17.2"
+    echo "bazel version must 4.2.2"
     echo "now version is ${bazel_version}"
     exit
 fi

diff --git a/copts.bzl b/copts.bzl
@@ -120,9 +120,8 @@ CURVE_LLVM_FLAGS = [
     "-Wvla",
     "-Wwrite-strings",
     "-Wno-float-conversion",
-    "-Wno-implicit-float-conversion",
-    "-Wno-implicit-int-float-conversion",
-    "-Wno-implicit-int-conversion",
+    "-Wno-float-conversion",
+    "-Wno-float-overflow-conversion",
     "-Wno-shorten-64-to-32",
     "-Wno-sign-conversion",
     "-DNOMINMAX",
@@ -146,10 +145,16 @@ CURVE_LLVM_TEST_FLAGS = [
     "-Wno-used-but-marked-unused",
     "-Wno-zero-as-null-pointer-constant",
     "-Wno-gnu-zero-variadic-macro-arguments",
+    "-Wbraced-scalar-init",
+]
+
+# FIXME: temporary disabled because triggered in many places
+CURVE_LLVM_DISABLED_FLGAS = [
+    "-Wno-c++11-narrowing",
 ]
 
 CURVE_DEFAULT_COPTS = select({
-    "//:clang_compiler": CURVE_LLVM_FLAGS + CXX_FLAGS + BASE_FLAGS,
+    "//:clang_compiler": CURVE_LLVM_FLAGS + CXX_FLAGS + BASE_FLAGS + CURVE_LLVM_DISABLED_FLGAS,
     "//conditions:default": CURVE_GCC_FLAGS + CXX_FLAGS + BASE_FLAGS + CURVE_GCC_DISABLED_FLGAS,
 })
 

diff --git a/curvefs/Makefile b/curvefs/Makefile
@@ -7,6 +7,7 @@ release?= 0
 only?= "*"
 hosts?= "*"
 tag?= "curvefs:unknown"
+os?= "debian9"
 
 define deploy_begin
 	@bash util/deploy.sh begin
@@ -31,13 +32,13 @@ define only_specify
 endef
 
 build:
-	@bash util/build.sh --only=$(only) --release=$(release)
+	@bash util/build.sh --only=$(only) --release=$(release) --os=$(os)
 
 install:
 	@bash util/install.sh --prefix=$(prefix) --only=$(only)
 
 image:
-	@bash util/image.sh $(tag)
+	@bash util/image.sh $(tag) $(os)
 
 deploy:
 	$(call deploy_begin)

diff --git a/curvefs/docker/debian10/Dockerfile b/curvefs/docker/debian10/Dockerfile
@@ -0,0 +1,8 @@
+FROM opencurvedocker/curve-base:debian10
+ENV TZ=Asia/Shanghai
+RUN mkdir -p /curvefs /etc/curvefs /core
+COPY curvefs /curvefs
+COPY entrypoint.sh /
+COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+RUN chmod a+x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/curvefs/docker/debian10/base/Dockerfile b/curvefs/docker/debian10/base/Dockerfile
@@ -0,0 +1,36 @@
+FROM debian:10
+
+RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        gcc \
+        gdb \
+        make \
+        openssl \
+        net-tools \
+        libcurl3-gnutls \
+        perl \
+        linux-perf \
+        vim \
+        curl \
+        cron \
+        procps \
+        lsof \
+        nginx \
+        less \
+        fuse3 \
+        libnl-3-200 \
+        libnl-genl-3-200 \
+        libjemalloc2 \
+    && ln -s /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+
+COPY libetcdclient.so /usr/lib/
diff --git a/curvefs/docker/debian10/base/Makefile b/curvefs/docker/debian10/base/Makefile
@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:debian10 .
diff --git a/curvefs/docker/debian10/compile/Dockerfile b/curvefs/docker/debian10/compile/Dockerfile
@@ -0,0 +1,36 @@
+FROM debian:10
+
+RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        wget \
+        git \
+        gcc \
+        g++ \
+        clang \
+        libssl-dev \
+        libnl-genl-3-dev \
+        libcurl4-gnutls-dev \
+        uuid-dev \
+        libfiu-dev \
+        libfuse3-dev \
+        zlib1g-dev \
+        make \
+        openjdk-11-jdk \
+        sudo \
+    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
+    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
+    && ln -s /usr/bin/bazelisk /usr/bin/bazel \
+    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
+    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
+    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
+    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk
diff --git a/curvefs/docker/debian10/compile/Makefile b/curvefs/docker/debian10/compile/Makefile
@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:build-debian10 .
diff --git a/curvefs/docker/entrypoint.sh → curvefs/docker/debian10/entrypoint.sh b/curvefs/docker/entrypoint.sh → curvefs/docker/debian10/entrypoint.sh
diff --git a/curvefs/docker/debian11/Dockerfile b/curvefs/docker/debian11/Dockerfile
@@ -0,0 +1,8 @@
+FROM opencurvedocker/curve-base:debian11
+ENV TZ=Asia/Shanghai
+RUN mkdir -p /curvefs /etc/curvefs /core
+COPY curvefs /curvefs
+COPY entrypoint.sh /
+COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+RUN chmod a+x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/curvefs/docker/debian11/base/Dockerfile b/curvefs/docker/debian11/base/Dockerfile
@@ -0,0 +1,36 @@
+FROM debian:11
+
+RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        gcc \
+        gdb \
+        make \
+        openssl \
+        net-tools \
+        libcurl3-gnutls \
+        perl \
+        linux-perf \
+        vim \
+        curl \
+        cron \
+        procps \
+        lsof \
+        nginx \
+        less \
+        fuse3 \
+        libnl-3-200 \
+        libnl-genl-3-200 \
+        libjemalloc2 \
+    && ln -s /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+
+COPY libetcdclient.so /usr/lib/