merge: Merged fixed code into the main branch

.bazelrc

@@ -7,4 +7,20 @@ build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
 
+build:sanitize-common --strip=never
+build:sanitize-common --copt -O1
+build:sanitize-common --copt -g
+build:sanitize-common --copt -fno-omit-frame-pointer
+
+build:asan --config=sanitize-common
+build:asan --copt -fsanitize=address
+build:asan --copt -DADDRESS_SANITIZER
+build:asan --linkopt -fsanitize=address
+
+build:asan --config=sanitize-common
+build:msan --copt -fsanitize=memory
+build:msan --copt -fsanitize=undefined
+build:msan --linkopt -fsanitize=address
+build:msan --linkopt -fsanitize=undefined
+
 run --copt=-fdiagnostics-color=always

.github/workflows/cppcheck.yml

@@ -0,0 +1,24 @@
+name: Static Checker
+
+on: pull_request
+
+jobs:
+  cppcheck:
+    name: CppCheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: chmorgan/[email protected]
+        with:
+          enable: all
+          std: c++11
+          inconclusive: disable
+          output_file: ./cppcheck_report.txt
+          other_options: "-j4 --suppressions-list=util/cppcheck/cppcheck.suppressions --error-exitcode=1 -itest -icurvefs/test -inebd/test -inbd/test -icurvefs_python -icurvesnapshot_python -ithirdparties"
+
+      - name: Show cppcheck report
+        if: failure()
+        run: |
+          cat ./cppcheck_report.txt
+          exit 1

Makefile

@@ -66,7 +66,7 @@ dep:
 	@bash util/build.sh --stor=$(stor) --only="" --dep=1
 
 ci-build:
-	@bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os)
+	@bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os) --sanitizer=$(sanitizer)
 
 ci-dep:
 	@bash util/build_in_image.sh --stor=$(stor) --only="" --dep=1

WORKSPACE

@@ -35,6 +35,12 @@ git_repository(
     name = "com_github_baidu_braft",
     remote = "https://github.com/baidu/braft",
     commit = "d12de388c97998f5ccd5cb97ed0da728815ef438",
+    patches = [
+        "//:thirdparties/braft/0001-fix-change-set_error-to-set_errorv.patch",
+    ],
+    patch_args = [
+        "-p1"
+    ],
 )
 
 bind(
@@ -133,7 +139,11 @@ git_repository(
     name = "com_github_brpc_brpc",
     remote = "https://github.com/apache/incubator-brpc",
     commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
-    patches = ["//:thirdparties/brpc/brpc.patch","//:thirdparties/brpc/fix-gcc11.patch"],
+    patches = [
+        "//:thirdparties/brpc/brpc.patch",
+        "//:thirdparties/brpc/fix-gcc11.patch",
+        "//:thirdparties/brpc/0001-bvar-warning-on-conflict-bvar-name.patch", 
+    ],
     patch_args = ["-p1"],
 )
 
@@ -272,11 +282,11 @@ http_archive(
     # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
-        "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
-        "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
+        "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-3dddf205a1f5cde20faf2444c1757abe0564ff4c.tar.gz",
+        "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/3dddf205a1f5cde20faf2444c1757abe0564ff4c.tar.gz",
     ],
-    strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
+    strip_prefix = "bazel-compile-commands-extractor-3dddf205a1f5cde20faf2444c1757abe0564ff4c",
+    sha256 = "3cd0e49f0f4a6d406c1d74b53b7616f5e24f5fd319eafc1bf8eee6e14124d115",
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

conf/client.conf

@@ -159,6 +159,14 @@ global.metricDummyServerStartPort=9000
 # Whether to turn off health check: true/turn off, false/do not turn off
 global.turnOffHealthCheck=true
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+# NOTE: open file limit will affect how may sockets we can create,
+#       the number of sockets is related to the number of chunkserver and mds in the cluster,
+#       and during some exception handling processes, client will create additional sockets
+#       the SAFE value is 2 * (#chunkserver + #mds)
+global.minOpenFileLimit=1024
+
 #
 ### throttle config
 #

conf/cs_client.conf

@@ -144,6 +144,10 @@ global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEA
 #
 global.metricDummyServerStartPort=9000
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+global.minOpenFileLimit=0
+
 #
 # session map file, storing the mapping from filename to path of the opened file
 #

conf/py_client.conf

@@ -123,6 +123,10 @@ global.fileMaxInFlightRPCNum=64
 # The maximum sharding KB for file IO distribution to the underlying chunkserver
 global.fileIOSplitMaxSizeKB=64
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+global.minOpenFileLimit=0
+
 #
 ################# Log related configuration###############
 #

conf/snap_client.conf

@@ -144,6 +144,10 @@ global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEA
 #
 global.metricDummyServerStartPort=9000
 
+# minimal open file limit
+# if set value to 0, then will skip check and set open file limit
+global.minOpenFileLimit=0
+
 #
 # session map file, storing the mapping from filename to path of the opened file
 #

curve-sdk/DEBIAN/control

@@ -3,7 +3,7 @@ Section:
 Priority: optional
 Depends: libunwind8
 Suggests:
-Architecture:amd64
+Architecture: amd64
 Installed-Size:
 Maintainer: curve-dev
 Provides:

curvefs/docker/openeuler/Dockerfile

@@ -0,0 +1,9 @@
+FROM opencurvedocker/curve-base:openeuler
+COPY libmemcached.so libmemcached.so.11 libhashkit.so.2  libfuse3.so.3.10.5 libsnappy.so.1.1.9  libetcdclient.so /usr/lib/
+COPY curvefs /curvefs
+RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
+    && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
+    && cp /curvefs/tools-v2/sbin/curve /usr/bin/ \
+    && ln -s /usr/lib/libfuse3.so.3.10.5 /usr/lib/libfuse3.so.3 \
+    && ln -s /usr/lib64/libsnappy.so.1.1.9 /usr/lib64/libsnappy.so.1 \
+    && ldconfig

curvefs/docker/openeuler/entrypoint.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+############################  GLOBAL VARIABLES
+g_role=""
+g_args=""
+g_prefix=""
+g_binary=""
+g_start_args=""
+g_preexec="/curvefs/tools-v2/sbin/daemon"
+
+############################  BASIC FUNCTIONS
+function msg() {
+    printf '%b' "$1" >&2
+}
+
+function success() {
+    msg "\33[32m[✔]\33[0m ${1}${2}"
+}
+
+function die() {
+    msg "\33[31m[✘]\33[0m ${1}${2}"
+    exit 1
+}
+
+############################ FUNCTIONS
+function usage () {
+    cat << _EOC_
+Usage:
+    entrypoint.sh --role=ROLE
+    entrypoint.sh --role=ROLE --args=ARGS
+
+Examples:
+    entrypoint.sh --role=etcd
+    entrypoint.sh --role=client --args="-o default_permissions"
+_EOC_
+}
+
+function get_options() {
+    local long_opts="role:,args:,help"
+    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
+    eval set -- "${args}"
+    while true
+    do
+        case "$1" in
+            -r|--role)
+                g_role=$2
+                shift 2
+                ;;
+            -a|--args)
+                g_args=$2
+                shift 2
+                ;;
+            -h)
+                usage
+                exit 1
+                ;;
+            --)
+                shift
+                break
+                ;;
+            *)
+                exit 1
+                ;;
+        esac
+    done
+}
+
+function prepare() {
+    g_prefix="/curvefs/$g_role"
+    conf_path="$g_prefix/conf/$g_role.conf"
+
+    case $g_role in
+        etcd)
+            g_binary="$g_prefix/sbin/etcd"
+            g_start_args="--config-file $conf_path"
+            ;;
+        mds)
+            g_binary="$g_prefix/sbin/curvefs-mds"
+            g_start_args="--confPath $conf_path"
+            ;;
+        metaserver)
+            g_binary="$g_prefix/sbin/curvefs-metaserver"
+            g_start_args="--confPath $conf_path"
+            ;;
+        client)
+            g_binary="$g_prefix/sbin/curve-fuse"
+            g_start_args="--confPath $conf_path"
+            ;;
+        monitor)
+            g_binary="python3"
+            g_start_args="target_json.py"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+
+    if [ "$g_args" != "" ]; then
+        g_start_args=$g_args
+    fi
+}
+
+function create_directory() {
+    chmod 700 "$g_prefix/data"
+    if [ "$g_role" == "etcd" ]; then
+        mkdir -p "$g_prefix/data/wal"
+    elif [ "$g_role" == "metaserver" ]; then
+        mkdir -p "$g_prefix/data/storage"
+    elif [ "$g_role" == "client" ]; then
+        mkdir -p "$g_prefix/mnt"
+    fi
+}
+
+function main() {
+    get_options "$@"
+
+    prepare
+    create_directory
+    [[ $(command -v crontab) ]] && cron
+    [[ ! -z $g_preexec ]] && $g_preexec &
+    if [ $g_role == "etcd" ]; then
+        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
+    elif [ $g_role == "monitor" ]; then
+        cd $g_prefix
+        exec $g_binary $g_start_args
+    else
+        exec $g_binary $g_start_args
+    fi
+
+}
+
+############################  MAIN()
+main "$@"

curvefs/proto/common.proto

@@ -92,6 +92,14 @@ message PartitionInfo {
     optional bool manageFlag = 13; // if a partition has recyclebin inode, set this flag true
 }
 
+message AppliedIndex {
+    required int64 index = 1;
+}
+
+message ItemCount {
+    required uint64 count = 1;
+}
+
 message Peer {
     optional uint64 id          = 1;
     optional string address     = 2;

curvefs/proto/metaserver.proto

@@ -145,6 +145,15 @@ message PrepareRenameTxRequest {
     repeated Dentry dentrys = 4;
 }
 
+message TransactionRequest {
+    enum TransactionType {
+        None = 0;
+        Rename = 1;
+    }
+    required TransactionType type = 1;
+    required string rawPayload = 2;
+}
+
 message PrepareRenameTxResponse {
     required MetaStatusCode statusCode = 1;
     optional uint64 appliedIndex = 2;

curvefs/src/client/common/common.cpp

@@ -79,17 +79,16 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
 }
 
 const char kCurveFsWarmupOpAdd[] = "add";
-const char kCurveFsWarmupOpQuery[] = "query";
+const char kCurveFsWarmupOpCancel[] = "cancel";
 const char kCurveFsWarmupTypeList[] = "list";
 const char kCurveFsWarmupTypeSingle[] = "single";
 
 WarmupOpType GetWarmupOpType(const std::string& op) {
     auto ret = WarmupOpType::kWarmupOpUnknown;
     if (op == kCurveFsWarmupOpAdd) {
         ret = WarmupOpType::kWarmupOpAdd;
-    }
-    if (op == kCurveFsWarmupOpQuery) {
-        ret = WarmupOpType::kWarmupOpQuery;
+    } else if (op == kCurveFsWarmupOpCancel) {
+        ret = WarmupOpType::kWarmupOpCancel;
     }
     return ret;
 }

curvefs/src/client/common/common.h

@@ -69,18 +69,27 @@ const uint32_t MAX_XATTR_NAME_LENGTH = 255;
 const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
 
 const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";
+const char kCurveFsWarmupXAttrList[] = "curvefs.warmup.op.list";
 
-
-constexpr int kWarmupOpNum = 4;
+constexpr size_t kMinWarmupOpArgsNum = 1;
+constexpr size_t kWarmupAddArgsNum = 6;
+constexpr size_t kWarmupCancelArgsNum = 2;
 
 enum class WarmupOpType {
     kWarmupOpUnknown = 0,
     kWarmupOpAdd = 1,
-    kWarmupOpQuery = 2,
+    kWarmupOpCancel = 2,
 };
 
 WarmupOpType GetWarmupOpType(const std::string& op);
 
+constexpr int kWarmupOpType = 0;
+constexpr int kWarmupDataType = 1;
+constexpr int kEntryFilePathInClient = 2;
+constexpr int kWarmupCacheStorageType = 3;
+constexpr int kMountPointInCurvefs = 4;
+constexpr int kRootPathInCurvefs = 5;
+
 enum class WarmupType {
     kWarmupTypeUnknown = 0,
     kWarmupTypeList = 1,