diff --git a/.gitignore b/.gitignore index 9e9322d990..5d9699a309 100755 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,12 @@ runlog/ test/client/configs/* !test/client/configs/.gitkeep +projects/* +docker/curvebs +docker/base/* +!docker/base/Dockerfile +!docker/base/Makefile + # curvefs curvefs/devops/projects/ curvefs/devops/ssh/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..d6ca86677c --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. + +.PHONY: list build install image + +prefix?= "$(PWD)/projects" +release?= 0 +only?= "*" +tag?= "curvebs:unknown" +case?= "*" + +list: + @bash util/build.sh --list + +build: + @bash util/build.sh --only=$(only) --release=$(release) + +install: + @bash util/install.sh --prefix=$(prefix) --only=$(only) + +image: + @bash util/image.sh $(tag) \ No newline at end of file diff --git a/conf/chunkserver.conf b/conf/chunkserver.conf new file mode 100644 index 0000000000..565978841f --- /dev/null +++ b/conf/chunkserver.conf @@ -0,0 +1,220 @@ +# +# Global settings +# +# log等级INFO=0/WARNING=1/ERROR=2/FATAL=3 +global.ip=127.0.0.1 # __CURVEADM_TEMPLATE__ ${service_addr} __CURVEADM_TEMPLATE__ +global.port=8200 # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__ +global.subnet=127.0.0.0/24 +global.enable_external_server=true +global.external_ip=127.0.0.1 # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__ +global.external_subnet=127.0.0.0/24 +# chunk大小,一般16MB +global.chunk_size=16777216 +# chunk 元数据页大小,一般4KB +global.meta_page_size=4096 +# clone chunk允许的最长location长度 +global.location_limit=3000 +# minimum alignment for io request +global.min_io_alignment=512 + +# +# MDS settings +# +#支持mds多地址,以逗号分隔 127.0.0.1:6666,127.0.0.1:7777 +mds.listen.addr=127.0.0.1:6666 # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__ +# 向mds注册的最大重试次数 +mds.register_retries=100 +# 向mds注册的rpc超时时间,一般1000ms +mds.register_timeout=1000 +# 向mds发送心跳的间隔,一般10s +mds.heartbeat_interval=10 +# 向mds发送心跳的rpc超时间,一般1000ms +mds.heartbeat_timeout=5000 + +# +# Chunkserver settings +# +# chunkserver主目录 +chunkserver.stor_uri=local://./0/ # __CURVEADM_TEMPLATE__ local://${prefix}/data __CURVEADM_TEMPLATE__ +# chunkserver元数据文件 +chunkserver.meta_uri=local://./0/chunkserver.dat # __CURVEADM_TEMPLATE__ local://${prefix}/data/chunkserver.dat __CURVEADM_TEMPLATE__ +# disk类型 +chunkserver.disk_type=nvme +# raft内部install snapshot带宽上限,一般20MB +chunkserver.snapshot_throttle_throughput_bytes=20971520 +# check cycles是为了更精细的进行带宽控制,以snapshotThroughputBytes=100MB, +# check cycles=10为例,它可以保证每1/10秒的带宽是10MB,且不累积,例如第1个 +# 1/10秒的带宽是10MB,但是就过期了,在第2个1/10秒依然只能用10MB的带宽,而 +# 不是20MB的带宽 +chunkserver.snapshot_throttle_check_cycles=4 + +# +# Testing purpose settings +# +test.create_testcopyset=false +test.testcopyset_poolid=666 +test.testcopyset_copysetid=888888 +test.testcopyset_conf=127.0.0.1:8200:0,127.0.0.1:8201:0,127.0.0.1:8202:0 + +# +# Copyset settings +# +# 是否检查任期,一般检查 +copyset.check_term=true +# 是否关闭raft配置变更的服务,一般不关闭 +copyset.disable_cli=false +copyset.log_applied_task=false +# raft选举超时时间,一般是5000ms +copyset.election_timeout_ms=1000 +# raft打快照间隔,一般是1800s,也就是30分钟 +copyset.snapshot_interval_s=1800 +# add一个节点,add的节点首先以类似learner的角色拷贝数据 +# 在跟leader差距catchup_margin个entry的时候,leader +# 会尝试将配置变更的entry进行提交(一般来说提交的entry肯定 +# 会commit&apply,catchup_margin较小可以大概率确保learner +# 后续很快可以加入复制组 +copyset.catchup_margin=1000 +# copyset chunk数据目录 +copyset.chunk_data_uri=local://./0/copysets # __CURVEADM_TEMPLATE__ local://${prefix}/data/copysets __CURVEADM_TEMPLATE__ +# raft wal log目录 +copyset.raft_log_uri=curve://./0/copysets # __CURVEADM_TEMPLATE__ curve://${prefix}/data/copysets __CURVEADM_TEMPLATE__ +# raft元数据目录 +copyset.raft_meta_uri=local://./0/copysets # __CURVEADM_TEMPLATE__ local://${prefix}/data/copysets __CURVEADM_TEMPLATE__ +# raft snapshot目录 +copyset.raft_snapshot_uri=curve://./0/copysets # __CURVEADM_TEMPLATE__ curve://${prefix}/data/copysets __CURVEADM_TEMPLATE__ +# copyset回收目录 +copyset.recycler_uri=local://./0/recycler # __CURVEADM_TEMPLATE__ local://${prefix}/data/recycler __CURVEADM_TEMPLATE__ +copyset.max_inflight_requests=5000 +# chunkserver启动时,copyset并发加载的阈值,为0则表示不做限制 +copyset.load_concurrency=10 +# 检查copyset是否加载完成出现异常时的最大重试次数 +copyset.check_retrytimes=3 +# 当前peer的applied_index与leader上的committed_index差距小于该值 +# 则判定copyset已经加载完成 +copyset.finishload_margin=2000 +# 循环判定copyset是否加载完成的内部睡眠时间 +copyset.check_loadmargin_interval_ms=1000 +# scan copyset interval +copyset.scan_interval_sec=5 +# the size each scan 4MB +copyset.scan_size_byte=4194304 +# the follower send scanmap to leader rpc timeout +copyset.scan_rpc_timeout_ms=1000 +# the follower send scanmap to leader rpc retry times +copyset.scan_rpc_retry_times=3 +# the follower send scanmap to leader rpc retry interval +copyset.scan_rpc_retry_interval_us=100000 + +# +# Clone settings +# +# 禁止使用curveclient +clone.disable_curve_client=false +# 禁止使用s3adapter +clone.disable_s3_adapter=false +# 克隆的分片大小,一般1MB +clone.slice_size=1048576 +# 读clone chunk时是否需要paste到本地 +# 该配置对recover chunk请求类型无效 +clone.enable_paste=false +# 克隆的线程数量 +clone.thread_num=10 +# 克隆的队列深度 +clone.queue_depth=6000 +# curve用户名 +curve.root_username=root +# curve密码 +curve.root_password=root_password +# client配置文件 +curve.config_path=conf/cs_client.conf # __CURVEADM_TEMPLATE__ ${prefix}/conf/cs_client.conf __CURVEADM_TEMPLATE__ +# s3配置文件 +s3.config_path=conf/s3.conf # __CURVEADM_TEMPLATE__ ${prefix}/conf/s3.conf __CURVEADM_TEMPLATE__ +# Curve File time to live +curve.curve_file_timeout_s=30 + +# +# Local FileSystem settings +# +# 是否开启使用renameat2,ext4内核3.15以后开始支持 +fs.enable_renameat2=true + +# +# metrics settings +# true means on, false means off +# +metric.onoff=true + +# +# Storage engine settings +# +storeng.sync_write=false + +# +# QoS settings +# + +# +# Concurrent apply module +# 并发模块写线程的并发度,一般是10 +wconcurrentapply.size=10 +# 并发模块写线程的队列深度 +wconcurrentapply.queuedepth=1 +# 并发模块读线程的并发度,一般是5 +rconcurrentapply.size=5 +# 并发模块读线程的队列深度 +rconcurrentapply.queuedepth=1 + +# +# Chunkfile pool +# +# 是否开启从chunkfilepool获取chunk,一般是true +chunkfilepool.enable_get_chunk_from_pool=true +# chunkfilepool目录 +chunkfilepool.chunk_file_pool_dir=./0/ # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__ +# chunkfilepool meta文件路径 +chunkfilepool.meta_path=./chunkfilepool.meta # __CURVEADM_TEMPLATE__ ${prefix}/data/chunkfilepool.meta __CURVEADM_TEMPLATE__ +# chunkfilepool meta文件大小 +chunkfilepool.cpmeta_file_size=4096 +# chunkfilepool get chunk最大重试次数 +chunkfilepool.retry_times=5 +# Enable clean chunk +chunkfilepool.clean.enable=true +# The bytes per write for cleaning chunk (max: 1MB) +chunkfilepool.clean.bytes_per_write=4096 +# The throttle iops for cleaning chunk (4KB/IO) +chunkfilepool.clean.throttle_iops=500 + +# +# WAL file pool +# +# walpool是否共用chunkfilepool,如果为true,则以下配置无效 +walfilepool.use_chunk_file_pool=true +# 是否开启从walfilepool获取chunk,一般是true +walfilepool.enable_get_segment_from_pool=true +# walpool目录 +walfilepool.file_pool_dir=./0/ # __CURVEADM_TEMPLATE__ ${prefix}/data/walfilepool.meta __CURVEADM_TEMPLATE__ +# walpool meta文件路径 +walfilepool.meta_path=./walfilepool.meta # __CURVEADM_TEMPLATE__ ${prefix}/data/walfilepool.meta __CURVEADM_TEMPLATE__ +# walpool meta文件大小 +walfilepool.segment_size=8388608 +# WAL metapage大小 +walfilepool.metapage_size=4096 +# WAL filepool 元数据文件大小 +walfilepool.meta_file_size=4096 +# WAL filepool get chunk最大重试次数 +walfilepool.retry_times=5 + +# +# trash settings +# +# chunkserver回收数据彻底删除的过期时间 +trash.expire_afterSec=300 +# chunkserver检查回收数据过期时间的周期 +trash.scan_periodSec=120 + +# common option +# +# chunkserver 日志存放文件夹 +chunkserver.common.logDir=./ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__ +# 单元测试情况下 +# chunkserver.common.logDir=./runlog/ diff --git a/conf/client.conf b/conf/client.conf index 559ff73f83..12db0a1888 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -132,7 +132,7 @@ global.fileIOSplitMaxSizeKB=64 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3 global.logLevel=0 # 设置log的路径 -global.logPath=/data/log/curve/ +global.logPath=/data/log/curve/ # __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__ # 单元测试情况下 # logpath=./runlog/ diff --git a/conf/cs_client.conf b/conf/cs_client.conf index 1e98564b80..847f7ba9bd 100644 --- a/conf/cs_client.conf +++ b/conf/cs_client.conf @@ -3,7 +3,7 @@ # # mds的地址信息,对于mds集群,地址以逗号隔开 -mds.listen.addr=127.0.0.1:6666 +mds.listen.addr=127.0.0.1:6666 # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__ # 初始化阶段向mds注册开关,默认为开 mds.registerToMDS=false @@ -138,7 +138,7 @@ global.fileIOSplitMaxSizeKB=64 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3 global.logLevel=0 # 设置log的路径 -global.logPath=/data/log/curve/ +global.logPath=/data/log/curve/ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__ # 单元测试情况下 # logpath=./runlog/ diff --git a/conf/etcd.conf b/conf/etcd.conf new file mode 100644 index 0000000000..7b1dc8778d --- /dev/null +++ b/conf/etcd.conf @@ -0,0 +1,113 @@ +# This is the configuration file for the etcd server. + +# Human-readable name for this member. +name: # __CURVEADM_TEMPLATE__ etcd${service_host_sequence}${service_replica_sequence} __CURVEADM_TEMPLATE__ + +# Path to the data directory. +data-dir: # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__ + +# Path to the dedicated wal directory. +wal-dir: # __CURVEADM_TEMPLATE__ ${prefix}/data/wal __CURVEADM_TEMPLATE__ + +# Number of committed transactions to trigger a snapshot to disk. +snapshot-count: 10000 + +# Time (in milliseconds) of a heartbeat interval. +heartbeat-interval: 100 + +# Time (in milliseconds) for an election to timeout. +election-timeout: 1000 +quota-backend-bytes: 0 + +# List of comma separated URLs to listen on for peer traffic. +listen-peer-urls: # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_port} __CURVEADM_TEMPLATE__ + +# List of comma separated URLs to listen on for client traffic. +listen-client-urls: # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_client_port} __CURVEADM_TEMPLATE__ + +# Maximum number of snapshot files to retain (0 is unlimited). +max-snapshots: 5 + +# Maximum number of wal files to retain (0 is unlimited). +max-wals: 5 + +# Comma-separated white list of origins for CORS (cross-origin resource sharing). +cors: + +# List of this member's peer URLs to advertise to the rest of the cluster. +# The URLs needed to be a comma-separated list. +initial-advertise-peer-urls: # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_port} __CURVEADM_TEMPLATE__ + +# List of this member's client URLs to advertise to the public. +# The URLs needed to be a comma-separated list. +advertise-client-urls: # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_client_port} __CURVEADM_TEMPLATE__ + +# Discovery URL used to bootstrap the cluster. +discovery: + +# Valid values include 'exit', 'proxy' +discovery-fallback: proxy + +# HTTP proxy to use for traffic to discovery service. +discovery-proxy: + +# DNS domain used to bootstrap initial cluster. +discovery-srv: + +# Initial cluster configuration for bootstrapping. +initial-cluster: # __CURVEADM_TEMPLATE__ ${cluster_etcd_http_addr} __CURVEADM_TEMPLATE__ + +# Initial cluster token for the etcd cluster during bootstrap. +initial-cluster-token: etcd-cluster + +# Initial cluster state ('new' or 'existing'). +initial-cluster-state: new + +# Reject reconfiguration requests that would cause quorum loss. +strict-reconfig-check: False + +# Accept etcd V2 client requests +enable-v2: True + +# Enable runtime profiling data via HTTP server +enable-pprof: True + +# Valid values include 'on', 'readonly', 'off' +proxy: 'off' + +# Time (in milliseconds) an endpoint will be held in a failed state. +proxy-failure-wait: 5000 + +# Time (in milliseconds) of the endpoints refresh interval. +proxy-refresh-interval: 30000 + +# Time (in milliseconds) for a dial to timeout. +proxy-dial-timeout: 1000 + +# Time (in milliseconds) for a write to timeout. +proxy-write-timeout: 5000 + +# Time (in milliseconds) for a read to timeout. +proxy-read-timeout: 0 + +# Enable debug-level logging for etcd. +debug: False + +logger: zap + +# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd. +log-outputs: [stderr] + +# Force to create a new one member cluster. +force-new-cluster: False + +auto-compaction-mode: periodic +auto-compaction-retention: "1" + +# Set level of detail for exported metrics, specify 'extensive' to include histogram metrics. +metrics: extensive + +# Enable to run an additional Raft election phase. +pre-vote: True + +enable-grpc-gateway: True diff --git a/conf/mds.conf b/conf/mds.conf index 3a5a967b45..8dde5ab24b 100644 --- a/conf/mds.conf +++ b/conf/mds.conf @@ -1,16 +1,16 @@ # # mds服务端口 # -mds.listen.addr=127.0.0.1:6666 -mds.dummy.listen.port=6667 +mds.listen.addr=127.0.0.1:6666 #__CURVEADM_TEMPLATE__ ${service_addr}:${service_port} __CURVEADM_TEMPLATE__ +mds.dummy.listen.port=6667 # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__ global.subnet=127.0.0.0/24 -global.port=6666 +global.port=6666 # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__ # # etcd相关配置 # # etcd地址 -mds.etcd.endpoint=127.0.0.1:2379 +mds.etcd.endpoint=127.0.0.1:2379 # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__ # client建立连接的超时时间 mds.etcd.dailtimeoutMs=5000 # client进行put/get/txn等操作的超时时间 @@ -214,13 +214,13 @@ mds.chunkserverclient.updateLeaderRetryIntervalMs=5000 # snapshotclone config # # snapshot clone server 地址 -mds.snapshotcloneclient.addr=127.0.0.1:5555 +mds.snapshotcloneclient.addr=127.0.0.1:5555 # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_proxy_addr} __CURVEADM_TEMPLATE__ # # common options # # 日志存放文件夹 -mds.common.logDir=./ +mds.common.logDir=./ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__ # 单元测试情况下 # mds.common.logDir=./runlog/ diff --git a/conf/nginx.conf b/conf/nginx.conf new file mode 100644 index 0000000000..40aa905fb2 --- /dev/null +++ b/conf/nginx.conf @@ -0,0 +1,50 @@ +# -*- mode: nginx -*- +# vim: set expandtab tabstop=4 shiftwidth=4: + +worker_processes 4; +pid ${prefix}/logs/nginx.pid; + +events { + worker_connections 65535; +} + +http { + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for", "$http_range", "$request_time" ' + '"$upstream_addr" "-" "-" "-" '; + + access_log ${prefix}/logs/access.log main; + error_log ${prefix}/logs/error.log; + + sendfile on; + keepalive_timeout 65; + + send_timeout 60; + proxy_read_timeout 60; + proxy_send_timeout 60; + proxy_connect_timeout 1; + proxy_max_temp_file_size 0m; + + client_header_buffer_size 4k; + large_client_header_buffers 8 16k; + client_body_buffer_size 128k; + client_max_body_size 200m; + port_in_redirect off; + + resolver 119.29.29.29 223.5.5.5 valid=1s; + resolver_timeout 5s; + + server { + listen ${service_addr}:${service_proxy_port}; + location / { + proxy_pass http://curvebs.163.com; + } + } + + upstream curvebs.163.com { + ${cluster_snapshotclone_nginx_upstream} + } +} diff --git a/conf/snap_client.conf b/conf/snap_client.conf index 2c410d3c62..1f3cc08bea 100644 --- a/conf/snap_client.conf +++ b/conf/snap_client.conf @@ -3,7 +3,7 @@ # # mds的地址信息,对于mds集群,地址以逗号隔开 -mds.listen.addr=127.0.0.1:6666 +mds.listen.addr=127.0.0.1:6666 # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__ # 初始化阶段向mds注册开关,默认为开 mds.registerToMDS=false @@ -138,7 +138,7 @@ global.fileIOSplitMaxSizeKB=64 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3 global.logLevel=0 # 设置log的路径 -global.logPath=/data/log/curve/ +global.logPath=/data/log/curve/ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__ # 单元测试情况下 # logpath=./runlog/ diff --git a/conf/snapshot_clone_server.conf b/conf/snapshot_clone_server.conf index 42abcfc235..222aafcdda 100644 --- a/conf/snapshot_clone_server.conf +++ b/conf/snapshot_clone_server.conf @@ -2,7 +2,7 @@ # curvefs client options # # client配置文件位置 -client.config_path=conf/cs_client.conf +client.config_path=conf/cs_client.conf # __CURVEADM_TEMPLATE__ ${prefix}/conf/snap_client.conf __CURVEADM_TEMPLATE__ # mds root 用户名 mds.rootUser=root # mds root 密码 @@ -13,19 +13,19 @@ client.methodRetryTimeSec=300 client.methodRetryIntervalMs=5000 # 日志文件位置 -log.dir=./ +log.dir=./ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__ # # S3 config path # -s3.config_path=./conf/s3.conf +s3.config_path=./conf/s3.conf # __CURVEADM_TEMPLATE__ ${prefix}/conf/s3.conf __CURVEADM_TEMPLATE__ # #server options # # for snapshot -server.address=127.0.0.1:5556 +server.address=127.0.0.1:5556 # __CURVEADM_TEMPLATE__ ${service_addr}:${service_port} __CURVEADM_TEMPLATE__ server.subnet=127.0.0.0/24 -server.port=5556 +server.port=5556 # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__ # 调用client异步方法重试总时间 server.clientAsyncMethodRetryTimeSec=300 # 调用client异步方法重试时间间隔 @@ -73,7 +73,7 @@ server.backEndReferenceFuncScanIntervalMs=3600000 # etcd相关配置 # # etcd地址 -etcd.endpoint=127.0.0.1:2379 +etcd.endpoint=127.0.0.1:2379 # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__ # client建立连接的超时时间 etcd.dailtimeoutMs=5000 # client进行put/get/txn等操作的超时时间 @@ -102,4 +102,4 @@ leader.election.timeoutms=0 # # dummyserver相关配置 # -server.dummy.listen.port=8081 +server.dummy.listen.port=8081 # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__ diff --git a/conf/tools.conf b/conf/tools.conf index e3e69e1ccd..3b963ee880 100644 --- a/conf/tools.conf +++ b/conf/tools.conf @@ -1,7 +1,7 @@ # mds地址 -mdsAddr=127.0.0.1:6666 +mdsAddr=127.0.0.1:6666 # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__ # mds dummy port -mdsDummyPort=6667 +mdsDummyPort=6700 # __CURVEADM_TEMPLATE__ ${cluster_mds_dummy_port} __CURVEADM_TEMPLATE__ # 发送rpc的超时时间 rpcTimeout=500 # rpc重试次数 @@ -9,10 +9,10 @@ rpcRetryTimes=5 # the rpc concurrency to chunkserver rpcConcurrentNum=10 # etcd地址 -etcdAddr=127.0.0.1:2379 +etcdAddr=127.0.0.1:2379 # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__ # snapshot clone server 地址 -snapshotCloneAddr=127.0.0.1:5555 +snapshotCloneAddr=127.0.0.1:5555 # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_addr} __CURVEADM_TEMPLATE__ # snapshot clone server dummy port -snapshotCloneDummyPort=8081 +snapshotCloneDummyPort=8081 # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_dummy_port} __CURVEADM_TEMPLATE__ rootUserName=root rootUserPassword=root_password diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..8950da1ec0 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,8 @@ +FROM opencurvedocker/curve-base:debian9 +ENV TZ=Asia/Shanghai +RUN mkdir -p /curvebs /etc/curve /etc/nebd /core +COPY curvebs /curvebs +COPY entrypoint.sh exec.sh / +COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/ +RUN chmod a+x /entrypoint.sh /exec.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile new file mode 100644 index 0000000000..7828677da9 --- /dev/null +++ b/docker/base/Dockerfile @@ -0,0 +1,32 @@ +FROM debian:9 + +RUN echo "deb http://mirrors.163.com/debian/ stretch main\n" \ + "deb http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \ + "deb-src http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \ + "deb http://mirrors.163.com/debian-security/ stretch/updates main non-free contrib\n" \ + "deb http://httpredir.debian.org/debian stretch-backports main contrib non-free\n" \ + > /etc/apt/sources.list \ + && apt-get clean \ + && apt-get -y update \ + && apt-get -y install \ + gcc \ + gdb \ + make \ + openssl \ + net-tools \ + libcurl3-gnutls \ + podlators-perl \ + linux-perf \ + vim \ + curl \ + cron \ + procps \ + lsof \ + nginx \ + less \ + more + +COPY fusermount3 /usr/local/bin +COPY libetcdclient.so /usr/lib/ +COPY libfuse3.so libfuse3.so.3 libfuse3.so.3.10.5 libnl-3.so.200 libnl-genl-3.so.200 /usr/lib/x86_64-linux-gnu/ +COPY libjemalloc.so libjemalloc.so.1 libjemalloc.so.2 /usr/local/lib/ diff --git a/docker/base/Makefile b/docker/base/Makefile new file mode 100644 index 0000000000..12f8cbad18 --- /dev/null +++ b/docker/base/Makefile @@ -0,0 +1,4 @@ +.PHONY: build + +build: + docker build -t opencurvedocker/curve-base:debian9 . diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000000..8614954fe5 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. + +############################ GLOBAL VARIABLES +g_role="" +g_args="" +g_prefix="" +g_preexec="" +g_binary="" +g_start_args="" + +############################ BASIC FUNCTIONS +function msg() { + printf '%b' "$1" >&2 +} + +function success() { + msg "\33[32m[✔]\33[0m ${1}${2}" +} + +function die() { + msg "\33[31m[✘]\33[0m ${1}${2}" + exit 1 +} + +############################ FUNCTIONS +function usage () { + cat << _EOC_ +Usage: + entrypoint.sh --role=ROLE + entrypoint.sh --role=ROLE --args=ARGS + +Examples: + entrypoint.sh --role=etcd + entrypoint.sh --role=client --args="-o default_permissions" +_EOC_ +} + +function get_options() { + local long_opts="role:,args:,help" + local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"` + eval set -- "${args}" + while true + do + case "$1" in + -r|--role) + g_role=$2 + shift 2 + ;; + -a|--args) + g_args=$2 + shift 2 + ;; + -h) + usage + exit 1 + ;; + --) + shift + break + ;; + *) + exit 1 + ;; + esac + done +} + +function prepare() { + g_prefix="/curvebs/$g_role" + conf_path="$g_prefix/conf/$g_role.conf" + + case $g_role in + etcd) + g_binary="$g_prefix/sbin/etcd" + g_start_args="--config-file $conf_path" + ;; + mds) + g_binary="$g_prefix/sbin/curvebs-mds" + g_start_args="--confPath $conf_path" + ;; + chunkserver) + g_binary="$g_prefix/sbin/curvebs-chunkserver" + g_start_args="--conf=$conf_path" + ;; + snapshotclone) + g_preexec="/usr/sbin/nginx -c $g_prefix/conf/nginx.conf" + g_binary="$g_prefix/sbin/curvebs-snapshotclone" + g_start_args="--conf=$conf_path" + ;; + nebd) + g_binary="$g_prefix/sbin/nebd-server" + g_start_args="-confPath=$g_prefix/conf/nebd-server.conf -log_dir=$g_prefix/logs" + ;; + *) + usage + exit 1 + ;; + esac + + if [ "$g_args" != "" ]; then + g_start_args=$g_args + fi +} + +function create_directory() { + chmod 700 "$g_prefix/data" + if [ "$g_role" == "etcd" ]; then + mkdir -p "$g_prefix/data/wal" + elif [ "$g_role" == "client" ]; then + mkdir -p "$g_prefix/mnt" + fi +} + +function main() { + get_options "$@" + + prepare + create_directory + [[ $(command -v crontab) ]] && cron + [[ ! -z $g_preexec ]] && $g_preexec + exec $g_binary $g_start_args +} + +############################ MAIN() +main "$@" diff --git a/nebd/etc/nebd/nebd-client.conf b/nebd/etc/nebd/nebd-client.conf index dcd2e3fca4..1207e5bbd0 100644 --- a/nebd/etc/nebd/nebd-client.conf +++ b/nebd/etc/nebd/nebd-client.conf @@ -1,8 +1,8 @@ # part2 socket file address -nebdserver.serverAddress=/data/nebd/nebd.sock +nebdserver.serverAddress=/data/nebd/nebd.sock # __CURVEADM_TEMPLATE__ ${prefix}/data/nebd.sock __CURVEADM_TEMPLATE__ # 文件锁路径 -metacache.fileLockPath=/data/nebd/lock +metacache.fileLockPath=/data/nebd/lock # __CURVEADM_TEMPLATE__ ${prefix}/data/lock __CURVEADM_TEMPLATE__ # 同步rpc的最大重试次数 request.syncRpcMaxRetryTimes=50 @@ -25,4 +25,4 @@ heartbeat.intervalS=5 heartbeat.rpcTimeoutMs=500 # 日志路径 -log.path=/data/log/nebd/client +log.path=/data/log/nebd/client # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__ diff --git a/nebd/etc/nebd/nebd-server.conf b/nebd/etc/nebd/nebd-server.conf index 708b833a49..a6d2fbe534 100644 --- a/nebd/etc/nebd/nebd-server.conf +++ b/nebd/etc/nebd/nebd-server.conf @@ -1,11 +1,11 @@ # curve-client配置文件地址 -curveclient.confPath=/etc/curve/client.conf +curveclient.confPath=/etc/curve/client.conf # __CURVEADM_TEMPLATE__ ${prefix}/conf/client.conf __CURVEADM_TEMPLATE__ #brpc server监听端口 -listen.address=/data/nebd/nebd.sock +listen.address=/data/nebd/nebd.sock # __CURVEADM_TEMPLATE__ ${prefix}/data/nebd.sock __CURVEADM_TEMPLATE__ #元数据文件地址,包含文件名 -meta.file.path=/data/nebd/nebdserver.meta +meta.file.path=/data/nebd/nebdserver.meta # __CURVEADM_TEMPLATE__ ${prefix}/data/nebdserver.meta __CURVEADM_TEMPLATE__ #心跳超时时间 heartbeat.timeout.sec=30 diff --git a/src/tools/curve_tool_main.cpp b/src/tools/curve_tool_main.cpp index 2a561d0b12..9769199e97 100644 --- a/src/tools/curve_tool_main.cpp +++ b/src/tools/curve_tool_main.cpp @@ -85,6 +85,9 @@ void UpdateFlagsFromConf(curve::common::Configuration* conf) { if (GetCommandLineFlagInfo("mdsAddr", &info) && info.is_default) { conf->GetStringValue("mdsAddr", &FLAGS_mdsAddr); } + if (GetCommandLineFlagInfo("mdsDummyPort", &info) && info.is_default) { + conf->GetStringValue("mdsDummyPort", &FLAGS_mdsDummyPort); + } if (GetCommandLineFlagInfo("etcdAddr", &info) && info.is_default) { conf->GetStringValue("etcdAddr", &FLAGS_etcdAddr); } diff --git a/util/build.sh b/util/build.sh new file mode 100644 index 0000000000..122f51defe --- /dev/null +++ b/util/build.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. + +############################ GLOBAL VARIABLES + +g_list=0 +g_target="" +g_release=0 +g_build_opts=( + "--define=with_glog=true" + "--define=libunwind=true" + "--copt -DHAVE_ZLIB=1" + "--copt -DGFLAGS_NS=google" + "--copt -DUSE_BTHREAD_MUTEX" +) + +############################ BASIC FUNCTIONS +msg() { + printf '%b' "$1" >&2 +} + +success() { + msg "\33[32m[✔]\33[0m ${1}${2}" +} + +die() { + msg "\33[31m[✘]\33[0m ${1}${2}" + exit 1 +} + +print_title() { + local delimiter=$(printf '=%.0s' {1..20}) + msg "$delimiter [$1] $delimiter\n" +} + +############################ FUNCTIONS +usage () { + cat << _EOC_ +Usage: + build.sh --list + build.sh --only=target +Examples: + build.sh --only=//src/chunkserver:chunkserver + build.sh --only=src/* + build.sh --only=test/* + build.sh --only=test/chunkserver +_EOC_ +} + +get_options() { + local args=`getopt -o lorh --long list,only:,release: -n "$0" -- "$@"` + eval set -- "${args}" + while true + do + case "$1" in + -l|--list) + g_list=1 + shift 1 + ;; + -o|--only) + g_target=$2 + shift 2 + ;; + -r|--release) + g_release=$2 + shift 2 + ;; + -h) + usage + exit 1 + ;; + --) + shift + break + ;; + *) + exit 1 + ;; + esac + done +} + +list_target() { + print_title " SOURCE TARGETS " + bazel query 'kind("cc_binary", //src/...)' + bazel query 'kind("cc_binary", //tools/...)' + bazel query 'kind("cc_binary", //nebd/src/...)' + bazel query 'kind("cc_binary", //nbd/src/...)' + print_title " TEST TARGETS " + bazel query 'kind("cc_(test|binary)", //test/...)' + bazel query 'kind("cc_(test|binary)", //nebd/test/...)' + bazel query 'kind("cc_(test|binary)", //nbd/test/...)' +} + +get_target() { + bazel query 'kind("cc_(test|binary)", //...)' | grep -E "$g_target" +} + +build_target() { + local targets + local tag=$(git describe --tags --abbrev=0) + local commit_id=$(git rev-parse --short HEAD) + local version="${tag}+${commit_id}" + declare -A result + if [ $g_release -eq 1 ]; then + g_build_opts+=("--compilation_mode=opt --copt -g") + version="${version}+release" + echo "release" > .BUILD_MODE + else + g_build_opts+=("--compilation_mode=dbg") + version="${version}+debug" + echo "debug" > .BUILD_MODE + fi + g_build_opts+=("--copt -DCURVEVERSION=${version}") + + for target in `get_target` + do + bazel build ${g_build_opts[@]} $target + local ret="$?" + targets+=("$target") + result["$target"]=$ret + if [ "$ret" -ne 0 ]; then + break + fi + done + + echo "" + print_title " BUILD SUMMARY " + for target in "${targets[@]}" + do + if [ "${result[$target]}" -eq 0 ]; then + success "$target ${version}\n" + else + die "$target ${version}\n" + fi + done +} + +main() { + get_options "$@" + + if [ "$g_list" -eq 1 ]; then + list_target + elif [ "$g_target" == "" ]; then + usage + exit 1 + else + build_target + fi +} + +############################ MAIN() +main "$@" diff --git a/util/image.sh b/util/image.sh new file mode 100644 index 0000000000..dbe9c1f8fd --- /dev/null +++ b/util/image.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# tmpl.sh = /usr/local/metaserver.conf /tmp/metaserver.conf +function tmpl() { + dsv=$1 + src=$2 + dst=$3 + regex="^([^$dsv]+$dsv[[:space:]]*)(.+)__CURVEADM_TEMPLATE__[[:space:]]+(.+)[[:space:]]+__CURVEADM_TEMPLATE__(.*)$" + while IFS= read -r line; do + if [[ ! $line =~ $regex ]]; then + echo "$line" + else + echo "${BASH_REMATCH[1]}${BASH_REMATCH[3]}" + fi + done < $src > $dst +} + +prefix="$(pwd)/docker/curvebs" +mkdir -p $prefix $prefix/conf +make install prefix="$prefix" +make install prefix="$prefix" only=etcd +for path in `ls conf/* nebd/etc/nebd/*`; +do + dir=`dirname $path` + file=`basename $path` + + # delimiter + dsv="=" + if [ $file = "etcd.conf" ]; then + dsv=": " + fi + + # destination + dst=$file + if [ $file = "snapshot_clone_server.conf" ]; then + dst="snapshotclone.conf" + fi + + tmpl $dsv "$dir/$file" "$prefix/conf/$dst" +done + +docker pull opencurvedocker/curve-base:debian9 +docker build -t "$1" "$(pwd)/docker" diff --git a/util/install.sh b/util/install.sh new file mode 100644 index 0000000000..990be7fada --- /dev/null +++ b/util/install.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash + +# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. + +# chunkserver +# ├── conf +# │   └── chunkserver.conf +# ├── logs +# │   └── chunkserver.pid +# ├── sbin +# │  ├── servicectl +# │  └── curvebs-chunkserver +# └── lib +#    └── chunkserver_lib.so + +############################ GLOBAL VARIABLES +g_prefix="" +g_only="" +g_project_name="" +g_etcd_version="v3.4.10" +g_util_dir="$(dirname $(realpath $0))" +g_curve_dir="$(dirname $g_util_dir)" +g_build_release=0 + +g_color_yellow=`printf '\033[33m'` +g_color_red=`printf '\033[31m'` +g_color_normal=`printf '\033[0m'` + +############################ BASIC FUNCTIONS +msg() { + printf '%b' "$1" >&2 +} + +success() { + msg "$g_color_yellow[✔]$g_color_normal [$g_project_name] ${1}${2}" +} + +die() { + msg "$g_color_red[✘]$g_color_normal [$g_project_name] ${1}${2}" + exit 1 +} + + +############################ FUNCTIONS +usage () { + cat << _EOC_ +Usage: + install.sh --prefix=PREFIX --only=TARGET + +Examples: + install.sh --prefix=/usr/local/curvebs --only=* + install.sh --prefix=/usr/local/curvebs --only=chunkserver + install.sh --prefix=/usr/local/curvebs --only=etcd --etcd_version="v3.4.0" +_EOC_ +} + +get_options() { + local long_opts="prefix:,only:,etcd_version:,help" + local args=`getopt -o povh --long $long_opts -n "$0" -- "$@"` + eval set -- "${args}" + while true + do + case "$1" in + -p|--prefix) + g_prefix=$2 + shift 2 + ;; + -o|--only) + g_only=$2 + shift 2 + ;; + -v|--etcd_version) + g_etcd_version=$2 + shift 2 + ;; + -h|--help) + usage + exit 1 + ;; + --) + shift + break + ;; + *) + exit 1 + ;; + esac + done +} + +get_build_mode() { + grep "release" .BUILD_MODE > /dev/null 2>&1 + if [ $? -eq 0 ]; then + g_build_release=1 + fi +} + +strip_debug_info() { + if [ $g_build_release -eq 1 ]; then + # binary file generated by bazel isn't writable by default + chmod +w $1 + objcopy --only-keep-debug $1 $1.dbg-sym + objcopy --strip-debug $1 + objcopy --add-gnu-debuglink=$1.dbg-sym $1 + chmod -w $1 + fi +} + +create_project_dir() { + mkdir -p $1/{conf,logs,sbin,lib,data} + if [ $? -eq 0 ]; then + success "create project directory $1 success\n" + else + die "create directory $1 failed\n" + fi +} + +copy_file() { + cp -f "$1" "$2" + if [ $? -eq 0 ]; then + success "copy file $1 to $2 success\n" + else + die "copy file $1 to $2 failed\n" + fi +} + +list_targets() { + bazel query 'kind("cc_binary", //src/...)' + bazel query 'kind("cc_binary", //tools/...)' + bazel query 'kind("cc_binary", //nebd/src/...)' + bazel query 'kind("cc_binary", //nbd/src/...)' +} + +get_targets() { + list_targets | grep -E "$g_only" +} + +install_curvebs() { + declare -A rename_binary + rename_binary["chunkserver"]="curvebs-chunkserver" + rename_binary["curvemds"]="curvebs-mds" + rename_binary["snapshotcloneserver"]="curvebs-snapshotclone" + rename_binary["curve_tool"]="curve_ops_tool" + rename_binary["curvefsTool"]="curvebs-tool" + + for target in `get_targets` + do + # //src/tools:curve_tool + # //src/mds/main:curvemds + # //tools:curvefsTool + # //nebd/src/part2:nebd-server + # //nbd/src:curve-nbd + local regex_target="//((src/)?([^:/]+)([^:]*)):(.+)" + if [[ ! $target =~ $regex_target ]]; then + die "unknown target: $target\n" + fi + + local project_name="${BASH_REMATCH[3]}" # ex: chunkserver + if [ $project_name == "snapshotcloneserver" ]; then + project_name="snapshotclone" + fi + g_project_name=$project_name + local project_prefix="$g_prefix/$project_name" # ex: /curvebs/chunkserver + + local project_bin_filename=${BASH_REMATCH[5]} # ex: curvebs_chunkserver + local bazel_dir="$g_curve_dir/bazel-bin" + local binary="$bazel_dir/${BASH_REMATCH[1]}/$project_bin_filename" + project_bin_filename=${rename_binary[$project_bin_filename]:-${project_bin_filename}} + + # The below actions: + # 1) create project directory + # 2) copy binary to project directory + # 3) strip debug information + create_project_dir $project_prefix + copy_file "$binary" "$project_prefix/sbin/$project_bin_filename" + get_build_mode && strip_debug_info "$project_prefix/sbin/$project_bin_filename" + + success "install $project_name success\n" + done +} + +download_etcd() { + local now=`date +"%s%6N"` + local nos_url="https://curve-build.nos-eastchina1.126.net" + local src="${nos_url}/etcd-${g_etcd_version}-linux-amd64.tar.gz" + local tmpfile="/tmp/$now-etcd-${g_etcd_version}-linux-amd64.tar.gz" + local dst="$1" + + msg "download etcd: $src to $dst\n" + + # download etcd tarball and decompress to dst directory + mkdir -p $dst && + curl -L $src -o $tmpfile && + tar -zxvf $tmpfile -C $dst --strip-components=1 >/dev/null 2>&1 + + local ret=$? + rm -rf $tmpfile + if [ $ret -ne 0 ]; then + die "download etcd-$g_etcd_version failed\n" + else + success "download etcd-$g_etcd_version success\n" + fi +} + +install_etcd() { + local project_name="etcd" + g_project_name=$project_name + + # The below actions: + # 1) download etcd tarball from github + # 2) create project directory + # 3) copy binary to project directory + # 4) generate servicectl script into project directory. + local now=`date +"%s%6N"` + local dst="/tmp/$now/etcd-$g_etcd_version" + download_etcd $dst + local project_prefix="$g_prefix/etcd" + create_project_dir $project_prefix + copy_file "$dst/etcd" "$project_prefix/sbin" + copy_file "$dst/etcdctl" "$project_prefix/sbin" + + rm -rf "$dst" + success "install $project_name success\n" +} + +main() { + get_options "$@" + + if [[ $g_prefix == "" || $g_only == "" ]]; then + usage + exit 1 + elif [ "$g_only" == "etcd" ]; then + install_etcd + else + install_curvebs + fi +} + +############################ MAIN() +main "$@"