Add a deployment method using docker-compose

Descriptions of the changes in this PR: ### Motivation Provide a deployment method using docker-compose. ### Changes - Use helm as the template engine to generate docker compose file - Deploy zookeeper in cluster mode in docker-compose - Manages port mapping to make it able to access out side of the docker-composed cluster. Author: Khurrum Nasimm <[email protected]> Reviewers: Enrico Olivelli <[email protected]>, Sijie Guo <[email protected]> This closes apache#1530 from khurrumnasimm/docker_compose
horizonzy · Aug 13, 2018 · 83f78c3 · 83f78c3
1 parent 3ab6e92
commit 83f78c3
Show file tree

Hide file tree

Showing 6 changed files with 450 additions and 4 deletions.
diff --git a/conf/zookeeper.conf b/conf/zookeeper.conf
@@ -37,10 +37,6 @@ dataLogDir=data/zookeeper/txlog
 # the port at which the clients will connect
 clientPort=2181
 
-# the port at which the admin will listen
-adminPort=9990
-zookeeper.admin.enableServer=true
-
 # limit on queued clients - default: 1000
 globalOutstandingLimit=1000
 

diff --git a/deploy/docker-compose/README.md b/deploy/docker-compose/README.md
@@ -0,0 +1,61 @@
+# Apache BookKeeper docker-compose
+
+## Requirements
+
+* Docker >= 16.10
+* Docker Compose >= 1.6.0
+
+## Quick start
+
+```bash
+$ git clone https://github.com/apache/bookkeeper.git
+$ cd bookkeeper/deploy/docker-compose
+$ docker-compose pull # Get the latest Docker images
+$ docker-compose up -d
+$ cd ../../
+$ bin/bkctl bookies list
+$ bin/bkctl ledger simpletest
+```
+
+## Access Apache BookKeeper cluster
+
+
+### Ledger Service
+
+You can use `zk://localhost:2181/ledgers` as metadataServiceUri to access ledger storage service.
+
+```bash
+$ bin/bkctl -u 'zk://localhost:2181/ledgers' ledger simpletest 
+```
+
+### DistributedLog
+
+You can use `distributedlog://localhost:2181/distributedlog` as dlog uri to access ledger storage service
+using [distributedlog](http://bookkeeper.apache.org/docs/latest/api/distributedlog-api/) API.
+
+```bash
+$ bin/dlog tool create -u 'distributedlog://localhost:2181/distributedlog' --prefix test-stream -e 0-99
+```
+
+## Customize Apache BookKeeper Cluster
+
+### Install Helm
+
+[Helm](https://helm.sh) is used as a template render engine
+
+```
+curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+```
+
+Or if you use Mac, you can use homebrew to install Helm by `brew install kubernetes-helm`
+
+### Bring up Apache BookKeeper cluster
+
+```bash
+$ git clone https://github.com/apache/bookkeeper.git
+$ cd bookkeeper/deploy/docker-compose
+$ vi compose/values.yaml # custom cluster size, docker image, port mapping etc
+$ helm template compose > generated-docker-compose.yaml
+$ docker-compose -f generated-docker-compose.yaml pull # Get the latest Docker images
+$ docker-compose -f generated-docker-compose.yaml up -d
+```
diff --git a/deploy/docker-compose/compose/Chart.yaml b/deploy/docker-compose/compose/Chart.yaml
@@ -0,0 +1,28 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+apiVersion: v1
+description: apache-bookkeeper-docker-compose
+name: apache-bookkeeper-docker-compose
+version: 4.8.0
+home: https://github.com/apache/bookkeeper
+sources:
+  - https://github.com/apache/bookkeeper/deploy/docker-compose
+keywords:
+  - log storage
+  - stream storage
diff --git a/deploy/docker-compose/compose/templates/docker-compose.yml b/deploy/docker-compose/compose/templates/docker-compose.yml
@@ -0,0 +1,132 @@
+{{- /*
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+*/ -}}
+{{- $zkSize := .Values.zookeeper.size | int }}
+{{- $bkSize := .Values.bookkeeper.size | int }}
+{{- $zkAdminPort := .Values.zookeeper.adminPort | int }}
+{{- $zkClientPort := .Values.zookeeper.clientPort | int }}
+{{- $zkPeerPort := .Values.zookeeper.peerPort | int }}
+{{- $zkLeaderPort := .Values.zookeeper.leaderPort | int }}
+{{- $bookiePort := .Values.bookkeeper.bookiePort | int }}
+{{- $bookieGrpcPort := .Values.bookkeeper.bookieGrpcPort | int }}
+{{- $bookieHttpPort := .Values.bookkeeper.bookieHttpPort | int }}
+
+{{- define "zookeeper_servers" }}
+  {{- range until (.Values.zookeeper.size | int) }}
+    {{- if . -}}
+      ,
+    {{- end -}}
+    zookeeper-{{ . }}:2181
+  {{- end -}}
+{{- end -}}
+
+{{- define "metadata_service_uri" }}
+  {{- range until (.Values.zookeeper.size | int) }}
+    {{- if . -}}
+      ;
+    {{- end -}}
+    zookeeper-{{ . }}:2181
+  {{- end -}}
+{{- end -}}
+
+
+{{- define "zookeeper_server_list" }} 
+  {{- $zk := dict "servers" (list) -}}
+  {{- range until (.Values.zookeeper.size | int) }}
+    {{- $noop := printf "server.%d=zookeeper-%d:%d:%d:participant;0.0.0.0:%d" . . ($.Values.zookeeper.peerPort | int) ($.Values.zookeeper.leaderPort | int) ($.Values.zookeeper.clientPort | int) | append $zk.servers | set $zk "servers" -}} 
+  {{- end -}}
+  {{- join " " $zk.servers -}}
+{{- end -}}
+
+version: '3'
+
+services:
+  {{- range until $zkSize }}
+  zookeeper-{{ . }}:
+    image: {{ $.Values.zookeeper.image }}
+    hostname: zookeeper-{{ . }}
+    command: ["zookeeper"]
+    {{- if eq $.Values.networkMode "host" }}
+    network_mode: host
+    {{- else }}
+    ports:
+      - "{{ add $zkAdminPort . 1000 }}:{{ $zkAdminPort }}"
+      - "{{ add $zkClientPort . }}:{{ $zkClientPort }}"
+    {{- end }}
+    volumes:
+      - {{ $.Values.dataDir }}/zookeeper-{{ . }}/data:/data/zookeeper/data
+      - {{ $.Values.dataDir }}/zookeeper-{{ . }}/txlog:/data/zookeeper/txlog
+    environment:
+      - ZK_dataDir=/data/zookeeper/data
+      - ZK_dataLogDir=/data/zookeeper/txlog
+      - ZK_clientPort={{ $zkClientPort }}
+      - ZK_ID={{ . }}
+      - ZK_SERVERS={{- template "zookeeper_server_list" $ }}
+      - ZK_standaloneEnabled=false
+    healthcheck:
+      test: ["CMD", "curl", "-s", "http://localhost:{{ $zkAdminPort }}/commands/stat"]
+      interval: 60s
+      timeout: 3s
+      retries: 60
+    restart: on-failure
+  {{ end }}
+
+  {{- range until $bkSize }}
+  bookie-{{ . }}:
+    image: {{ $.Values.bookkeeper.image }}
+    hostname: bookie-{{ . }}
+    {{- if eq $.Values.networkMode "host" }}
+    network_mode: host
+    {{- else }}
+    ports:
+      {{- if eq $.Values.networkMode "host" }}
+      - "{{ add $bookiePort . }}:{{ $bookiePort }}"
+      {{- else }}
+      - "{{ add $bookiePort . }}:{{ add $bookiePort . }}"
+      {{- end }}
+      - "{{ add $bookieHttpPort . }}:{{ $bookieHttpPort }}"
+      - "{{ add $bookieGrpcPort . }}:{{ $bookieGrpcPort }}"
+    {{- end }}
+    volumes:
+      - {{ $.Values.dataDir }}/bookie-{{ . }}/journal:/data/bookkeeper/journal
+      - {{ $.Values.dataDir }}/bookie-{{ . }}/ledgers:/data/bookkeeper/ledgers
+    environment:
+      - BK_zkServers={{- template "zookeeper_servers" $ }}
+      - BK_zkLedgersRootPath=/ledgers
+      - BK_metadataServiceUri=zk://{{- template "metadata_service_uri" $ }}/ledgers
+      - BK_DATA_DIR=/data/bookkeeper
+      {{- if eq $.Values.networkMode "host" }}
+      - BK_bookiePort={{ $bookiePort }}
+      {{- else }}
+      - BK_advertisedAddress={{ $.Values.advertisedAddr }}
+      - BK_bookiePort={{ add $bookiePort . }}
+      {{- end }}
+      - BK_httpServerEnabled=true
+    depends_on:
+      {{- range until $zkSize }}
+      - "zookeeper-{{.}}"
+      {{- end }}
+    healthcheck:
+      test: ["CMD", "curl", "-s", "http://localhost:{{ $bookieHttpPort }}/heartbeat"]
+      interval: 60s
+      timeout: 3s
+      retries: 60
+    restart: on-failure
+  {{ end }}
diff --git a/deploy/docker-compose/compose/values.yaml b/deploy/docker-compose/compose/values.yaml
@@ -0,0 +1,45 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# data directories
+dataDir: ./data
+
+# advertised address that bookies used for advertising themselves.
+# host network mode is useless on Mac, so in order to let clients
+# on the host accessing bookies in the docker network, we can set
+# the advertisedAddr to let bookies advertised in a host network.
+advertisedAddr: 127.0.0.1
+
+# supported networkMode: bridge | host
+# host network mode is useless on Mac
+networkMode: bridge
+
+zookeeper:
+  size: 3
+  image: apachebookkeeper/bookkeeper-current:latest
+  adminPort: 8080
+  clientPort: 2181
+  peerPort: 2888
+  leaderPort: 3888
+
+bookkeeper:
+  size: 3
+  image: apachebookkeeper/bookkeeper-current:latest
+  bookiePort: 3181
+  bookieGrpcPort: 4181
+  bookieHttpPort: 8080