Skip to content

Commit

Permalink
added new docker-compose (#37)
Browse files Browse the repository at this point in the history 
* added new docker-compose

* remove wait

* add back empty wait
  • Loading branch information
@roblaszczak
roblaszczak authored Jan 13, 2025
1 parent e81d941 commit d852df8
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 69 deletions.
5 changes: 2 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
up:
docker compose up -d
docker compose up -d --wait

test:
go test ./...
Expand All @@ -19,7 +19,7 @@ test_stress:
test_reconnect:
go test -tags=reconnect ./...

test_codecov: up wait
test_codecov: up
go test -coverprofile=coverage.out -covermode=atomic ./...

fmt:
Expand All @@ -30,7 +30,6 @@ build:
go build ./...

wait:
go run github.com/ThreeDotsLabs/wait-for@latest localhost:9091 localhost:9092 localhost:9093 localhost:9094 localhost:9095

update_watermill:
go get -u github.com/ThreeDotsLabs/watermill
Expand Down
9 changes: 0 additions & 9 deletions dev/docker-kafka.env
View file

This file was deleted.

182 changes: 126 additions & 56 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -1,82 +1,152 @@
# for Watermill development purposes.
# For Watermill based application docker please check https://watermill.io/docs/getting-started/

version: '3'
# Generated by https://github.com/sknop/kafka-docker-composer
---
services:
zookeeper:
image: confluentinc/cp-zookeeper:7.3.1
restart: unless-stopped
zookeeper-1:
image: confluentinc/cp-zookeeper:7.8.0
hostname: zookeeper-1
container_name: zookeeper-1

environment:
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_CLIENT_PORT: "2181"
ZOOKEEPER_TICK_TIME: "2000"
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
KAFKA_JMX_PORT: 10001
KAFKA_JMX_HOSTNAME: localhost
ZOOKEEPER_SERVERS: zookeeper-1:2888:3888
cap_add:
- NET_ADMIN
ports:
- 2181:2181
- 10001:10001
- 10101:8091

kafka-1:
image: confluentinc/cp-server:7.8.0
hostname: kafka-1
container_name: kafka-1

kafka1:
image: confluentinc/cp-kafka:7.3.1
healthcheck:
test: curl -fail --silent http://kafka-1:8090/kafka/v3/clusters/ --output /dev/null || exit 1
interval: 10s
retries: 10
start_period: 20s
depends_on:
- zookeeper
env_file:
- dev/docker-kafka.env
- zookeeper-1
environment:
KAFKA_LISTENERS: PLAINTEXT://kafka-1:19091, EXTERNAL://0.0.0.0:9091
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:19091, EXTERNAL://localhost:9091
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JMX_PORT: 10002
KAFKA_JMX_HOSTNAME: localhost
KAFKA_BROKER_RACK: rack-0
KAFKA_MIN_INSYNC_REPLICAS: 2
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: True
KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: True
KAFKA_DEFAULT_REPLICATION_FACTOR: 3
KAFKA_BROKER_ID: 1
KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9091,INTERNAL://0.0.0.0:29091
KAFKA_ADVERTISED_LISTENERS: EXTERNAL://localhost:9091,INTERNAL://kafka1:29091
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
KAFKA_CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 3
KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-1:19091,kafka-2:19092,kafka-3:19093
cap_add:
- NET_ADMIN
ports:
- 9091:9091
restart: unless-stopped
- 10002:10002
- 10102:8091
- 10201:8090

kafka2:
image: confluentinc/cp-kafka:7.3.1
kafka-2:
image: confluentinc/cp-server:7.8.0
hostname: kafka-2
container_name: kafka-2

healthcheck:
test: curl -fail --silent http://kafka-2:8090/kafka/v3/clusters/ --output /dev/null || exit 1
interval: 10s
retries: 10
start_period: 20s
depends_on:
- zookeeper
env_file:
- dev/docker-kafka.env
- zookeeper-1
environment:
KAFKA_LISTENERS: PLAINTEXT://kafka-2:19092, EXTERNAL://0.0.0.0:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:19092, EXTERNAL://localhost:9092
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JMX_PORT: 10003
KAFKA_JMX_HOSTNAME: localhost
KAFKA_BROKER_RACK: rack-0
KAFKA_MIN_INSYNC_REPLICAS: 2
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: True
KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: True
KAFKA_DEFAULT_REPLICATION_FACTOR: 3
KAFKA_BROKER_ID: 2
KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9092,INTERNAL://0.0.0.0:29092
KAFKA_ADVERTISED_LISTENERS: EXTERNAL://localhost:9092,INTERNAL://kafka2:29092
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
KAFKA_CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 3
KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-1:19091,kafka-2:19092,kafka-3:19093
cap_add:
- NET_ADMIN
ports:
- 9092:9092
restart: unless-stopped
- 10003:10003
- 10103:8091
- 10202:8090

kafka-3:
image: confluentinc/cp-server:7.8.0
hostname: kafka-3
container_name: kafka-3

kafka3:
image: confluentinc/cp-kafka:7.3.1
healthcheck:
test: curl -fail --silent http://kafka-3:8090/kafka/v3/clusters/ --output /dev/null || exit 1
interval: 10s
retries: 10
start_period: 20s
depends_on:
- zookeeper
env_file:
- dev/docker-kafka.env
- zookeeper-1
environment:
KAFKA_LISTENERS: PLAINTEXT://kafka-3:19093, EXTERNAL://0.0.0.0:9093
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:19093, EXTERNAL://localhost:9093
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JMX_PORT: 10004
KAFKA_JMX_HOSTNAME: localhost
KAFKA_BROKER_RACK: rack-0
KAFKA_MIN_INSYNC_REPLICAS: 2
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_CONFLUENT_CLUSTER_LINK_ENABLE: True
KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: True
KAFKA_DEFAULT_REPLICATION_FACTOR: 3
KAFKA_BROKER_ID: 3
KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9093,INTERNAL://0.0.0.0:29093
KAFKA_ADVERTISED_LISTENERS: EXTERNAL://localhost:9093,INTERNAL://kafka3:29093
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
KAFKA_CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 3
KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-1:19091,kafka-2:19092,kafka-3:19093
cap_add:
- NET_ADMIN
ports:
- 9093:9093
restart: unless-stopped
- 10004:10004
- 10104:8091
- 10203:8090

kafka4:
image: confluentinc/cp-kafka:7.3.1
depends_on:
- zookeeper
env_file:
- dev/docker-kafka.env
environment:
KAFKA_BROKER_ID: 4
KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9094,INTERNAL://0.0.0.0:29094
KAFKA_ADVERTISED_LISTENERS: EXTERNAL://localhost:9094,INTERNAL://kafka4:29094
ports:
- 9094:9094
restart: unless-stopped

kafka5:
image: confluentinc/cp-kafka:7.3.1
depends_on:
- zookeeper
env_file:
- dev/docker-kafka.env
environment:
KAFKA_BROKER_ID: 5
KAFKA_LISTENERS: EXTERNAL://0.0.0.0:9095,INTERNAL://0.0.0.0:29095
KAFKA_ADVERTISED_LISTENERS: EXTERNAL://localhost:9095,INTERNAL://kafka5:29095
ports:
- 9095:9095
restart: unless-stopped
kafka-ui:
image: "provectuslabs/kafka-ui:latest"
container_name: kafka-ui
ports:
- "8080:8080"
environment:
- KAFKA_CLUSTERS_0_BOOTSTRAP_SERVERS=kafka-1:19091
depends_on:
- kafka-1
2 changes: 1 addition & 1 deletion pkg/kafka/pubsub_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func kafkaBrokers() []string {
if brokers != "" {
return strings.Split(brokers, ",")
}
return []string{"localhost:9091", "localhost:9092", "localhost:9093", "localhost:9094", "localhost:9095"}
return []string{"localhost:9091", "localhost:9092", "localhost:9093"}
}

func newPubSub(t *testing.T, marshaler kafka.MarshalerUnmarshaler, consumerGroup string) (*kafka.Publisher, *kafka.Subscriber) {
Expand Down

0 comments on commit d852df8

Please sign in to comment.