initial commit

.gitignore

@@ -0,0 +1,2 @@
+.idea
+cmd

Makefile

@@ -0,0 +1,32 @@
+GOBIN=./cmd/main
+
+## build: Build go binary
+build:
+	go build -o $(GOBIN)
+
+## run: Run go server
+.PHONY: run
+run:
+	$(GOBIN)
+
+## get: Run go get missing dependencies
+get:
+	go get ./...
+
+## deploy: Run commands to deploy app to container
+deploy:
+	make get
+	GOOS=linux GOARCH=amd64 go build \
+          -ldflags='-w -s -extldflags "-static"' -a \
+          -o /go/bin/main .
+
+.PHONY: help
+all: help
+help: Makefile
+	@echo
+	@echo " Choose a command"
+	@echo
+	@sed -n 's/^##//p' $< | column -t -s ':' |  sed -e 's/^/ /'
+	@echo
+
+.DEFAULT_GOAL := help

README.md

@@ -0,0 +1,46 @@
+# Kafka server client example
+## Installation
+### Docker
+You need pre install kafka and zookeeper in docker.
+Enter at deploy folder and run
+```sh
+$ docker-compose up -d
+```
+This command automatically installs kafka and zookeeper and starts them.
+Kafka default expose port `<you docker ip>:9092`
+### Go cleint and server
+For install all dependencies run
+```sh
+$ make get
+```
+Then build your binary
+```sh
+$ make build
+```
+
+## Run
+For starting consumer start your binary
+```sh
+$ ./cmd/main --act consumer
+```
+For Windows:
+```sh
+$ ./cmd/main.exe --act consumer
+```
+For starting producer
+```sh
+$ ./cmd/main --act producer
+```
+For Windows:
+```sh
+$ ./cmd/main.exe --act producer
+```
+## Usage
+For publish message at broker in producer cli simply write next command to cli
+```sh
+$ send###<your message here>
+```
+`send` this is event. So far, there is one event in the app.
+
+Then in consumer cli you will see that message delivered.
+

client/client.go

@@ -0,0 +1,11 @@
+package client
+
+import (
+	"kafka-server-client/client/consumer"
+	"kafka-server-client/configuration"
+)
+
+func Start(brokers []string, topic ...string) {
+	cons := consumer.New(brokers, configuration.NewKafkaConfiguration())
+	consumer.Start(cons, topic...)
+}

client/consumer/consumer.go

@@ -0,0 +1,89 @@
+package consumer
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"github.com/Shopify/sarama"
+	"kafka-server-client/events"
+	"log"
+	"os"
+	"strings"
+)
+
+func New(brokers []string, config ...*sarama.Config) sarama.Consumer {
+	var conf *sarama.Config
+	if len(config) > 0 {
+		conf = config[0]
+	}
+	cons, err := sarama.NewConsumer(brokers, conf)
+	if err != nil {
+		log.Println("NEW FUNC")
+		log.Fatal(err)
+	}
+
+	return cons
+}
+
+func Consume(consumer sarama.PartitionConsumer) {
+	for {
+		select {
+		case err := <- consumer.Errors():
+			fmt.Println(err)
+		case msg := <-consumer.Messages():
+			message := msg.Value
+			var logs events.Event
+			if err := json.Unmarshal(message, &logs); err != nil {
+				fmt.Printf("can not unmarshal value: %v", string(message))
+			} else {
+				switch logs.Type {
+				case "send":
+					var event events.MessageEvent
+					if err := json.Unmarshal(message, &event); err != nil {
+						log.Println(err)
+					}
+					fmt.Printf("new message: %s", event.Message)
+				default:
+					fmt.Println("unknown command")
+				}
+			}
+		}
+	}
+}
+
+func Start(consumer sarama.Consumer, topic ...string) {
+	defer consumer.Close()
+
+	go ConsumeAllTopics(topic, consumer)
+
+	fmt.Println("press ENTER to exit\n")
+	bufio.NewReader(os.Stdin).ReadString('\n')
+	fmt.Println("terminating...")
+}
+
+func ConsumeAllTopics(topics []string, master sarama.Consumer) {
+	for _, topic := range topics {
+		if strings.Contains(topic, "__consumer_offsets") {
+			continue
+		}
+		partitions, _ := master.Partitions(topic)
+		// this only consumes partition no 1, you would probably want to consume all partitions
+		consumer, err := master.ConsumePartition(topic, partitions[0], sarama.OffsetOldest)
+		if nil != err {
+			fmt.Printf("Topic %v Partitions: %v", topic, partitions)
+			log.Fatal(err)
+		}
+		fmt.Println(" Start consuming topic ", topic)
+		go func(topic string, consumer sarama.PartitionConsumer) {
+			for {
+				select {
+				case consumerError := <-consumer.Errors():
+					fmt.Println("consumerError: ", consumerError.Err)
+
+				case msg := <-consumer.Messages():
+					fmt.Println("Got message on topic ", topic, string(msg.Value))
+				}
+			}
+		}(topic, consumer)
+	}
+}

configuration/config.go

@@ -0,0 +1,15 @@
+package configuration
+
+import "github.com/Shopify/sarama"
+
+func NewKafkaConfiguration() *sarama.Config {
+	conf := sarama.NewConfig()
+	conf.ClientID = "kafka-server-client"
+	conf.Producer.RequiredAcks = sarama.WaitForAll
+	conf.Producer.Return.Successes = true
+	conf.Consumer.Return.Errors = true
+	conf.ChannelBufferSize = 1
+	conf.Version = sarama.V0_10_1_0
+
+	return conf
+}

deploy/docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.5'
+
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    container_name: zookeeper
+    ports: 
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+
+  kafka:
+    image: wurstmeister/kafka
+    ports:
+      - "9092:9092"
+    expose:
+      - "9093"
+    hostname: kafka
+    environment:
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://192.168.99.100:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+    depends_on:
+      - zookeeper
+

events/event.go

@@ -0,0 +1,6 @@
+package events
+
+type Event struct {
+	ID int `json:"ID,omitempty"`
+	Type string `json:"Type,omitempty"`
+}

events/new_message_event.go

@@ -0,0 +1,17 @@
+package events
+
+type MessageEvent struct {
+	Event
+	Message string `json:"Message,omitempty"`
+}
+
+func (m MessageEvent) Process() error {
+	return nil
+}
+
+func NewMessageEvent(message string) MessageEvent {
+	return MessageEvent{
+		Event: Event{Type: "send"},
+		Message: message,
+	}
+}

events/processor.go

@@ -0,0 +1,5 @@
+package events
+
+type Processor interface {
+	Process() error
+}

go.mod

@@ -0,0 +1,5 @@
+module kafka-server-client
+
+go 1.15
+
+require github.com/Shopify/sarama v1.27.2

go.sum

@@ -0,0 +1,70 @@
+github.com/Shopify/sarama v1.27.2 h1:1EyY1dsxNDUQEv0O/4TsjosHI2CgB1uo9H/v56xzTxc=
+github.com/Shopify/sarama v1.27.2/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/confluentinc/confluent-kafka-go v1.5.2 h1:l+qt+a0Okmq0Bdr1P55IX4fiwFJyg0lZQmfHkAFkv7E=
+github.com/confluentinc/confluent-kafka-go v1.5.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
+github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
+github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg=
+github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
+github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2 h1:g0WBLy6fobNUU8W/e9zx6I0Yl79Ya+BDW1NwzAlTiiQ=
+gopkg.in/confluentinc/confluent-kafka-go.v1 v1.5.2/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY=
+gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
+gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg=
+gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
+gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
+gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"flag"
+	"github.com/Shopify/sarama"
+	"kafka-server-client/client"
+	"kafka-server-client/server"
+	"log"
+	"os"
+)
+
+var (
+	brokers = []string { "192.168.99.100:9092" }
+	topic = "chart1"
+	topics = []string { topic, "chart2" }
+)
+
+func main() {
+	sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags)
+	act := flag.String("act", "consumer", "producer or consumer")
+	flag.Parse()
+
+	switch *act {
+	case "producer":
+		server.Start(brokers, topic)
+	case "consumer":
+		client.Start(brokers, topics...)
+	}
+}