adding ELK stack

docker-compose.yml

@@ -3,6 +3,8 @@ version: "3.8"
 services:
   zookeeper:
     image: "docker.io/bitnami/zookeeper:3-debian-10"
+    container_name: zookeeper
+    hostname: zookeeper
     ports:
       - "2181:2181"
     volumes:
@@ -13,6 +15,8 @@ services:
       - iot_network
   kafka:
     image: "docker.io/bitnami/kafka:2-debian-10"
+    container_name: kafka
+    hostname: kafka
     ports:
       - "9092:9092"
       - "9093:9093"
@@ -31,7 +35,6 @@ services:
       - zookeeper
     links:
       - zookeeper
-
   producer:
     build:
       context: ./kafka-producer
@@ -43,6 +46,50 @@ services:
       - iot_network
     depends_on:
       - kafka
+    links:
+      - kafka
+
+  elasticsearch:
+    image: elasticsearch:7.7.0
+    container_name: elasticsearch
+    hostname: elasticsearch
+    environment:
+      - "discovery.type=single-node"
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    networks:
+      - iot_network
+  kibana:
+    image: kibana:7.7.0
+    container_name: kibana
+    hostname: kibana
+    ports:
+      - "5601:5601"
+    links:
+      - elasticsearch:elasticsearch
+    depends_on:
+      - elasticsearch
+    networks:
+      - iot_network
+  logstash:
+    image: logstash:7.7.0
+    container_name: logstash
+    hostname: logstash
+    ports:
+      - "9600:9600"
+      - "8089:8089"
+    volumes:
+      - ./logstash:/usr/share/logstash/pipeline/
+    links:
+      - elasticsearch:elasticsearch
+      - kafka
+    depends_on:
+      - elasticsearch
+      - kafka
+    networks:
+      - iot_network
+
 
 volumes:
   zookeeper_data:
@@ -52,4 +99,4 @@ volumes:
 
 networks:
   iot_network:
-    driver: bridge
+    driver: bridge

kafka-producer/udp_to_kafka.py

@@ -18,7 +18,6 @@
 while True:
     data, addr = sock.recvfrom(1024)
     row = data.decode("utf-8").split(",")
-    print(row)
     payload = {
         "timestamp": datetime.now().isoformat(),
         "sender": addr[0],
@@ -40,4 +39,4 @@
             float(row[11]),
             float(row[12]),
         )
-    kafka_producer.send("my-topic", payload)
+    kafka_producer.send("smartphone-sensor", payload)

logstash/logstash.conf

@@ -0,0 +1,15 @@
+input {
+  kafka {
+    bootstrap_servers => "kafka:9092"
+    topics => ["smartphone-sensor"]
+  }
+}
+filter{
+  json{
+    source => "message"
+    remove_field => ["message"]
+  }
+}
+output {
+  elasticsearch { hosts => ["elasticsearch:9200"] }
+}