Skip to content

Commit 6dfb6ce

Browse files
author
Neil Buesing
committed
initial version
0 parents  commit 6dfb6ce

File tree

78 files changed

+4125
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+4125
-0
lines changed

‎.gitignore

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
.DS_Store
2+
.idea
3+
.gradle
4+
grafana.db
5+
build
6+
run.sh
7+
/tmp
8+
/streams/tmp
9+
.settings
10+
.classpath
11+
.project
12+
/druid/storage
13+
/applications/stores/*
14+
15+
/statestore-ui/node_modules

‎LICENSE

+549
Large diffs are not rendered by default.

‎applications/.env

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#kafka-streams-dashboards-applications
2+
COMPOSE_PROJECT_NAME=ksda

‎applications/docker-compose.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
networks:
2+
default:
3+
external: true
4+
name: ksd
5+
6+
services:
7+
8+
stream_v1:
9+
image: nbuesing/ksd_streams:latest
10+
#hostname: stream
11+
#container_name: stream
12+
stop_grace_period: 120s
13+
healthcheck:
14+
test: /healthcheck.sh || exit 1
15+
start_period: 10s
16+
interval: 10s
17+
timeout: 5s
18+
retries: 3
19+
deploy:
20+
replicas: 2
21+
volumes:
22+
- ./streams.properties:/streams.properties
23+
- ./scripts:/scripts
24+
environment:
25+
BOOTSTRAP_SERVERS: broker-1:9092,broker-2:9092,broker-3:9092
26+
CLIENT_ID_PREFIX: "stream_v1"
27+
28+
stream_v2:
29+
image: nbuesing/ksd_streams:latest
30+
#hostname: stream
31+
#container_name: stream
32+
stop_grace_period: 120s
33+
healthcheck:
34+
test: /healthcheck.sh || exit 1
35+
start_period: 10s
36+
interval: 10s
37+
timeout: 5s
38+
retries: 3
39+
deploy:
40+
replicas: 2
41+
volumes:
42+
- ./streams.properties:/streams.properties
43+
- ./scripts:/scripts
44+
environment:
45+
BOOTSTRAP_SERVERS: broker-1:9092,broker-2:9092,broker-3:9092
46+
CLIENT_ID_PREFIX: "stream_v2"

‎applications/nginx.conf

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
2+
events {
3+
worker_connections 1024;
4+
}
5+
6+
http {
7+
server {
8+
location /tumbling {
9+
proxy_pass http://analytics_tumbling:8080;
10+
rewrite ^/tumbling(.*)$ /$1 break;
11+
12+
proxy_set_header HOST $host;
13+
proxy_set_header X-Forwarded-For $remote_addr;
14+
proxy_set_header 'Access-Control-Allow-Origin' '*';
15+
proxy_set_header 'Access-Control-Allow-Credentials' 'true';
16+
proxy_set_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS';
17+
proxy_set_header 'Access-Control-Allow-Headers' 'X-LOC,Location,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
18+
19+
}
20+
location /hopping {
21+
proxy_pass http://analytics_hopping:8080;
22+
rewrite /hopping(.*)$ /$1 break;
23+
24+
proxy_set_header HOST $host;
25+
proxy_set_header X-Forwarded-For $remote_addr;
26+
proxy_set_header 'Access-Control-Allow-Origin' '*';
27+
proxy_set_header 'Access-Control-Allow-Credentials' 'true';
28+
proxy_set_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS';
29+
proxy_set_header 'Access-Control-Allow-Headers' 'X-LOC,Location,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
30+
31+
}
32+
location /sliding {
33+
proxy_pass http://analytics_sliding:8080;
34+
rewrite /sliding(.*)$ /$1 break;
35+
36+
proxy_set_header HOST $host;
37+
proxy_set_header X-Forwarded-For $remote_addr;
38+
proxy_set_header 'Access-Control-Allow-Origin' '*';
39+
proxy_set_header 'Access-Control-Allow-Credentials' 'true';
40+
proxy_set_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS';
41+
proxy_set_header 'Access-Control-Allow-Headers' 'X-LOC,Location,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
42+
43+
}
44+
45+
location /session {
46+
proxy_pass http://analytics_session:8080;
47+
rewrite /session(.*)$ /$1 break;
48+
49+
proxy_set_header HOST $host;
50+
proxy_set_header X-Forwarded-For $remote_addr;
51+
proxy_set_header 'Access-Control-Allow-Origin' '*';
52+
proxy_set_header 'Access-Control-Allow-Credentials' 'true';
53+
proxy_set_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS';
54+
proxy_set_header 'Access-Control-Allow-Headers' 'X-LOC,Location,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
55+
56+
}
57+
58+
location /none {
59+
proxy_pass http://analytics_none:8080;
60+
rewrite /session(.*)$ /$1 break;
61+
62+
proxy_set_header HOST $host;
63+
proxy_set_header X-Forwarded-For $remote_addr;
64+
proxy_set_header 'Access-Control-Allow-Origin' '*';
65+
proxy_set_header 'Access-Control-Allow-Credentials' 'true';
66+
proxy_set_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS';
67+
proxy_set_header 'Access-Control-Allow-Headers' 'X-LOC,Location,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
68+
69+
}
70+
}
71+
}

‎applications/prometheus_setup.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
if ! [ -x "$(command -v jq)" ]; then
4+
echo "jq is not installed." >&2
5+
exit 1
6+
fi
7+
8+
if ! [ -x "$(command -v docker)" ]; then
9+
echo "docker is not installed." >&2
10+
exit 1
11+
fi
12+
13+
# TODO check for docker compose
14+
15+
docker info > /dev/null 2>&1
16+
if [ $? -ne 0 ]; then
17+
echo "docker server is not running." >&2
18+
exit 1
19+
fi
20+
21+
cd $(dirname $0)
22+
23+
# collect all of the stream hostnames and create a comma delimited string with host:port
24+
#
25+
for i in $(docker compose ps stream --format json | jq -r ".[] | .Name"); do
26+
INPUT+="\"${i}:7071\","
27+
done
28+
INPUT=$(echo $INPUT | sed 's/,$//g')
29+
30+
31+
TEMPLATE=$(cat <<EOF
32+
[
33+
{
34+
"targets": \$v,
35+
"labels": {
36+
"job": "streams"
37+
"cluster_type": "streams",
38+
"cluster_id": "streams"
39+
}
40+
}
41+
]
42+
EOF
43+
)
44+
45+
46+
FILE=../monitoring/prometheus/application_streams.json
47+
48+
# take template and add each host:port as an element of that array
49+
#
50+
jq -n --argjson v "[ $INPUT ]" "$TEMPLATE" > $FILE
51+
52+
cat $FILE | jq
53+
54+
# force Prometheus to reload the sd_file immediately, --web.enable-lifecycle must be enabled in Prometheus.
55+
#
56+
curl -X POST http://localhost:9090/-/reload

‎applications/scripts/broker_latency.sh

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
LATENCY="${1}ms"
4+
shift
5+
6+
echo "latency: $LATENCY"
7+
8+
tc qdisc del dev eth0 root
9+
tc qdisc add dev eth0 root handle 1: prio
10+
tc qdisc add dev eth0 parent 1:3 handle 30: netem delay ${LATENCY}
11+
12+
13+
declare -a HOSTS=(broker-1 broker-2 broker-3 broker-4)
14+
15+
for host in "${HOSTS[@]}"; do
16+
17+
ip=$(ping -c 1 $host | head -1 | awk -F'[()]' '/PING/{print $2}')
18+
19+
echo "$host : $ip"
20+
tc filter add dev eth0 protocol ip parent 1:0 prio 3 u32 match ip dst $ip flowid 1:3
21+
22+
done
23+
24+
25+

‎applications/streams.properties

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
num.stream.threads=2

‎build.gradle

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
plugins {
2+
id 'idea'
3+
}
4+
5+
allprojects {
6+
repositories {
7+
mavenLocal()
8+
mavenCentral()
9+
// maven {
10+
// url "https://packages.confluent.io/maven/"
11+
// }
12+
// // confluent's schema registry client depends on 'com.github.everit-org.json-schema:org.everit.json.schema'.
13+
// // the version, for some reason, is not stored on maven central, and mavenrepository.com says it is located
14+
// // here.
15+
// maven {
16+
// url "https://repository.mulesoft.org/nexus/content/repositories/public"
17+
// }
18+
}
19+
}
20+
21+
subprojects {
22+
version = '1.0'
23+
}
24+
25+
26+
configure(subprojects.findAll { project -> file("${project.name}/src/main/java").exists() }) {
27+
28+
apply plugin: 'java'
29+
apply plugin: 'application'
30+
apply plugin: 'eclipse'
31+
32+
group = 'dev.buesing.ksd'
33+
description = "${project.name}"
34+
sourceCompatibility = JavaVersion.VERSION_18
35+
targetCompatibility = JavaVersion.VERSION_18
36+
37+
idea {
38+
module {
39+
inheritOutputDirs = false
40+
outputDir = compileJava.destinationDir
41+
testOutputDir = compileTestJava.destinationDir
42+
}
43+
}
44+
45+
dependencies {
46+
47+
implementation group: 'ch.qos.logback', name: 'logback-classic', version: logback_version
48+
implementation group: 'com.beust', name: 'jcommander', version: jcommander_version
49+
implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: jackson_version
50+
implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jackson_version
51+
implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: jackson_version
52+
implementation group: 'org.apache.commons', name: 'commons-lang3', version: apache_commons_version
53+
implementation(group: 'org.apache.kafka', name: 'kafka-clients', version: kafka_version) {
54+
version {
55+
strictly kafka_version
56+
}
57+
}
58+
implementation group: 'org.slf4j', name: 'slf4j-api', version: slf4j_version
59+
60+
// lombok dependencies
61+
compileOnly group: 'org.projectlombok', name: 'lombok', version: lombok_version
62+
annotationProcessor group: 'org.projectlombok', name: 'lombok', version: lombok_version
63+
64+
testCompileOnly group: 'org.projectlombok', name: 'lombok', version: lombok_version
65+
testAnnotationProcessor group: 'org.projectlombok', name: 'lombok', version: lombok_version
66+
}
67+
68+
test {
69+
useJUnitPlatform()
70+
}
71+
72+
}
73+
74+
configure(subprojects.findAll { project -> file("${project.name}/src/main/java").exists() && "${project.name}" != "common" }) {
75+
76+
task buildRunScript() {
77+
doLast {
78+
79+
def cp = sourceSets.main.runtimeClasspath.collect{"export CP=\"\${CP}:${it}\""}.join("\n")
80+
81+
def f = new File(projectDir, "run.sh")
82+
f.text = """#!/bin/sh
83+
set -e
84+
gradle assemble > /dev/null
85+
86+
export CP=""
87+
${cp}
88+
89+
java -cp \${CP} dev.buesing.ksd.${project.name}.Main \$*
90+
"""
91+
f.setExecutable(true)
92+
}
93+
}
94+
95+
build.finalizedBy buildRunScript
96+
97+
}
98+

‎builder/build.gradle

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
2+
dependencies {
3+
implementation project(':common')
4+
implementation group: 'org.apache.commons', name: 'commons-csv', version: apache_commons_csv_version
5+
}
6+
7+
application {
8+
mainClass = "dev.buesing.ksd.${project.name}.Main"
9+
}

‎builder/lombok.config

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
lombok.addLombokGeneratedAnnotation = true
2+

0 commit comments

Comments
 (0)