Di 335

.drone.yml

@@ -0,0 +1,19 @@
+# This file was generated by gygdev. Do not manually modify it
+pipeline:
+  build-image:
+    image: hub.gyg.io/drone-plugins/gyg-registry
+    privileged: true
+    context: .
+    dockerfile: ./Dockerfile
+    repo: hub.gyg.io/service/burrow
+    tags:
+      - ${DRONE_BUILD_NUMBER}.${DRONE_COMMIT_SHA:0:7}
+    when:
+      event: push
+  register-dss:
+    image: hub.gyg.io/drone-plugins/dss
+    privileged: true
+    tags:
+      - ${DRONE_BUILD_NUMBER}.${DRONE_COMMIT_SHA:0:7}
+    when:
+      event: push

.gitignore

@@ -1,12 +1,4 @@
-burrow-src
-.*.swp
-!config
-dist
-log
-.idea
-*.cov
-Burrow
-Burrow.exe
-Burrow.iml
-tmp
-vendor
+/.gygdev-dc.yml
+/.gygdev_generated
+/.config
+/.secrets

DESIGN.md

@@ -0,0 +1,10 @@
+# Burrow
+
+## Description
+
+This repo is a fork from the linkedin project. It analyses and exposes metrics
+over http. Then a custom datadog check is consuming it an produces `burrow.*`
+metrics.
+
+As it's just a fork turned into a gygservice, if you want to know more about how
+it works, please read the readme or [developers documentation](https://github.com/linkedin/Burrow/wiki).

Dockerfile

@@ -2,14 +2,14 @@ FROM golang:1.11-alpine as builder
 
 ENV DEP_VERSION="0.5.0"
 RUN apk add --no-cache git curl gcc libc-dev && \
-	curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep && \
-	chmod +x $GOPATH/bin/dep && \
-	mkdir -p $GOPATH/src/github.com/linkedin/Burrow
+    curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep && \
+    chmod +x $GOPATH/bin/dep && \
+    mkdir -p $GOPATH/src/github.com/linkedin/Burrow
 
 ADD . $GOPATH/src/github.com/linkedin/Burrow/
 RUN cd $GOPATH/src/github.com/linkedin/Burrow && \
-	dep ensure && \
-	go build -o /tmp/burrow .
+    dep ensure && \
+    go build -o /tmp/burrow .
 
 FROM iron/go
 LABEL maintainer="LinkedIn Burrow https://github.com/linkedin/Burrow"
@@ -18,4 +18,6 @@ WORKDIR /app
 COPY --from=builder /tmp/burrow /app/
 ADD /docker-config/burrow.toml /etc/burrow/
 
-CMD ["/app/burrow", "--config-dir", "/etc/burrow"]
+EXPOSE 8000
+
+CMD ["/app/burrow", "--config-dir", "/etc/burrow"]

config/burrow.toml

@@ -19,7 +19,7 @@ root-path="/burrow"
 
 [client-profile.test]
 client-id="burrow-test"
-kafka-version="0.10.0"
+kafka-version="2.1.0"
 
 [cluster.local]
 class-name="kafka"

core/internal/helpers/sarama.go

@@ -14,6 +14,8 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"io/ioutil"
+	"log"
+	"os"
 
 	"github.com/Shopify/sarama"
 	"github.com/spf13/viper"
@@ -78,6 +80,13 @@ func GetSaramaConfigFromClientProfile(profileName string) *sarama.Config {
 	saramaConfig.Version = parseKafkaVersion(viper.GetString(configRoot + ".kafka-version"))
 	saramaConfig.Consumer.Return.Errors = true
 
+	if (viper.IsSet("logging")) {
+		viper.SetDefault("logging.level", "info")
+		if (viper.GetString("logging.level") == "debug") {
+			sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags)
+		}
+	}
+
 	// Configure TLS if enabled
 	if viper.IsSet(configRoot + ".tls") {
 		tlsName := viper.GetString(configRoot + ".tls")

core/internal/httpserver/config.go

@@ -186,6 +186,7 @@ func (hc *Coordinator) configEvaluatorDetail(w http.ResponseWriter, r *http.Requ
 			Module: httpResponseConfigModuleEvaluator{
 				ClassName:   viper.GetString(configRoot + ".class-name"),
 				ExpireCache: viper.GetInt64(configRoot + ".expire-cache"),
+				MinimumComplete: viper.GetInt64(configRoot + ".minimum-complete"),
 			},
 			Request: requestInfo,
 		})

core/internal/httpserver/structs.go

@@ -184,6 +184,7 @@ type httpResponseConfigModuleConsumer struct {
 type httpResponseConfigModuleEvaluator struct {
 	ClassName   string `json:"class-name"`
 	ExpireCache int64  `json:"expire-cache"`
+	MinimumComplete int64 `json:"minimum-complete"`
 }
 
 type httpResponseConfigModuleNotifierHTTP struct {

gyg-how-to-update.txt

@@ -0,0 +1,3 @@
+git remote add upstream https://github.com/linkedin/Burrow.git
+git fetch upstream
+git rebase v1.2.2 # replace by the release, dont take master

gygservice.yml

@@ -0,0 +1,36 @@
+version: '1'
+service:
+  name: burrow
+  owner: di
+  image_name: service/burrow
+  level: office hours
+  containers:
+    web:
+      resources:
+        requests:
+          memory: 256Mi
+          cpu: 1500m
+        limits:
+          memory: 256Mi
+          cpu: 1500m
+      dockerfile: Dockerfile
+      needs:
+        - kafka
+      secret_files:
+        burrow: /etc/burrow/
+      ports:
+      - name: http
+        container: 8000
+      livenessProbe:
+        httpGet:
+          path: burrow/admin
+          port: 8000
+        periodSeconds: 60
+        initialDelaySeconds: 60
+        successThreshold: 1
+        failureThreshold: 3
+        timeoutSeconds: 15
+      annotations:
+        ad.datadoghq.com/web.check_names: ["burrow"]
+        ad.datadoghq.com/web.init_configs: [{}]
+        ad.datadoghq.com/web.instances: [{"burrow_uri": "http://%%host%%:8000", "clusters": ["kafka1"]}]

secrets/production/burrow.json

@@ -0,0 +1,10 @@
+{
+ "data": {
+  "burrow.toml": {
+   "iv": "zoLTLFEmReExIuxbSLW6RA==",
+   "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAEoC+eSBIaBtxd8g1Dbri8AAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMHv95qqtigawXIGfCAgEQgDv++32NNEMD2I8ToDPouTw14NL0/y+PVIGjrPashxiijiA4y59kPqe4faR7k78ZkLneqS8Xy3YQIxgocg==",
+   "data": "wCeuH8fzE1GdGoQ34JycuysPwTroq2WsIq6NThKawxKsaWSxFO0v7tXwh9fN+UwdqvusMWb9sDGp9z8wxix+yVCL3kSHR7Y520iPh4nBCZYZE6gVADDnRSdwaUIW5vA+v4n3tLIqQcGVW58rxDT87iOkgy9df6Pv0pjmAN19g7SA+wxWCwaq4ZsvaoI6h5ov3vAL2DimOk+9fIEscpV3diQrJH3eQVcUsKNtt5o7bv3Q1OXER44jnYBfB3zs8Nja/H/4BRflwSSDrolH542YnwntCXL/XvNPpgzSB+Bcw03O0HbdEo0ACp/3mgAPjPGACXs4b5d6lNcO06pq8xIwFcFiMIQjfdyrH64cR18vsfqQELYOllH3MlvC9zdt5585op8Gx2wEkZv7DRiMxtt5AQX6DGmUvUAT+NJU9tdp2dKMMfHyElr5QQXi5zAP102gg0kXRu/4beZp2P9WOEczAJDtYzEa8Eawf5fBbEuDIkWPfLE+8O68M2juQAWdaWZb8qDIcejm+RauaeuvHyEshIWVeksfXila7AiefS+vWmi79zlFuFmW9iRDmeZBf0tkYF/uU6GMnrvJBYmvCaVEor56N7xIPb1575+cZzK5eWLle7HYkB5d45d4hBONisztK4bIq+AcOS1nnN9CDkEKoV3P1UbcuU/bbLDSEnl5Vx5GVsEzYhDD8NkvCFxUpoMbrBOWgYFgvX2gEl5Np7jH+uUk31SVYhptomXad4AIU1AvOD74ovV9Aqm1dRNypth6IDb6WqXcM9IzWF3Tn+yotzPG3eXCq39sfzlqKcBZdfFD0sA+EZC4zNuQ/Q82i7Y/GgVm7nVbBHhY52/gGmK3o4e7AcKCU53UoTqLXVB85fBCFW73XHT1forg7kdAnlvz+4jjPA8t/OYiul7/7v0SxoWHyuJvgLxnLqwAJsC9t1GFfW0PoYzKaSU7oCWZM2+b35dmWQ3oimkxbCnnfdqS2vEn1YnUI7dTXxdA8zjPtyx7InTBlITTHiPJeTIn8AwLLT+xnOiLyQ1UCidLtrlwKOnU/ldnqon0yNfouOU2Y6rOQGW6WIDrqpMoqedKFhzKaLXZhXBKxJqXSRfibqwcBe0J5G5IFwE537h/qfWTrQc+o6mqUDCwBOnYN7NIOoyG7rUC5fzd0bYlb0sE5JzJzMW+dqjKHtVuWiK4tS5W6zUXOGKAwM14tyWZyuwBp8us3HKA7l3fL3qvYqs4n07vE309Avf+tEf4jPDvPDTXRIM01gHKcMp8niQEJWI3QhH47f4KXQSvhbWwLfsKFtyAeYaI9j2ts0ZxnjcS7llkMuwCLBHwof4E9bv3aGI6j/09oRfRnlzkUB09J3HIW1iv8B/651V4koOIMPqnA5UnsAmyVz/LR1+lvtMcDBGAlvzrVvIxw4eLbRF3kK/vcJuduqjNZWpvTKD2De4PKlglaGHfNCTuXXqWplvX+t6kIcgAZ42ThyPUmE3w2n6Xq1C7LWej5UtF29Uirdh76nwRstmKVVQY/MqVWZLDzW+uzl1fDIo2hrGSy6X2/E+FL2Ax3iqeqDKB3bAoe3035O2BJdet90pEk/hDs4VlublxL+ppQH5DnAZ4MJ762UTjKBdeWHhXfsMaHHnMa5k+nlLbuykIcUr2TPNx3GKYAWMR/t2uUZQsvxq+ihOpvncDGjn41RiROEra0Zj+CdL46Y2AxLwdeFkZocnoI7hlDB1mpmvnZEzj4ws7lq51/5UF9zsNZmI84Z2iRxAaofrYqNQQVTodwsy+RTqJ7gvGG7n/ZXtq/vu8YetCS7UFRRcgZNFDm6ekBEQ7U1N725by+zOUolzu82xby6ThojHo4OVjZeRbcXcphqeszFW8YUDZ2+KjJ7la7uJblgshgMeGEsn4tPs+oT0SeEVvLXLX/QwrmGbyhiP9GTvy8qbsVEp15Ia89PgtBbFKigcTZA1JThCgTzc8QJe6k4tLDt65c9gGeuD8ACHc/u3YaTqXran6iW+3QRT/vVXyUI50V1WBCMDQseSrlSGX7nSZotalnejB4dAq1LOfyU5Frajqksn2w6sRzwUlOl/iPFkRoUlXElj0Ct68PvLgDTFLC9DEJikCgBPTUA2ADkrca945ASOQMKCOvO4QPX7yXqoyElVyhVitciGNxYnE4xZ+gmqwBTnqZ5UPUC28v2Q7ofKxtFnFtQfOs+35nrjyLOQ1tOHb4qJPIAMyUseo0b3ByVMC4QqOHwer7N5D5ojZYZm8Z1mqBfPEkEHuvMEOBjzvs+WLk39afUfLK1tdC/2B+HqymoA1hu7LIJlMOmBLN4vIxlrMYQZfiEFa4mPmFJyzmPoJ2UBA0GItmTcd+6DRDUO3IuVpZmIJA8RJbAheYZyKc2H6rN6qIOT2rAQDt8xfIuXHLTBftfwwg/soEHIbR940dn/CWzcOBASIETuaBE8T3f8BvvEZ5lxibZTVmNQo6WDbQpBcN1UwKldu/AgtyB2zfnPey56OhcA5C+jEm6x0eEr4aRVagg=="
+  }
+ },
+ "key": "burrow"
+}

secrets/production/kafka.json

@@ -0,0 +1,25 @@
+{
+ "data": {
+  "security_protocol": {
+   "iv": "wKEDOI0MV82LtkmgIMPiuw==",
+   "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAEULjtozvRKe90xxXb8yABfAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMtRLJ/AmBsmhbx0NhAgEQgDtvNHyBVnBEWMReMNR784R9MFCv8oDCOp7MLkZxFTXbDIDvyI6hqg9q7xALRlLumMd0rKwngtYbI64V8Q==",
+   "data": "/1K7hf2bbtyfsUyII/nT/Q=="
+  },
+  "sasl_mechanism": {
+   "iv": "PWam3Z9KEL5HN1zd0qDR2g==",
+   "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAH+AUh06+FQhCdR184bra+KAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMffZ4d39tk+x8+Y+vAgEQgDuX1QyYyR9IeoNmADna5snv+Z5zqgdLZwDULqK5gpQjb9EjVjobyvZ89YuhyOIFGXKXX4CjbpybWnRgxA==",
+   "data": "6H5SQX242lAvOI8ldbyMvw=="
+  },
+  "sasl_plain_username": {
+   "iv": "Px+YLaw3mA/AmaTgS1f6/g==",
+   "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAE9OIbxJfGy4NQkCuC3nwjEAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMaD2KAyFywMnDXN7oAgEQgDvAk3dIBtdE4hhLx+xtdnCtOlGW6PRtZWYT+cDWP1hWfaqUOPdTuF97QIyD+fxH9l+30ADj+kEGnisa2Q==",
+   "data": "8m8pdEy6w/Z+47WiSL23Hg=="
+  },
+  "sasl_plain_password": {
+   "iv": "6lQwpK1bZX1Rh1ec3URXmA==",
+   "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAHwlZ/Ia2qv8S5EH+Q5g9lrAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM7j9C8bKS8kM9mcuLAgEQgDtKmwZ54fFY64tgIISAmmY10eXMoB39spcK7ICXHBTngCYQvF0UOQ29+s6OXPv2aR6Rs4HO63T+tHr/Qw==",
+   "data": "obW1c96OC2g9QPHrCjpN1g=="
+  }
+ },
+ "key": "burrow"
+}