diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 00000000..84227819 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,19 @@ +# This file was generated by gygdev. Do not manually modify it +pipeline: + build-image: + image: hub.gyg.io/drone-plugins/gyg-registry + privileged: true + context: . + dockerfile: ./Dockerfile + repo: hub.gyg.io/service/burrow + tags: + - ${DRONE_BUILD_NUMBER}.${DRONE_COMMIT_SHA:0:7} + when: + event: push + register-dss: + image: hub.gyg.io/drone-plugins/dss + privileged: true + tags: + - ${DRONE_BUILD_NUMBER}.${DRONE_COMMIT_SHA:0:7} + when: + event: push diff --git a/.gitignore b/.gitignore index f0208c50..d054eddb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,4 @@ -burrow-src -.*.swp -!config -dist -log -.idea -*.cov -Burrow -Burrow.exe -Burrow.iml -tmp -vendor +/.gygdev-dc.yml +/.gygdev_generated +/.config +/.secrets diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 00000000..aed77378 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,10 @@ +# Burrow + +## Description + +This repo is a fork from the linkedin project. It analyses and exposes metrics +over http. Then a custom datadog check is consuming it an produces `burrow.*` +metrics. + +As it's just a fork turned into a gygservice, if you want to know more about how +it works, please read the readme or [developers documentation](https://github.com/linkedin/Burrow/wiki). diff --git a/Dockerfile b/Dockerfile index 4a309024..c32265ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,14 @@ FROM golang:1.11-alpine as builder ENV DEP_VERSION="0.5.0" RUN apk add --no-cache git curl gcc libc-dev && \ - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep && \ - chmod +x $GOPATH/bin/dep && \ - mkdir -p $GOPATH/src/github.com/linkedin/Burrow + curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep && \ + chmod +x $GOPATH/bin/dep && \ + mkdir -p $GOPATH/src/github.com/linkedin/Burrow ADD . $GOPATH/src/github.com/linkedin/Burrow/ RUN cd $GOPATH/src/github.com/linkedin/Burrow && \ - dep ensure && \ - go build -o /tmp/burrow . + dep ensure && \ + go build -o /tmp/burrow . FROM iron/go LABEL maintainer="LinkedIn Burrow https://github.com/linkedin/Burrow" @@ -18,4 +18,6 @@ WORKDIR /app COPY --from=builder /tmp/burrow /app/ ADD /docker-config/burrow.toml /etc/burrow/ -CMD ["/app/burrow", "--config-dir", "/etc/burrow"] \ No newline at end of file +EXPOSE 8000 + +CMD ["/app/burrow", "--config-dir", "/etc/burrow"] diff --git a/config/burrow.toml b/config/burrow.toml index aae38dfa..02f7413b 100644 --- a/config/burrow.toml +++ b/config/burrow.toml @@ -19,7 +19,7 @@ root-path="/burrow" [client-profile.test] client-id="burrow-test" -kafka-version="0.10.0" +kafka-version="2.1.0" [cluster.local] class-name="kafka" diff --git a/core/internal/helpers/sarama.go b/core/internal/helpers/sarama.go index f8d6f64f..a52c3079 100644 --- a/core/internal/helpers/sarama.go +++ b/core/internal/helpers/sarama.go @@ -14,6 +14,8 @@ import ( "crypto/tls" "crypto/x509" "io/ioutil" + "log" + "os" "github.com/Shopify/sarama" "github.com/spf13/viper" @@ -78,6 +80,13 @@ func GetSaramaConfigFromClientProfile(profileName string) *sarama.Config { saramaConfig.Version = parseKafkaVersion(viper.GetString(configRoot + ".kafka-version")) saramaConfig.Consumer.Return.Errors = true + if (viper.IsSet("logging")) { + viper.SetDefault("logging.level", "info") + if (viper.GetString("logging.level") == "debug") { + sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags) + } + } + // Configure TLS if enabled if viper.IsSet(configRoot + ".tls") { tlsName := viper.GetString(configRoot + ".tls") diff --git a/core/internal/httpserver/config.go b/core/internal/httpserver/config.go index 8191af90..470f9f42 100644 --- a/core/internal/httpserver/config.go +++ b/core/internal/httpserver/config.go @@ -186,6 +186,7 @@ func (hc *Coordinator) configEvaluatorDetail(w http.ResponseWriter, r *http.Requ Module: httpResponseConfigModuleEvaluator{ ClassName: viper.GetString(configRoot + ".class-name"), ExpireCache: viper.GetInt64(configRoot + ".expire-cache"), + MinimumComplete: viper.GetInt64(configRoot + ".minimum-complete"), }, Request: requestInfo, }) diff --git a/core/internal/httpserver/structs.go b/core/internal/httpserver/structs.go index 5ba6a7be..02675ddf 100644 --- a/core/internal/httpserver/structs.go +++ b/core/internal/httpserver/structs.go @@ -184,6 +184,7 @@ type httpResponseConfigModuleConsumer struct { type httpResponseConfigModuleEvaluator struct { ClassName string `json:"class-name"` ExpireCache int64 `json:"expire-cache"` + MinimumComplete int64 `json:"minimum-complete"` } type httpResponseConfigModuleNotifierHTTP struct { diff --git a/gyg-how-to-update.txt b/gyg-how-to-update.txt new file mode 100644 index 00000000..d1f79ae8 --- /dev/null +++ b/gyg-how-to-update.txt @@ -0,0 +1,3 @@ +git remote add upstream https://github.com/linkedin/Burrow.git +git fetch upstream +git rebase v1.2.2 # replace by the release, dont take master diff --git a/gygservice.yml b/gygservice.yml new file mode 100644 index 00000000..faa00931 --- /dev/null +++ b/gygservice.yml @@ -0,0 +1,36 @@ +version: '1' +service: + name: burrow + owner: di + image_name: service/burrow + level: office hours + containers: + web: + resources: + requests: + memory: 256Mi + cpu: 1500m + limits: + memory: 256Mi + cpu: 1500m + dockerfile: Dockerfile + needs: + - kafka + secret_files: + burrow: /etc/burrow/ + ports: + - name: http + container: 8000 + livenessProbe: + httpGet: + path: burrow/admin + port: 8000 + periodSeconds: 60 + initialDelaySeconds: 60 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 15 + annotations: + ad.datadoghq.com/web.check_names: ["burrow"] + ad.datadoghq.com/web.init_configs: [{}] + ad.datadoghq.com/web.instances: [{"burrow_uri": "http://%%host%%:8000", "clusters": ["kafka1"]}] diff --git a/secrets/production/burrow.json b/secrets/production/burrow.json new file mode 100644 index 00000000..9d3dc0df --- /dev/null +++ b/secrets/production/burrow.json @@ -0,0 +1,10 @@ +{ + "data": { + "burrow.toml": { + "iv": "zoLTLFEmReExIuxbSLW6RA==", + "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAEoC+eSBIaBtxd8g1Dbri8AAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMHv95qqtigawXIGfCAgEQgDv++32NNEMD2I8ToDPouTw14NL0/y+PVIGjrPashxiijiA4y59kPqe4faR7k78ZkLneqS8Xy3YQIxgocg==", + "data": "wCeuH8fzE1GdGoQ34JycuysPwTroq2WsIq6NThKawxKsaWSxFO0v7tXwh9fN+UwdqvusMWb9sDGp9z8wxix+yVCL3kSHR7Y520iPh4nBCZYZE6gVADDnRSdwaUIW5vA+v4n3tLIqQcGVW58rxDT87iOkgy9df6Pv0pjmAN19g7SA+wxWCwaq4ZsvaoI6h5ov3vAL2DimOk+9fIEscpV3diQrJH3eQVcUsKNtt5o7bv3Q1OXER44jnYBfB3zs8Nja/H/4BRflwSSDrolH542YnwntCXL/XvNPpgzSB+Bcw03O0HbdEo0ACp/3mgAPjPGACXs4b5d6lNcO06pq8xIwFcFiMIQjfdyrH64cR18vsfqQELYOllH3MlvC9zdt5585op8Gx2wEkZv7DRiMxtt5AQX6DGmUvUAT+NJU9tdp2dKMMfHyElr5QQXi5zAP102gg0kXRu/4beZp2P9WOEczAJDtYzEa8Eawf5fBbEuDIkWPfLE+8O68M2juQAWdaWZb8qDIcejm+RauaeuvHyEshIWVeksfXila7AiefS+vWmi79zlFuFmW9iRDmeZBf0tkYF/uU6GMnrvJBYmvCaVEor56N7xIPb1575+cZzK5eWLle7HYkB5d45d4hBONisztK4bIq+AcOS1nnN9CDkEKoV3P1UbcuU/bbLDSEnl5Vx5GVsEzYhDD8NkvCFxUpoMbrBOWgYFgvX2gEl5Np7jH+uUk31SVYhptomXad4AIU1AvOD74ovV9Aqm1dRNypth6IDb6WqXcM9IzWF3Tn+yotzPG3eXCq39sfzlqKcBZdfFD0sA+EZC4zNuQ/Q82i7Y/GgVm7nVbBHhY52/gGmK3o4e7AcKCU53UoTqLXVB85fBCFW73XHT1forg7kdAnlvz+4jjPA8t/OYiul7/7v0SxoWHyuJvgLxnLqwAJsC9t1GFfW0PoYzKaSU7oCWZM2+b35dmWQ3oimkxbCnnfdqS2vEn1YnUI7dTXxdA8zjPtyx7InTBlITTHiPJeTIn8AwLLT+xnOiLyQ1UCidLtrlwKOnU/ldnqon0yNfouOU2Y6rOQGW6WIDrqpMoqedKFhzKaLXZhXBKxJqXSRfibqwcBe0J5G5IFwE537h/qfWTrQc+o6mqUDCwBOnYN7NIOoyG7rUC5fzd0bYlb0sE5JzJzMW+dqjKHtVuWiK4tS5W6zUXOGKAwM14tyWZyuwBp8us3HKA7l3fL3qvYqs4n07vE309Avf+tEf4jPDvPDTXRIM01gHKcMp8niQEJWI3QhH47f4KXQSvhbWwLfsKFtyAeYaI9j2ts0ZxnjcS7llkMuwCLBHwof4E9bv3aGI6j/09oRfRnlzkUB09J3HIW1iv8B/651V4koOIMPqnA5UnsAmyVz/LR1+lvtMcDBGAlvzrVvIxw4eLbRF3kK/vcJuduqjNZWpvTKD2De4PKlglaGHfNCTuXXqWplvX+t6kIcgAZ42ThyPUmE3w2n6Xq1C7LWej5UtF29Uirdh76nwRstmKVVQY/MqVWZLDzW+uzl1fDIo2hrGSy6X2/E+FL2Ax3iqeqDKB3bAoe3035O2BJdet90pEk/hDs4VlublxL+ppQH5DnAZ4MJ762UTjKBdeWHhXfsMaHHnMa5k+nlLbuykIcUr2TPNx3GKYAWMR/t2uUZQsvxq+ihOpvncDGjn41RiROEra0Zj+CdL46Y2AxLwdeFkZocnoI7hlDB1mpmvnZEzj4ws7lq51/5UF9zsNZmI84Z2iRxAaofrYqNQQVTodwsy+RTqJ7gvGG7n/ZXtq/vu8YetCS7UFRRcgZNFDm6ekBEQ7U1N725by+zOUolzu82xby6ThojHo4OVjZeRbcXcphqeszFW8YUDZ2+KjJ7la7uJblgshgMeGEsn4tPs+oT0SeEVvLXLX/QwrmGbyhiP9GTvy8qbsVEp15Ia89PgtBbFKigcTZA1JThCgTzc8QJe6k4tLDt65c9gGeuD8ACHc/u3YaTqXran6iW+3QRT/vVXyUI50V1WBCMDQseSrlSGX7nSZotalnejB4dAq1LOfyU5Frajqksn2w6sRzwUlOl/iPFkRoUlXElj0Ct68PvLgDTFLC9DEJikCgBPTUA2ADkrca945ASOQMKCOvO4QPX7yXqoyElVyhVitciGNxYnE4xZ+gmqwBTnqZ5UPUC28v2Q7ofKxtFnFtQfOs+35nrjyLOQ1tOHb4qJPIAMyUseo0b3ByVMC4QqOHwer7N5D5ojZYZm8Z1mqBfPEkEHuvMEOBjzvs+WLk39afUfLK1tdC/2B+HqymoA1hu7LIJlMOmBLN4vIxlrMYQZfiEFa4mPmFJyzmPoJ2UBA0GItmTcd+6DRDUO3IuVpZmIJA8RJbAheYZyKc2H6rN6qIOT2rAQDt8xfIuXHLTBftfwwg/soEHIbR940dn/CWzcOBASIETuaBE8T3f8BvvEZ5lxibZTVmNQo6WDbQpBcN1UwKldu/AgtyB2zfnPey56OhcA5C+jEm6x0eEr4aRVagg==" + } + }, + "key": "burrow" +} \ No newline at end of file diff --git a/secrets/production/kafka.json b/secrets/production/kafka.json new file mode 100644 index 00000000..1a7861fa --- /dev/null +++ b/secrets/production/kafka.json @@ -0,0 +1,25 @@ +{ + "data": { + "security_protocol": { + "iv": "wKEDOI0MV82LtkmgIMPiuw==", + "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAEULjtozvRKe90xxXb8yABfAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMtRLJ/AmBsmhbx0NhAgEQgDtvNHyBVnBEWMReMNR784R9MFCv8oDCOp7MLkZxFTXbDIDvyI6hqg9q7xALRlLumMd0rKwngtYbI64V8Q==", + "data": "/1K7hf2bbtyfsUyII/nT/Q==" + }, + "sasl_mechanism": { + "iv": "PWam3Z9KEL5HN1zd0qDR2g==", + "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAH+AUh06+FQhCdR184bra+KAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMffZ4d39tk+x8+Y+vAgEQgDuX1QyYyR9IeoNmADna5snv+Z5zqgdLZwDULqK5gpQjb9EjVjobyvZ89YuhyOIFGXKXX4CjbpybWnRgxA==", + "data": "6H5SQX242lAvOI8ldbyMvw==" + }, + "sasl_plain_username": { + "iv": "Px+YLaw3mA/AmaTgS1f6/g==", + "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAE9OIbxJfGy4NQkCuC3nwjEAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMaD2KAyFywMnDXN7oAgEQgDvAk3dIBtdE4hhLx+xtdnCtOlGW6PRtZWYT+cDWP1hWfaqUOPdTuF97QIyD+fxH9l+30ADj+kEGnisa2Q==", + "data": "8m8pdEy6w/Z+47WiSL23Hg==" + }, + "sasl_plain_password": { + "iv": "6lQwpK1bZX1Rh1ec3URXmA==", + "edk": "AQIDAHiwVn67IJeh/zLC2ECLGkH+6xwCXfpNOUv/P9B/e8pXtAHwlZ/Ia2qv8S5EH+Q5g9lrAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM7j9C8bKS8kM9mcuLAgEQgDtKmwZ54fFY64tgIISAmmY10eXMoB39spcK7ICXHBTngCYQvF0UOQ29+s6OXPv2aR6Rs4HO63T+tHr/Qw==", + "data": "obW1c96OC2g9QPHrCjpN1g==" + } + }, + "key": "burrow" +} \ No newline at end of file