diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..be46abc48 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,318 @@ +#!/usr/bin/env groovy + +library identifier: 'apm@master', +changelog: false, +retriever: modernSCM( + [$class: 'GitSCMSource', + credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', + remote: 'git@github.com:elastic/apm-pipeline-library.git']) + +pipeline { + agent any + environment { + HOME = "${env.HUDSON_HOME}" + BASE_DIR="src/go.elastic.co/apm" + JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" + } + triggers { + cron('0 0 * * 1-5') + } + options { + timeout(time: 1, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '3', artifactNumToKeepStr: '2', daysToKeepStr: '30')) + timestamps() + preserveStashes() + //ansiColor('xterm') + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + } + parameters { + string(name: 'branch_specifier', defaultValue: "", description: "the Git branch specifier to build (, , , etc.)") + string(name: 'GO_VERSION', defaultValue: "1.10.3", description: "Go version to use.") + booleanParam(name: 'linux_ci', defaultValue: true, description: 'Enable Linux build') + booleanParam(name: 'test_ci', defaultValue: true, description: 'Enable test') + booleanParam(name: 'integration_test_ci', defaultValue: true, description: 'Enable run integration test') + booleanParam(name: 'integration_test_pr_ci', defaultValue: false, description: 'Enable run integration test') + booleanParam(name: 'integration_test_master_ci', defaultValue: false, description: 'Enable run integration test') + booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks') + booleanParam(name: 'doc_ci', defaultValue: true, description: 'Enable build documentation') + } + stages { + /** + Checkout the code and stash it, to use it on other stages. + */ + stage('Checkout') { + agent { label 'master || linux' } + options { skipDefaultCheckout() } + environment { + PATH = "${env.PATH}:${env.HUDSON_HOME}/go/bin/:${env.WORKSPACE}/bin" + GOPATH = "${env.WORKSPACE}" + } + steps { + withEnvWrapper() { + dir("${BASE_DIR}"){ + script{ + if(!env?.branch_specifier){ + echo "Checkout SCM" + checkout scm + } else { + echo "Checkout ${branch_specifier}" + checkout([$class: 'GitSCM', branches: [[name: "${branch_specifier}"]], + doGenerateSubmoduleConfigurations: false, + extensions: [], + submoduleCfg: [], + userRemoteConfigs: [[credentialsId: "${JOB_GIT_CREDENTIALS}", + url: "${GIT_URL}"]]]) + } + env.JOB_GIT_COMMIT = getGitCommitSha() + env.JOB_GIT_URL = "${GIT_URL}" + github_enterprise_constructor() + } + } + stash allowEmpty: true, name: 'source', useDefaultExcludes: false + } + } + } + /** + Build on a linux environment. + */ + stage('build') { + agent { label 'linux && immutable' } + options { skipDefaultCheckout() } + environment { + PATH = "${env.PATH}:${env.HUDSON_HOME}/go/bin/:${env.WORKSPACE}/bin" + GOPATH = "${env.WORKSPACE}" + } + when { + beforeAgent true + environment name: 'linux_ci', value: 'true' + } + steps { + withEnvWrapper() { + unstash 'source' + dir("${BASE_DIR}"){ + sh """#!/bin/bash + ./scripts/jenkins/build.sh + """ + } + } + } + } + /** + Run unit tests and store the results in Jenkins. + */ + stage('test') { + agent { label 'linux && immutable' } + options { skipDefaultCheckout() } + environment { + PATH = "${env.PATH}:${env.HUDSON_HOME}/go/bin/:${env.WORKSPACE}/bin" + GOPATH = "${env.WORKSPACE}" + } + when { + beforeAgent true + environment name: 'test_ci', value: 'true' + } + steps { + withEnvWrapper() { + unstash 'source' + dir("${BASE_DIR}"){ + sh """#!/bin/bash + ./scripts/jenkins/test.sh + """ + } + } + } + post { + always { + junit(allowEmptyResults: true, + keepLongStdio: true, + testResults: "${BASE_DIR}/build/junit-*.xml") + } + } + } + stage('Parallel stages') { + failFast true + parallel { + /** + Run Benchmarks and send the results to ES. + */ + stage('Benchmarks') { + agent { label 'linux && immutable' } + options { skipDefaultCheckout() } + environment { + PATH = "${env.PATH}:${env.HUDSON_HOME}/go/bin/:${env.WORKSPACE}/bin" + GOPATH = "${env.WORKSPACE}" + } + when { + beforeAgent true + allOf { + branch 'master'; + environment name: 'bench_ci', value: 'true' + } + } + steps { + withEnvWrapper() { + unstash 'source' + dir("${BASE_DIR}"){ + sh """#!/bin/bash + ./scripts/jenkins/bench.sh + """ + sendBenchmarks(file: 'build/bench.out', index: "benchmark-go") + } + } + } + post { + always { + junit(allowEmptyResults: true, + keepLongStdio: true, + testResults: "${BASE_DIR}/build/junit-*.xml") + } + } + } + /** + Run tests in a docker container and store the results in jenkins and codecov. + */ + stage('Docker tests') { + agent { label 'linux && docker && immutable' } + options { skipDefaultCheckout() } + environment { + PATH = "${env.PATH}:${env.HUDSON_HOME}/go/bin/:${env.WORKSPACE}/bin" + GOPATH = "${env.WORKSPACE}" + } + when { + beforeAgent true + environment name: 'integration_test_ci', value: 'true' + } + steps { + withEnvWrapper() { + unstash 'source' + dir("${BASE_DIR}"){ + sh """#!/bin/bash + ./scripts/jenkins/docker-test.sh + """ + } + } + } + post { + always { + coverageReport("${BASE_DIR}/build/coverage") + junit(allowEmptyResults: true, + keepLongStdio: true, + testResults: "${BASE_DIR}/build/junit-*.xml") + codecov(repo: 'apm-agent-go', basedir: "${BASE_DIR}") + } + } + } + /** + run Go integration test with the commit version on master branch. + */ + stage('Integration test master') { + agent { label 'linux && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + allOf { + branch 'master'; + environment name: 'integration_test_master_ci', value: 'true' + } + } + steps { + build( + job: 'apm-server-ci/apm-integration-test-axis-pipeline', + parameters: [ + string(name: 'BUILD_DESCRIPTION', value: "${BUILD_TAG}-INTEST"), + booleanParam(name: "go_Test", value: true), + booleanParam(name: "java_Test", value: false), + booleanParam(name: "ruby_Test", value: false), + booleanParam(name: "python_Test", value: false), + booleanParam(name: "nodejs_Test", value: false)], + wait: true, + propagate: true) + } + } + /** + run Go integration test with the commit version on a PR. + */ + stage('Integration test PR') { + agent { label 'linux && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + allOf { + changeRequest() + environment name: 'integration_test_pr_ci', value: 'true' + } + } + steps { + build( + job: 'apm-server-ci/apm-integration-test-pipeline', + parameters: [ + string(name: 'BUILD_DESCRIPTION', value: "${BUILD_TAG}-INTEST"), + string(name: 'APM_AGENT_GO_PKG', value: "${BUILD_TAG}"), + booleanParam(name: "go_Test", value: true), + booleanParam(name: "java_Test", value: false), + booleanParam(name: "ruby_Test", value: false), + booleanParam(name: "python_Test", value: false), + booleanParam(name: "nodejs_Test", value: false), + booleanParam(name: "kibana_Test", value: false), + booleanParam(name: "server_Test", value: false)], + wait: true, + propagate: true) + } + } + } + } + /** + Build the documenattions. + */ + stage('Documentation') { + agent { label 'linux && immutable' } + options { skipDefaultCheckout() } + environment { + PATH = "${env.PATH}:${env.HUDSON_HOME}/go/bin/:${env.WORKSPACE}/bin" + GOPATH = "${env.WORKSPACE}" + ELASTIC_DOCS = "${env.WORKSPACE}/elastic/docs" + } + when { + beforeAgent true + allOf { + branch 'master'; + environment name: 'doc_ci', value: 'true' + } + } + steps { + withEnvWrapper() { + unstash 'source' + dir("${ELASTIC_DOCS}"){ + git "https://github.com/elastic/docs.git" + } + dir("${BASE_DIR}"){ + sh """#!/bin/bash + make docs + """ + } + } + } + post{ + success { + tar(file: "doc-files.tgz", archive: true, dir: "html", pathPrefix: "${BASE_DIR}/docs") + } + } + } + } + post { + success { + echoColor(text: '[SUCCESS]', colorfg: 'green', colorbg: 'default') + } + aborted { + echoColor(text: '[ABORTED]', colorfg: 'magenta', colorbg: 'default') + } + failure { + echoColor(text: '[FAILURE]', colorfg: 'red', colorbg: 'default') + //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${NOTIFY_TO}", sendToIndividuals: false]) + } + unstable { + echoColor(text: '[UNSTABLE]', colorfg: 'yellow', colorbg: 'default') + } + } +} diff --git a/README.md b/README.md index bb1ae4dc2..f173f83d4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Build Status](https://apm-ci.elastic.co/buildStatus/icon?job=apm-agent-go/apm-agent-go-mbp/master)](https://apm-ci.elastic.co/job/apm-agent-go/job/apm-agent-go-mbp/job/master/) [![GoDoc](https://godoc.org/go.elastic.co/apm?status.svg)](http://godoc.org/go.elastic.co/apm) [![Travis-CI](https://travis-ci.org/elastic/apm-agent-go.svg)](https://travis-ci.org/elastic/apm-agent-go) [![AppVeyor](https://ci.appveyor.com/api/projects/status/28fhswvqqc7p90f7?svg=true)](https://ci.appveyor.com/project/AndrewWilkins/apm-agent-go) diff --git a/scripts/jenkins/bench.sh b/scripts/jenkins/bench.sh new file mode 100755 index 000000000..dbcc5b6b0 --- /dev/null +++ b/scripts/jenkins/bench.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euxo pipefail + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. +. ${srcdir}/common.bash + +jenkins_setup + +go get -v -u github.com/jstemmer/go-junit-report +go get -v -t ./... + +export OUT_FILE="build/bench.out" +mkdir -p build +go test -run=NONE -benchmem -bench=. ./... -v 2>&1 | tee ${OUT_FILE} +cat ${OUT_FILE} | go-junit-report > build/junit-apm-agent-go-bench.xml diff --git a/scripts/jenkins/build.sh b/scripts/jenkins/build.sh new file mode 100755 index 000000000..9e4e5a033 --- /dev/null +++ b/scripts/jenkins/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euxo pipefail + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. +. ${srcdir}/common.bash + +jenkins_setup + +go get -u -v golang.org/x/tools/cmd/goimports +go get -u -v golang.org/x/lint/golint + +make install precheck diff --git a/scripts/jenkins/common.bash b/scripts/jenkins/common.bash new file mode 100644 index 000000000..9f8ac1e1e --- /dev/null +++ b/scripts/jenkins/common.bash @@ -0,0 +1,113 @@ +# +# File: common.bash +# +# Common bash routines. +# + +# Script directory: +_sdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# debug "msg" +# Write a debug message to stderr. +debug() +{ + if [ "$VERBOSE" == "true" ]; then + echo "DEBUG: $1" >&2 + fi +} + +# err "msg" +# Write and error message to stderr. +err() +{ + echo "ERROR: $1" >&2 +} + +# get_go_version +# Read the project's Go version and return it in the GO_VERSION variable. +# On failure it will exit. +get_go_version() { + GO_VERSION=$(cat "${_sdir}/../.go-version") + if [ -z "$GO_VERSION" ]; then + err "Failed to detect the project's Go version" + exit 1 + fi +} + +# install_gimme +# Install gimme to HOME/bin. +install_gimme() { + # Install gimme + if [ ! -f "${HOME}/bin/gimme" ]; then + mkdir -p ${HOME}/bin + curl -sL -o ${HOME}/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/v1.1.0/gimme + chmod +x ${HOME}/bin/gimme + fi + + GIMME="${HOME}/bin/gimme" + debug "Gimme version $(${GIMME} version)" +} + +# setup_go_root "version" +# This configures the Go version being used. It sets GOROOT and adds +# GOROOT/bin to the PATH. It uses gimme to download the Go version if +# it does not already exist in the ~/.gimme dir. +setup_go_root() { + local version=${1} + + install_gimme + + # Setup GOROOT and add go to the PATH. + ${GIMME} "${version}" > /dev/null + source "${HOME}/.gimme/envs/go${version}.env" 2> /dev/null + + debug "$(go version)" +} + +# setup_go_path "gopath" +# This sets GOPATH and adds GOPATH/bin to the PATH. +setup_go_path() { + local gopath="${1}" + if [ -z "$gopath" ]; then return; fi + + # Setup GOPATH. + export GOPATH="${gopath}" + + # Add GOPATH to PATH. + export PATH="${GOPATH}/bin:${PATH}" + + debug "GOPATH=${GOPATH}" +} + +jenkins_setup() { + : "${HOME:?Need to set HOME to a non-empty value.}" + : "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" + + if [ -z ${GO_VERSION:-} ]; then + get_go_version + fi + + # Setup Go. + export GOPATH=${WORKSPACE} + export PATH=${GOPATH}/bin:${PATH} + eval "$(gvm ${GO_VERSION})" + + # Workaround for Python virtualenv path being too long. + export TEMP_PYTHON_ENV=$(mktemp -d) + export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" + + # Write cached magefile binaries to workspace to ensure + # each run starts from a clean slate. + export MAGEFILE_CACHE="${WORKSPACE}/.magefile" +} + +docker_setup() { + OS="$(uname)" + case $OS in + 'Darwin') + # Start the docker machine VM (ignore error if it's already running). + docker-machine start default || true + eval $(docker-machine env default) + ;; + esac +} diff --git a/scripts/jenkins/docker-test.sh b/scripts/jenkins/docker-test.sh new file mode 100755 index 000000000..4ac2016e6 --- /dev/null +++ b/scripts/jenkins/docker-test.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euxo pipefail + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. +. ${srcdir}/common.bash + +jenkins_setup + +go get -v -u github.com/jstemmer/go-junit-report +go get -v -u github.com/t-yuki/gocover-cobertura +go get -v -t ./... + +export COV_FILE="build/coverage/coverage.cov" +export OUT_FILE="build/test-report.out" +mkdir -p build/coverage + +./scripts/docker-compose-testing up -d --build +./scripts/docker-compose-testing run -T --rm go-agent-tests make coverage GOFLAGS=-v 2> >(tee ${OUT_FILE} 1>&2) > ${COV_FILE}.raw + +echo "mode: atomic" > ${COV_FILE} +grep -v "mode\: atomic" ${COV_FILE}.raw >> ${COV_FILE} + +go tool cover -html="${COV_FILE}" -o build/coverage/coverage-apm-agent-go-docker-report.html +gocover-cobertura < "${COV_FILE}" > build/coverage/coverage-apm-agent-go-docker-report.xml + +cat ${OUT_FILE} | go-junit-report > build/junit-apm-agent-go-docker.xml + + diff --git a/scripts/jenkins/test.sh b/scripts/jenkins/test.sh new file mode 100755 index 000000000..88d1f8248 --- /dev/null +++ b/scripts/jenkins/test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euxo pipefail + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. +. ${srcdir}/common.bash + +jenkins_setup + +go get -v -u github.com/jstemmer/go-junit-report +go get -v -t ./... + +mkdir -p build + +(go test -race ./... -v 2>&1 | go-junit-report > build/junit-apm-agent-go.xml) || echo -e "\033[31;49mTests FAILED\033[0m"