Create presubmit and end-to-end test infrastructure

test/OWNERS

@@ -0,0 +1,8 @@
+# The OWNERS file is used by prow to automatically merge approved PRs.
+
+approvers:
+- adrcunha
+- bobcatfish
+- jessiezcc
+- srinivashegde86
+- steuhs

test/README.md

@@ -0,0 +1,28 @@
+# Test
+
+This directory contains tests and testing docs.
+
+* [Unit tests](#running-unit-tests) currently reside in the codebase alongside the code they test
+* [End-to-end tests](#running-end-to-end-tests)
+
+
+## Running unit tests
+
+TODO(#22): Write real unit tests.
+
+## Running end-to-end tests
+
+### Dependencies
+
+You might need to install `kubetest` in order to run the end-to-end tests locally:
+
+```shell
+go get -u k8s.io/test-infra/kubetest
+```
+
+Simply run the `e2e-tests.sh` script, setting `$PROJECT_ID` first to your GCP project. The script
+will create a GKE cluster, install Knative, run the end-to-end tests and delete the cluster.
+
+If you already have a cluster set, ensure that `$PROJECT_ID` is empty and call the script with the
+`--run-tests` argument. Note that this requires you to have Knative Build installed and configured
+to your particular configuration setup.

test/build-buildpack.yaml

@@ -0,0 +1,10 @@
+apiVersion: build.dev/v1alpha1
+kind: Build
+metadata:
+  name: buildpack-build
+spec:
+  source:
+    git:
+      url: https://github.com/my-user/my-repo
+      branch: master
+  template:

test/e2e-tests.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Copyright 2018 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs the end-to-end tests for build templates.
+
+# If you already have a Knative Build cluster setup and kubectl pointing
+# to it, call this script with the --run-tests arguments and it will use
+# the cluster and run the tests.
+
+# Calling this script without arguments will create a new cluster in
+# project $PROJECT_ID, run the tests and delete the cluster.
+
+source "$(dirname $(readlink -f ${BASH_SOURCE}))/library.sh"
+
+# Test cluster parameters and location of test files
+readonly E2E_CLUSTER_NAME=bldtpl-e2e-cluster${BUILD_NUMBER}
+readonly E2E_NETWORK_NAME=bldtpl-e2e-net${BUILD_NUMBER}
+readonly E2E_CLUSTER_ZONE=us-central1-a
+readonly E2E_CLUSTER_NODES=2
+readonly E2E_CLUSTER_MACHINE=n1-standard-2
+readonly TEST_RESULT_FILE=/tmp/bldtpl-e2e-result
+readonly ISTIO_YAML=https://storage.googleapis.com/knative-releases/latest/istio.yaml
+readonly SERVING_RELEASE=https://storage.googleapis.com/knative-releases/latest/release.yaml
+
+# This script.
+readonly SCRIPT_CANONICAL_PATH="$(readlink -f ${BASH_SOURCE})"
+
+# Helper functions.
+
+function run_buildpack_test() {
+  subheader "Running buildpack test"
+  echo "Installing template:"
+  kubectl apply -f buildpack/buildpack.yaml || return 1
+  echo "Checking that template is installed:"
+  kubectl get buildtemplates || return 1
+  echo "Creating build:"
+  kubectl apply -f test/build-buildpack.yaml || return 1
+  # Wait 5s for processing to start
+  sleep 5
+  echo "Checking that build was started:"
+  kubectl get build buildpack-build -oyaml
+  # TODO(adrcunha): Add proper verification.
+}
+
+function exit_if_test_failed() {
+  [[ $? -eq 0 ]] && return 0
+  [[ -n $1 ]] && echo "ERROR: $1"
+  echo "***************************************"
+  echo "***           TEST FAILED           ***"
+  echo "***    Start of information dump    ***"
+  echo "***************************************"
+  if (( IS_PROW )) || [[ $PROJECT_ID != "" ]]; then
+    echo ">>> Project info:"
+    gcloud compute project-info describe
+  fi
+  echo ">>> All resources:"
+  kubectl get all --all-namespaces
+  echo "***************************************"
+  echo "***           TEST FAILED           ***"
+  echo "***     End of information dump     ***"
+  echo "***************************************"
+  exit 1
+}
+
+# Script entry point.
+
+cd ${BUILDTEMPLATES_ROOT_DIR}
+
+# Show help if bad arguments are passed.
+if [[ -n $1 && $1 != "--run-tests" ]]; then
+  echo "usage: $0 [--run-tests]"
+  exit 1
+fi
+
+# No argument provided, create the test cluster.
+
+if [[ -z $1 ]]; then
+  header "Creating test cluster"
+  # Smallest cluster required to run the end-to-end-tests
+  CLUSTER_CREATION_ARGS=(
+    --gke-create-args="--enable-autoscaling --min-nodes=1 --max-nodes=${E2E_CLUSTER_NODES} --scopes=cloud-platform"
+    --gke-shape={\"default\":{\"Nodes\":${E2E_CLUSTER_NODES}\,\"MachineType\":\"${E2E_CLUSTER_MACHINE}\"}}
+    --provider=gke
+    --deployment=gke
+    --gcp-node-image=cos
+    --cluster="${E2E_CLUSTER_NAME}"
+    --gcp-zone="${E2E_CLUSTER_ZONE}"
+    --gcp-network="${E2E_NETWORK_NAME}"
+    --gke-environment=prod
+  )
+  if (( ! IS_PROW )); then
+    CLUSTER_CREATION_ARGS+=(--gcp-project=${PROJECT_ID:?"PROJECT_ID must be set to the GCP project where the tests are run."})
+  fi
+  # SSH keys are not used, but kubetest checks for their existence.
+  # Touch them so if they don't exist, empty files are create to satisfy the check.
+  touch $HOME/.ssh/google_compute_engine.pub
+  touch $HOME/.ssh/google_compute_engine
+  # Assume test failed (see more details at the end of this script).
+  echo -n "1"> ${TEST_RESULT_FILE}
+  kubetest "${CLUSTER_CREATION_ARGS[@]}" \
+    --up \
+    --down \
+    --extract "gke-${GKE_VERSION}" \
+    --test-cmd "${SCRIPT_CANONICAL_PATH}" \
+    --test-cmd-args --run-tests
+  result="$(cat ${TEST_RESULT_FILE})"
+  echo "Test result code is $result"
+  exit $result
+fi
+
+# --run-tests passed as first argument, run the tests.
+
+# Install Knative Build if not using an existing cluster
+if (( IS_PROW )) || [[ -n ${PROJECT_ID} ]]; then
+  header "Starting Knative Serving"
+  acquire_cluster_admin_role $(gcloud config get-value core/account) ${E2E_CLUSTER_NAME} ${E2E_CLUSTER_ZONE}
+  subheader "Installing Istio"
+  kubectl apply -f ${ISTIO_YAML}
+  wait_until_pods_running istio-system
+  exit_if_test_failed "could not install Istio"
+  kubectl label namespace default istio-injection=enabled
+
+  subheader "Installing Knative Serving"
+  kubectl apply -f ${SERVING_RELEASE}
+  exit_if_test_failed "could not install Knative Serving"
+
+  wait_until_pods_running build-system
+fi
+
+header "Running tests"
+run_buildpack_test
+exit_if_test_failed
+# TODO(adrcunha): Add more tests.
+
+# kubetest teardown might fail and thus incorrectly report failure of the
+# script, even if the tests pass.
+# We store the real test result to return it later, ignoring any teardown
+# failure in kubetest.
+# TODO(adrcunha): Get rid of this workaround.
+echo -n "0"> ${TEST_RESULT_FILE}
+echo "**************************************"
+echo "***        ALL TESTS PASSED        ***"
+echo "**************************************"
+exit 0

test/library.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# Copyright 2018 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a collection of useful bash functions and constants, intended
+# to be used in test scripts and the like. It doesn't do anything when
+# called from command line.
+
+# Default GKE version to be used
+readonly GKE_VERSION=latest
+
+# Useful environment variables
+[[ -n "${PROW_JOB_ID}" ]] && IS_PROW=1 || IS_PROW=0
+readonly IS_PROW
+readonly BUILDTEMPLATES_ROOT_DIR="$(dirname $(readlink -f ${BASH_SOURCE}))/.."
+
+# Simple header for logging purposes.
+function header() {
+  echo "================================================="
+  echo ${1^^}
+  echo "================================================="
+}
+
+# Simple subheader for logging purposes.
+function subheader() {
+  echo "-------------------------------------------------"
+  echo $1
+  echo "-------------------------------------------------"
+}
+
+# Waits until all pods are running in the given namespace or Completed.
+# Parameters: $1 - namespace.
+function wait_until_pods_running() {
+  echo -n "Waiting until all pods in namespace $1 are up"
+  for i in {1..150}; do  # timeout after 5 minutes
+    local pods="$(kubectl get pods -n $1 2>/dev/null | grep -v NAME)"
+    local not_running=$(echo "${pods}" | grep -v Running | grep -v Completed | wc -l)
+    if [[ -n "${pods}" && ${not_running} == 0 ]]; then
+      echo -e "\nAll pods are up:"
+      kubectl get pods -n $1
+      return 0
+    fi
+    echo -n "."
+    sleep 2
+  done
+  echo -e "\n\nERROR: timeout waiting for pods to come up"
+  kubectl get pods -n $1
+  return 1
+}
+
+# Sets the given user as cluster admin.
+# Parameters: $1 - user
+#             $2 - cluster name
+#             $3 - cluster zone
+function acquire_cluster_admin_role() {
+  # Get the password of the admin and use it, as the service account (or the user)
+  # might not have the necessary permission.
+  local password=$(gcloud --format="value(masterAuth.password)" \
+      container clusters describe $2 --zone=$3)
+  kubectl --username=admin --password=$password \
+      create clusterrolebinding cluster-admin-binding \
+      --clusterrole=cluster-admin \
+      --user=$1
+}

test/presubmit-tests.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright 2018 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs the presubmit tests, in the right order.
+# It is started by prow for each PR.
+# For convenience, it can also be executed manually.
+
+set -o errexit
+set -o pipefail
+
+# Extensions or file patterns that don't require presubmit tests
+readonly NO_PRESUBMIT_FILES=(\.md \.png ^OWNERS)
+
+source "$(dirname $(readlink -f ${BASH_SOURCE}))/library.sh"
+
+# Helper functions.
+
+function build_tests() {
+  header "TODO(#23): write build tests"
+}
+
+function unit_tests() {
+  header "TODO(#22): Write unit tests"
+}
+
+function integration_tests() {
+  ./test/e2e-tests.sh
+}
+
+# Script entry point.
+
+# Parse script argument:
+# --all-tests or no arguments: run all tests
+# --build-tests: run only the build tests
+# --unit-tests: run only the unit tests
+# --integration-tests: run only the integration tests
+RUN_BUILD_TESTS=0
+RUN_UNIT_TESTS=0
+RUN_INTEGRATION_TESTS=0
+[[ -z "$1" || "$1" == "--all-tests" ]] && RUN_BUILD_TESTS=1 && RUN_UNIT_TESTS=1 && RUN_INTEGRATION_TESTS=1
+[[ "$1" == "--build-tests" ]] && RUN_BUILD_TESTS=1
+[[ "$1" == "--unit-tests" ]] && RUN_UNIT_TESTS=1
+[[ "$1" == "--integration-tests" ]] && RUN_INTEGRATION_TESTS=1
+readonly RUN_BUILD_TESTS
+readonly RUN_UNIT_TESTS
+readonly RUN_INTEGRATION_TESTS
+
+if ! (( RUN_BUILD_TESTS+RUN_UNIT_TESTS+RUN_INTEGRATION_TESTS )); then
+  echo "error: unknown argument $1";
+  exit 1
+fi
+
+cd ${BUILDTEMPLATES_ROOT_DIR}
+
+# Skip presubmit tests if whitelisted files were changed.
+if [[ -n "${PULL_PULL_SHA}" ]]; then
+  # On a presubmit job
+  changes="$(git diff --name-only ${PULL_PULL_SHA} ${PULL_BASE_SHA})"
+  no_presubmit_pattern="${NO_PRESUBMIT_FILES[*]}"
+  no_presubmit_pattern="\(${no_presubmit_pattern// /\\|}\)$"
+  echo -e "Changed files in commit ${PULL_PULL_SHA}:\n${changes}"
+  if [[ -z "$(echo "${changes}" | grep -v ${no_presubmit_pattern})" ]]; then
+    # Nothing changed other than files that don't require presubmit tests
+    header "Commit only contains changes that don't affect tests, skipping"
+    exit 0
+  fi
+fi
+
+# Tests to be performed, in the right order if --all-tests is passed.
+
+if (( RUN_BUILD_TESTS )); then build_tests; fi
+if (( RUN_UNIT_TESTS )); then unit_tests; fi
+if (( RUN_INTEGRATION_TESTS )); then integration_tests; fi