[Federation][init-11] Switch federation e2e tests to use the new fede…

…ration control plane bootstrap via the `kubefed init` command.

build/push-federation-images.sh

@@ -22,8 +22,5 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 
-source "${KUBE_ROOT}/build/util.sh"
-
-source "${KUBE_ROOT}/federation/cluster/common.sh"
-
-FEDERATION_IMAGE_TAG="$(kube::release::semantic_image_tag_version)" push-federation-images
+make -C "${KUBE_ROOT}/federation/" build_image
+make -C "${KUBE_ROOT}/federation/" push

build/release.sh

@@ -42,13 +42,5 @@ fi
 
 kube::build::copy_output
 
-if [[ "${FEDERATION:-}" == "true" ]];then
-    (
-	source "${KUBE_ROOT}/build/util.sh"
-	# Write federated docker image tag to workspace
-	kube::release::semantic_image_tag_version > "${KUBE_ROOT}/federation/manifests/federated-image.tag"
-    )
-fi
-
 kube::release::package_tarballs
 kube::release::package_hyperkube

cluster/kube-util.sh

@@ -35,11 +35,21 @@ fi
 
 # Federation utils
 
+# Sets the kubeconfig context value for the current cluster.
+#
+# Vars set:
+#   CLUSTER_CONTEXT
+function kubeconfig-federation-context() {
+  CLUSTER_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone"
+}
+
+
 # Should NOT be called within the global scope, unless setting the desired global zone vars
 # This function is currently NOT USED in the global scope
 function set-federation-zone-vars {
   zone="$1"
-  export OVERRIDE_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone"
+  kubeconfig-federation-context "${zone}"
+  export OVERRIDE_CONTEXT="${CLUSTER_CONTEXT}"
   echo "Setting zone vars to: $OVERRIDE_CONTEXT"
   if [[ "$KUBERNETES_PROVIDER" == "gce"  ]];then
 

federation/cluster/common.sh

@@ -46,7 +46,7 @@ if [[ -z "${FEDERATION_PUSH_REPO_BASE}" ]]; then
 fi
 
 FEDERATION_IMAGE_REPO_BASE=${FEDERATION_IMAGE_REPO_BASE:-'gcr.io/google_containers'}
-FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation}
+FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation-system}
 
 KUBE_PLATFORM=${KUBE_PLATFORM:-linux}
 KUBE_ARCH=${KUBE_ARCH:-amd64}

federation/cluster/federation-up.sh

@@ -18,15 +18,66 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../)
+# This script is only used for e2e tests! Don't use it in production!
+# This is also a temporary bridge to slowly switch over everything to
+# federation/develop.sh. Carefully moving things step-by-step, ensuring
+# things don't break.
+# TODO(madhusudancs): Remove this script and its dependencies.
 
-. ${KUBE_ROOT}/federation/cluster/common.sh
 
-tagfile="${KUBE_ROOT}/federation/manifests/federated-image.tag"
-if [[ ! -f "$tagfile" ]]; then
-    echo "FATAL: tagfile ${tagfile} does not exist. Make sure that you have run build/push-federation-images.sh"
-    exit 1
-fi
-export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")"
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+# For `kube::log::status` function since it already sources
+# "${KUBE_ROOT}/cluster/lib/logging.sh" and DEFAULT_KUBECONFIG
+source "${KUBE_ROOT}/cluster/common.sh"
+# For $KUBE_PLATFORM, $KUBE_ARCH, $KUBE_BUILD_STAGE,
+# $FEDERATION_PUSH_REPO_BASE and $FEDERATION_NAMESPACE.
+source "${KUBE_ROOT}/federation/cluster/common.sh"
+# For `get_version` function and $KUBE_REGISTRY.
+# TODO(madhusudancs): Remove this when the code here is moved
+# to federation/develop.sh
+source "${KUBE_ROOT}/federation/develop/develop.sh"
 
-create-federation-api-objects
+FEDERATION_NAME="${FEDERATION_NAME:-e2e-federation}"
+DNS_ZONE_NAME="${FEDERATION_DNS_ZONE_NAME:-}"
+HOST_CLUSTER_CONTEXT="${FEDERATION_HOST_CLUSTER_CONTEXT:-${1}}"
+
+readonly CLIENT_BIN_DIR="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/client/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/client/bin"
+kubefed="${CLIENT_BIN_DIR}/kubefed"
+kubectl="${CLIENT_BIN_DIR}/kubectl"
+
+# Initializes the control plane.
+# TODO(madhusudancs): Move this to federation/develop.sh.
+function init() {
+  kube::log::status "Deploying federation control plane for ${FEDERATION_NAME} in cluster ${HOST_CLUSTER_CONTEXT}"
+
+  local -r kube_version="$(get_version)"
+
+  ${kubefed} init \
+      "${FEDERATION_NAME}" \
+      --host-cluster-context="${HOST_CLUSTER_CONTEXT}" \
+      --dns-zone-name="${DNS_ZONE_NAME}" \
+      --image="${KUBE_REGISTRY}/hyperkube-amd64:${kube_version}"
+}
+
+# create_cluster_secrets creates the secrets containing the kubeconfigs
+# of the participating clusters in the host cluster. The kubeconfigs itself
+# are created while deploying clusters, i.e. when kube-up is run.
+function create_cluster_secrets() {
+  local -r kubeconfig_dir="$(dirname ${DEFAULT_KUBECONFIG})"
+  local -r base_dir="${kubeconfig_dir}/federation/kubernetes-apiserver"
+
+  # Create secrets with all the kubernetes-apiserver's kubeconfigs.
+  for dir in $(ls "${base_dir}"); do
+    # We create a secret with the same name as the directory name (which is
+    # same as cluster name in kubeconfig).
+    # Massage the name so that it is valid (should not contain "_" and max 253
+    # chars)
+    name=$(echo "${dir}" | sed -e "s/_/-/g")  # Replace "_" by "-"
+    name=${name:0:252}
+    kube::log::status "Creating secret with name: ${name} in namespace ${FEDERATION_NAMESPACE}"
+    ${kubectl} create secret generic ${name} --from-file="${base_dir}/${dir}/kubeconfig" --namespace="${FEDERATION_NAMESPACE}"
+  done
+}
+
+init
+create_cluster_secrets

federation/develop/develop.sh

@@ -117,14 +117,19 @@ function build_image() {
     make -C "${KUBE_ROOT}/cluster/images/hyperkube" build
 }
 
-function push() {
+function get_version() {
   local kube_version=""
   if [[ -n "${KUBE_VERSION:-}" ]]; then
     kube_version="${KUBE_VERSION}"
   else
     # Read the version back from the versions file if no version is given.
     kube_version="$(jq -r '.KUBE_VERSION' ${VERSIONS_FILE})"
   fi
+  echo "${kube_version}"
+}
+
+function push() {
+  local -r kube_version="$(get_version)"
 
   kube::log::status "Pushing hyperkube image to the registry"
   gcloud docker -- push "${KUBE_REGISTRY}/hyperkube-amd64:${kube_version}"

hack/e2e-internal/e2e-up.sh

@@ -43,14 +43,11 @@ if [[ "${FEDERATION:-}" == "true" ]]; then
         )
         cur_ip_octet2="$((cur_ip_octet2 + 1))"
     done
-    tagfile="${KUBE_ROOT}/federation/manifests/federated-image.tag"
-    if [[ ! -f "$tagfile" ]]; then
-        echo "FATAL: tagfile ${tagfile} does not exist. Make sure that you have run build/push-federation-images.sh"
-        exit 1
-    fi
-    export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")"
-
-    "${KUBE_ROOT}/federation/cluster/federation-up.sh"
+
+    # Sets ${CLUSTER_CONTEXT}
+    kubeconfig-federation-context "${zone}"
+
+    "${KUBE_ROOT}/federation/cluster/federation-up.sh" "${CLUSTER_CONTEXT}"
 else
     test-setup
 fi

hack/ginkgo-e2e.sh

@@ -128,6 +128,7 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}"
   --node-instance-group="${NODE_INSTANCE_GROUP:-}" \
   --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \
   --network="${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-e2e}}" \
+  --federated-kube-context="${FEDERATION_KUBE_CONTEXT:-e2e-federation}" \
   ${KUBE_CONTAINER_RUNTIME:+"--container-runtime=${KUBE_CONTAINER_RUNTIME}"} \
   ${MASTER_OS_DISTRIBUTION:+"--master-os-distro=${MASTER_OS_DISTRIBUTION}"} \
   ${NODE_OS_DISTRIBUTION:+"--node-os-distro=${NODE_OS_DISTRIBUTION}"} \

test/e2e/framework/framework.go

@@ -750,7 +750,7 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext {
 
 	e2eContexts := []E2EContext{}
 	for _, context := range kubeconfig.Contexts {
-		if strings.HasPrefix(context.Name, "federation") && context.Name != "federation-cluster" {
+		if strings.HasPrefix(context.Name, "federation") && context.Name != federatedKubeContext {
 
 			user := kubeconfig.findUser(context.Context.User)
 			if user == nil {

test/e2e/framework/test_context.go

@@ -163,7 +163,7 @@ func RegisterClusterFlags() {
 	flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
 	flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
 	flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver")
-	flag.StringVar(&federatedKubeContext, "federated-kube-context", "federation-cluster", "kubeconfig context for federation-cluster.")
+	flag.StringVar(&federatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.")
 
 	flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
 	flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")

test/e2e/framework/util.go

@@ -363,7 +363,7 @@ func SkipUnlessServerVersionGTE(v *utilversion.Version, c discovery.ServerVersio
 func SkipUnlessFederated(c clientset.Interface) {
 	federationNS := os.Getenv("FEDERATION_NAMESPACE")
 	if federationNS == "" {
-		federationNS = "federation"
+		federationNS = "federation-system"
 	}
 
 	_, err := c.Core().Namespaces().Get(federationNS, metav1.GetOptions{})