initial commit

LICENSE

@@ -0,0 +1,39 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Greg May <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+This repository contains the code which has the following license
+notice:
+
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -0,0 +1,39 @@
+# kubectl-node-restart
+
+`kubectl-node-restart` is a [kubectl plugin](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/) that sequentially and gracefully performs a rolling restart of nodes within a Kubernetes cluster
+
+![using kubectl-node-restart plugin](demo/usage.gif)
+
+# Installing
+- install `krew` using instructions [here](https://github.com/kubernetes-sigs/krew#installation)
+- run `kubectl krew update`
+- run `kubectl krew install node-restart`
+
+![installing kubectl-node-restart plugin](demo/installation.gif)
+
+
+# Usage
+
+- perform rolling restart of all nodes in a cluster
+
+```bash
+    kubectl node-restart all
+```
+
+- restart only specific nodes selected through labels
+
+```bash
+    kubectl node-restart --selector=kubernetes.io/master
+```
+
+- perform a dry-run
+
+```bash
+    kubectl node-restart all --dry-run
+```
+
+- restart node(s) without first draining
+
+```bash
+    kubectl node-restart all --force
+```

node-restart.sh

@@ -0,0 +1,176 @@
+#!/bin/bash
+
+image='container-registry01.nonprod.wsgc.com/official/alpine:3.9'
+nodesleep=10   #Time between node restarts - give pods time to start up
+force=false
+dryrun=false
+blue='\033[0;34m'
+nocolor='\033[0m'
+
+function print_usage() {
+  echo "Usage: $0 [<options>]"
+  echo ""
+  echo "all                                 Restarts all nodes within the cluster"
+  echo ""
+  echo "-l|--selector key=value             Selector (label query) to target specific nodes"
+  echo ""
+  echo "-f|--force                          Restart node(s) without first draining"
+  echo ""
+  echo "-d|--dry-run                        Just print what to do; don't actually do it"
+  echo ""
+  echo "-h|--help                           Print usage and exit"
+}
+
+while [[ $# -gt 0 ]]
+do
+  key="$1"
+
+  case $key in
+    all)
+    allnodes=true
+    shift
+    ;;
+    -l|--selector)
+    selector="$2"
+    shift
+    shift
+    ;;
+    -f|--force)
+    force=true
+    shift
+    ;;
+    -d|--dry-run)
+    dryrun=true
+    shift
+    ;;
+    -h|--help)
+    print_usage
+    exit 0
+    ;;
+    *)
+    print_usage
+    exit 1
+    ;;
+  esac
+done
+
+function wait_for_job_completion() {
+  pod=$1
+  i=0
+  while [[ $i -lt 30 ]]; do
+    status=$(kubectl get job $pod -o "jsonpath={.status.succeeded}" 2>/dev/null)
+    if [[ $status -gt 0 ]]; then
+      echo "Restart complete after $((i*10)) seconds"
+      break;
+    else
+      i=$(($i+1))
+      sleep 10s
+      echo "$node - $((i*10)) seconds"
+    fi
+  done
+  if [[ $i == 30 ]]; then
+    echo "Error: Restart job did not complete within 5 minutes"
+    exit 1
+  fi
+}
+
+function wait_for_status() {
+  node=$1
+  i=0
+  while [[ $i -lt 30 ]]; do
+    status=$(kubectl get node $node -o "jsonpath={.status.conditions[?(.reason==\"KubeletReady\")].type}" 2>/dev/null)
+    if [[ "$status" == "Ready" ]]; then
+      echo "KubeletReady after $((i*10)) seconds"
+      break;
+    else
+      i=$(($i+1))
+      sleep 10s
+      echo "$node NotReady - waited $((i*10)) seconds"
+    fi
+  done
+  if [[ $i == 30 ]]; then
+    echo "Error: Did not reach KubeletReady state within 5 minute"
+    exit 1
+  fi
+}
+
+if [ "$allnodes" == "true" ]; then
+  nodes=$(kubectl get nodes -o jsonpath={.items[*].metadata.name})
+  echo -e "${blue}Targeting nodes:${nocolor}"
+  for node in $nodes; do
+    echo " $node"
+  done
+elif [ ! -z "$selector" ]; then
+  nodes=$(kubectl get nodes --selector=$selector -o jsonpath={.items[*].metadata.name})
+  echo -e "${blue}Targeting nodes:${nocolor}"
+  for node in $nodes; do
+    echo " $node"
+  done
+else
+  print_usage
+fi
+
+for node in $nodes; do
+  if $force; then
+    echo -e "\nWARNING: --force specified, restarting node $node without draining first"
+    if $dryrun; then
+      echo "kubectl cordon $node"
+    else
+      kubectl cordon "$node"
+    fi
+  else
+    echo -e "\n${blue}Draining node $node...${nocolor}"
+    if $dryrun; then
+      echo "kubectl drain $node --ignore-daemonsets --delete-local-data"
+    else
+      kubectl drain "$node" --ignore-daemonsets --delete-local-data
+    fi
+  fi
+
+  echo -e "${blue}Initiating node restart job on $node...${nocolor}"
+  pod="node-restart-$(env LC_CTYPE=C tr -dc a-z0-9 < /dev/urandom | head -c 5)"
+  if $dryrun; then
+    echo "kubectl create job $pod"
+  else
+cat <<EOT | kubectl apply -f -
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: $pod
+spec:
+  backoffLimit: 3
+  ttlSecondsAfterFinished: 30
+  template:
+    spec:
+      nodeName: $node
+      hostPID: true
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      containers:
+      - name: $pod
+        image: $image
+        command: [ "nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--pid", "--", "bash", "-c" ]
+        args: [ "if [ -f /node-restart-flag ]; then rm /node-restart-flag && exit 0; else touch /node-restart-flag && reboot && exit 1; fi" ]
+        securityContext:
+          privileged: true
+      restartPolicy: Never
+EOT
+  fi
+
+  if ! $dryrun; then
+    echo -e "${blue}Waiting for restart job to complete on node $node...${nocolor}"
+    wait_for_job_completion $pod
+    wait_for_status $node KubeletReady
+  fi
+
+  echo -e "${blue}Uncordoning node $node${nocolor}"
+
+  if $dryrun; then
+    echo "kubectl uncordon $node"
+  else
+    kubectl uncordon "$node"
+    kubectl delete job $pod
+    sleep $nodesleep
+  fi
+done

node-restart.yaml

@@ -0,0 +1,32 @@
+apiVersion: krew.googlecontainertools.github.com/v1alpha2
+kind: Plugin
+metadata:
+  name: node-restart               # plugin name must match your manifest file name (e.g. foo.yaml)
+spec:
+  version: "v0.0.1"       # required, must be in semver format, prefixed with "v"
+  platforms:
+  # specify installation script for linux and darwin (macOS)
+  - selector:             # a regular Kubernetes selector
+      matchExpressions:
+      - {key: os, operator: In, values: [darwin, linux]}
+    # url for downloading the package archive:
+    uri: https://github.com/mnrgreg/kubectl-node-restart/releases/v0.0.1.zip
+    # sha256sum of the above archive file:
+    sha256: ""
+    # copy the used files out of the zip archive, defaults to `[{from: "*", to: "."}]`
+    files:
+    - from: "kubectl-node-restart/*.sh" # path to the files extracted from archive
+      to:  "."                 # '.' refers to the root of plugin install directory
+    - from: "kubectl-node-restart/LICENSE"   # always install your LICENSE file
+      to: "."
+    bin: "kubectl-node-restart"  # path to the plugin executable after copying files above
+  shortDescription: >-    # short description gets truncated at ~50 chars
+    Restarts K8S nodes sequentially & gracefully
+  homepage: https://github.com/mnrgreg/kubectl-node-restart
+  # (optional) use caveats field to show post-installation recommendations
+  caveats: |
+    This plugin requires execution with cluster-admin priveleges.
+  description: |
+    This plugin performs a sequential, rolling restart of selected nodes by first
+    draining each node, then running a kubernetes job to reboot each host, and
+    finally uncordoning each node when Ready.