docker-pv-resizer/check_full_pvs.sh

#!/bin/bash

showHelp () {
  echo "Script to check for full PVs in Kubernetes."
  echo "Requires kubectl with a configuration file in ~/.kube/config and df-pv krew plugin."
  echo "Usage: "
  echo "  ./check_full_pvs.sh [-t threshold] [-f]"
  echo "Parameters:"
  echo "  -t (optional) : Threshold to use for near full PV evaluation (in %). Defaults to 85%."
  echo "  -f (optional) : Fix near full PVs (resize them). Defaults to false."
  echo "  -i (optional) : Percentage of PV increase when resizing. Defaults to 50%."
}

checkReqs () {
  if [ ! -x "$(command -v "kubectl")" ]; then
    echo "ERROR: Kubectl is required for this script to run."
    exit
  fi
  kubectl df-pv -h > /dev/null
  if [ $? -ne 0 ]; then
    echo "ERROR: df-pv kubectl plugin is required for this script to run. First install krew, then install df-pv plugin."
    exit 
  fi

}

scaleworkload () {
  NAMESPACE=$1
  TYPE=$2
  WORKLOADNAME=$3
  SCALE=$4
      
  echo -n " |-- Scaling $TYPE $WORKLOADNAME in $NAMESPACE to $SCALE... " && \
    kubectl scale --replicas=$SCALE $TYPE $WORKLOADNAME --namespace ${NAMESPACE}
  if [ $? -ne 0 ]; then
    echo " |-- Error scaling $TYPE $WORKLOADNAME to $SCALE."
  fi
}


waitforresize (){
  ns=$1
  pvcname=$2
 
  echo " |-- Waiting for volume resize for $pvcname..."
  pendingresize=0
  for count in $(seq 1 60); do
    pvcstatus=$(kubectl get pvc $pvcname -n $ns -o=jsonpath='{.status.conditions[0].type}')
    if [[ "$pvcstatus" == "FileSystemResizePending" ]]; then
      echo " |-- $pvcname resized. Need now to resize inner filesystem. This happens automatically when a pod mounts the volume."
      pendingresize=1
      break
    else
      sleep 10
    fi
  done
  if [ $pendingresize -eq 0 ]; then
    echo " |-- Timeout trying to resize $pvcname."
  fi

}

restartforresize () {
  NAMESPACE=$1
  TYPE=$2
  PVC=$3

  WORKLOADS=$(kubectl get $TYPE -n $NAMESPACE -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.template.spec.volumes[].persistentVolumeClaim.claimName}{" "}{.status.replicas}{"\n"}{end}')
  if [ $? -ne 0 ]; then
    echo " |-- Error trying to get deployments for namespace $NAMESPACE."
  else
    for WORKLOAD in $WORKLOADS; do
      name=$(echo $WORKLOAD | awk '{print $1}')
      pvcname=$(echo $WORKLOAD | awk '{print $2}')
      scale=$(echo $WORKLOAD | awk '{print $3}')

      if [[ "$pvcname" == $PVC ]]; then
        scaleworkload $NAMESPACE $TYPE $name 0
	waitforresize $NAMESPACE $pvcname
	scaleworkload $NAMESPACE $TYPE $name $scale
      fi
    done
  fi

}

FULLTHRESHOLD=85
FIX=0
INCREASEPERC=50

while getopts ":t:i:f" opt; do
  case ${opt} in
    t )
      FULLTHRESHOLD=$OPTARG
      ;;
    f )
      FIX=1
      ;;
    i )
      INCREASEPERC=$OPTARG
      ;;
    \? )
      echo "ERROR: Invalid option: $OPTARG" 1>&2
      showHelp
      exit 1
      ;;
    : )
      echo "ERROR: Invalid option: $OPTARG requires an argument" 1>&2
      showHelp
      exit 1
      ;;
  esac
done
shift $((OPTIND -1))

checkReqs

ALLNS=$(kubectl get ns -o=jsonpath='{.items[*].metadata.name}')

IFS=$' '
for ns in $ALLNS; do
  usedstr=$(kubectl df-pv -n ${ns} -d | grep -v "PV NAME" | sed -r '/^\s*$/d')
  IFS=$'\n'
  for pv in $usedstr; do 
    usedperc=$(echo "$pv" | awk '{print $10}')
    pvcname=$(echo "$pv" | awk '{print $2}')
    size=$(echo "$pv" | awk '{print $7}' | numfmt --from=iec-i --to=iec-i --to-unit=Gi --round=up)
    if (( $(bc <<< "($usedperc > $FULLTHRESHOLD)") )); then
      echo "Volume $pvcname is almost full: ${usedperc}%!"
      if [ $FIX -ne 0 ]; then
	pvcstatus=$(kubectl get pvc $pvcname -n $ns -o=jsonpath='{.status.conditions[0].type}')
	if [[ "$pvcstatus" == 'Resizing' ]]; then
          echo " |-- Volume $pvcname already has a Resizing operation going on."
	  restartforresize $ns deployment $pvcname
	  restartforresize $ns statefulset $pvcname
        else
	  newsize=$(echo ${size}*1.${INCREASEPERC}| bc | grep -v "$\.0")
          echo " |-- Resizing $pvcname: ${size}Gi --> ${newsize}Gi..."
	fi
      fi
    fi
  done
done