test scripts

reboot-test/drain-test.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+HOSTS=(
+#  "ocp-a-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-a-play-phy-infra-2-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-2-g1.cloud.svc.co.at"
+  "ocp-b-play-phy-worker-1-g1.cloud.svc.co.at"
+  "ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+#  "ocp-a-play-phy-worker-2-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-worker-2-g1.cloud.svc.co.at"
+)
+
+for i in 1 2 ; do
+for H in ${HOSTS[@]} ; do
+  echo "Draining Host $H ... (Run: $i)"
+  oc adm cordon $H
+  echo "  -  draining"
+  oc adm drain $H --ignore-daemonsets --delete-emptydir-data --force
+  sleep 10
+  # echo "  -  reboot"
+  # ssh core@$H sudo systemctl reboot
+  # sleep 10
+  # echo "  -  wait for not ready"
+  # oc wait --for=condition=Ready=Unknown --timeout=10m nodes $H
+  # sleep 5
+  # echo "  -  wait for ready"
+  # oc wait --for=condition=Ready --timeout=10m nodes $H
+  # #echo "  -  rescan scsi bus"
+  # #ssh core@$H sudo rescan-scsi-bus.sh -r
+  # sleep 10 
+  echo "  -  set schedulable"
+  oc adm uncordon $H
+  echo "  -  sleep 2min"
+  sleep 120
+done
+done

reboot-test/reboot-test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+HOSTS=(
+#  "ocp-a-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-a-play-phy-infra-2-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-2-g1.cloud.svc.co.at"
+  "ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+  "ocp-b-play-phy-worker-1-g1.cloud.svc.co.at"
+  "ocp-a-play-phy-worker-2-g1.cloud.svc.co.at"
+  "ocp-b-play-phy-worker-2-g1.cloud.svc.co.at"
+)
+
+for H in ${HOSTS[@]} ; do
+  echo "Rebooting Host $H ..."
+  oc adm cordon $H
+  echo "  -  draining"
+  oc adm drain $H --ignore-daemonsets --delete-emptydir-data --force
+  echo "  -  sleep 120"
+  sleep 120
+  
+  oc get pods -o wide -A | grep $H > $H-pods.log
+  oc get volumeattachments > $H-volumeattachments.log
+  oc logs $( oc get pods -n powermax -l app=powermax-node --field-selector spec.nodeName=$H -o name ) -n powermax | xz > $H-powermax-node.log.xz
+  ssh core@$H "sudo multipath -ll" > $H-multipath_ll.log
+
+  echo "  -  reboot"
+  ssh core@$H sudo systemctl reboot
+  sleep 10
+  echo "  -  wait for not ready"
+  oc wait --for=condition=Ready=Unknown --timeout=10m nodes $H
+  sleep 5
+  echo "  -  wait for ready"
+  oc wait --for=condition=Ready --timeout=10m nodes $H
+  sleep 10 
+  echo "  -  set schedulable"
+  oc adm uncordon $H
+  #sleep 30
+  read -p "PAUSE PRESS ENTER"
+done