test scripts

dell-publish-unpublish-test/prepare.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+export INDEX
+export NODENAME
+
+for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do
+  echo "PVC: $INDEX"
+  envsubst < test-block-00.yaml | oc apply -f -
+done
+
+NODENAME="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do
+  echo "Shell: $INDEX"
+  envsubst < test-shell-00.yaml | oc apply -f -
+done

dell-publish-unpublish-test/publish-unpublish-test.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+while : ; do
+  ./switch.sh 1
+  #sleep 480
+  sleep 120
+  echo "$(date +'%Y-%m-%d %H:%M:%S') Pods State not in running state"
+  oc get pods -n test-richph | grep -v "Running"
+  ./switch.sh 2
+  # sleep 480
+  sleep 120
+  echo "$(date +'%Y-%m-%d %H:%M:%S') Pods State not in running state"
+  oc get pods -n test-richph | grep -v "Running"
+done

dell-publish-unpublish-test/remove.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+export INDEX
+export NODENAME
+
+for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do
+  echo "PVC: $INDEX"
+  envsubst < test-block-00.yaml | oc delete -f -
+done
+
+NODENAME="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do
+  echo "Shell: $INDEX"
+  envsubst < test-shell-00.yaml | oc delete -f -
+done

dell-publish-unpublish-test/removens.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export INDEX
+
+for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do
+  oc patch deployments test-shell-$INDEX -n test-richph -p '{"spec": {"template": {"spec": {"nodeSelector": null }}}}' >/dev/null
+done

dell-publish-unpublish-test/scale.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export INDEX
+export NODENAME
+SCALE=${1:-0}
+
+for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do
+  echo "Shell: $INDEX"
+  oc scale --replicas=$SCALE deployment/test-shell-$INDEX -n test-richph
+done

dell-publish-unpublish-test/switch.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+export INDEX
+export NODENAME
+
+case $1 in
+1)
+  NODENAME1="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+  NODENAME2="ocp-b-play-phy-worker-1-g1.cloud.svc.co.at"
+  ;;
+2)
+  NODENAME1="ocp-b-play-phy-worker-1-g1.cloud.svc.co.at"
+  NODENAME2="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+  ;;
+esac
+
+NODENAME="$NODENAME1"
+echo "$(date +'%Y-%m-%d %H:%M:%S') Move 0x to $NODENAME"
+for INDEX in 0{0,1,2,3,4,5,6,7,8,9} ; do
+  oc patch deployments test-shell-$INDEX -n test-richph -p '{"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": "'$NODENAME'"}}}}}' >/dev/null
+done
+
+NODENAME="$NODENAME2"
+echo "$(date +'%Y-%m-%d %H:%M:%S') Move 1x to $NODENAME"
+for INDEX in 1{0,1,2,3,4,5,6,7,8,9} ; do
+  oc patch deployments test-shell-$INDEX -n test-richph -p '{"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": "'$NODENAME'"}}}}}' >/dev/null
+done
+

dell-publish-unpublish-test/test-block-00.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test-block-$INDEX
+  namespace: test-richph
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: powermax-x
+  volumeMode: Block

dell-publish-unpublish-test/test-shell-00.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-shell-$INDEX
+  namespace: test-richph
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 0
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: test-shell-$INDEX
+  template:
+    metadata:
+      labels:
+        app: test-shell-$INDEX
+        deploymentconfig: test-shell-$INDEX
+    spec:
+      containers:
+      - args:
+        - tail
+        - -f
+        - /dev/null
+        image: ocp-play-quay.cloud.svc.co.at/rhel8/support-tools:latest
+        imagePullPolicy: IfNotPresent
+        name: tools
+        resources: {}
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeDevices:
+        - devicePath: /dev/test-block
+          name: test-block
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      nodeSelector:
+        kubernetes.io/hostname: $NODENAME
+      volumes:
+      - name: test-block
+        persistentVolumeClaim:
+          claimName: test-block-$INDEX
+  strategy:
+    type: Recreate

reboot-test/drain-test.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+HOSTS=(
+#  "ocp-a-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-a-play-phy-infra-2-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-2-g1.cloud.svc.co.at"
+  "ocp-b-play-phy-worker-1-g1.cloud.svc.co.at"
+  "ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+#  "ocp-a-play-phy-worker-2-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-worker-2-g1.cloud.svc.co.at"
+)
+
+for i in 1 2 ; do
+for H in ${HOSTS[@]} ; do
+  echo "Draining Host $H ... (Run: $i)"
+  oc adm cordon $H
+  echo "  -  draining"
+  oc adm drain $H --ignore-daemonsets --delete-emptydir-data --force
+  sleep 10
+  # echo "  -  reboot"
+  # ssh core@$H sudo systemctl reboot
+  # sleep 10
+  # echo "  -  wait for not ready"
+  # oc wait --for=condition=Ready=Unknown --timeout=10m nodes $H
+  # sleep 5
+  # echo "  -  wait for ready"
+  # oc wait --for=condition=Ready --timeout=10m nodes $H
+  # #echo "  -  rescan scsi bus"
+  # #ssh core@$H sudo rescan-scsi-bus.sh -r
+  # sleep 10 
+  echo "  -  set schedulable"
+  oc adm uncordon $H
+  echo "  -  sleep 2min"
+  sleep 120
+done
+done

reboot-test/reboot-test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+HOSTS=(
+#  "ocp-a-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-1-g1.cloud.svc.co.at"
+#  "ocp-a-play-phy-infra-2-g1.cloud.svc.co.at"
+#  "ocp-b-play-phy-infra-2-g1.cloud.svc.co.at"
+  "ocp-a-play-phy-worker-1-g1.cloud.svc.co.at"
+  "ocp-b-play-phy-worker-1-g1.cloud.svc.co.at"
+  "ocp-a-play-phy-worker-2-g1.cloud.svc.co.at"
+  "ocp-b-play-phy-worker-2-g1.cloud.svc.co.at"
+)
+
+for H in ${HOSTS[@]} ; do
+  echo "Rebooting Host $H ..."
+  oc adm cordon $H
+  echo "  -  draining"
+  oc adm drain $H --ignore-daemonsets --delete-emptydir-data --force
+  echo "  -  sleep 120"
+  sleep 120
+  
+  oc get pods -o wide -A | grep $H > $H-pods.log
+  oc get volumeattachments > $H-volumeattachments.log
+  oc logs $( oc get pods -n powermax -l app=powermax-node --field-selector spec.nodeName=$H -o name ) -n powermax | xz > $H-powermax-node.log.xz
+  ssh core@$H "sudo multipath -ll" > $H-multipath_ll.log
+
+  echo "  -  reboot"
+  ssh core@$H sudo systemctl reboot
+  sleep 10
+  echo "  -  wait for not ready"
+  oc wait --for=condition=Ready=Unknown --timeout=10m nodes $H
+  sleep 5
+  echo "  -  wait for ready"
+  oc wait --for=condition=Ready --timeout=10m nodes $H
+  sleep 10 
+  echo "  -  set schedulable"
+  oc adm uncordon $H
+  #sleep 30
+  read -p "PAUSE PRESS ENTER"
+done