diff --git a/dell-publish-unpublish-test/prepare.sh b/dell-publish-unpublish-test/prepare.sh new file mode 100755 index 0000000..09e1656 --- /dev/null +++ b/dell-publish-unpublish-test/prepare.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +export INDEX +export NODENAME + +for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do + echo "PVC: $INDEX" + envsubst < test-block-00.yaml | oc apply -f - +done + +NODENAME="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at" +for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do + echo "Shell: $INDEX" + envsubst < test-shell-00.yaml | oc apply -f - +done diff --git a/dell-publish-unpublish-test/publish-unpublish-test.sh b/dell-publish-unpublish-test/publish-unpublish-test.sh new file mode 100755 index 0000000..04c48af --- /dev/null +++ b/dell-publish-unpublish-test/publish-unpublish-test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +while : ; do + ./switch.sh 1 + #sleep 480 + sleep 120 + echo "$(date +'%Y-%m-%d %H:%M:%S') Pods State not in running state" + oc get pods -n test-richph | grep -v "Running" + ./switch.sh 2 + # sleep 480 + sleep 120 + echo "$(date +'%Y-%m-%d %H:%M:%S') Pods State not in running state" + oc get pods -n test-richph | grep -v "Running" +done diff --git a/dell-publish-unpublish-test/remove.sh b/dell-publish-unpublish-test/remove.sh new file mode 100755 index 0000000..f0ce19f --- /dev/null +++ b/dell-publish-unpublish-test/remove.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +export INDEX +export NODENAME + +for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do + echo "PVC: $INDEX" + envsubst < test-block-00.yaml | oc delete -f - +done + +NODENAME="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at" +for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do + echo "Shell: $INDEX" + envsubst < test-shell-00.yaml | oc delete -f - +done diff --git a/dell-publish-unpublish-test/removens.sh b/dell-publish-unpublish-test/removens.sh new file mode 100755 index 0000000..51d6416 --- /dev/null +++ b/dell-publish-unpublish-test/removens.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +export INDEX + +for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do + oc patch deployments test-shell-$INDEX -n test-richph -p '{"spec": {"template": {"spec": {"nodeSelector": null }}}}' >/dev/null +done diff --git a/dell-publish-unpublish-test/scale.sh b/dell-publish-unpublish-test/scale.sh new file mode 100755 index 0000000..fdf79ef --- /dev/null +++ b/dell-publish-unpublish-test/scale.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export INDEX +export NODENAME +SCALE=${1:-0} + +for INDEX in {0,1}{0,1,2,3,4,5,6,7,8,9} ; do + echo "Shell: $INDEX" + oc scale --replicas=$SCALE deployment/test-shell-$INDEX -n test-richph +done diff --git a/dell-publish-unpublish-test/switch.sh b/dell-publish-unpublish-test/switch.sh new file mode 100755 index 0000000..370ba01 --- /dev/null +++ b/dell-publish-unpublish-test/switch.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +export INDEX +export NODENAME + +case $1 in +1) + NODENAME1="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at" + NODENAME2="ocp-b-play-phy-worker-1-g1.cloud.svc.co.at" + ;; +2) + NODENAME1="ocp-b-play-phy-worker-1-g1.cloud.svc.co.at" + NODENAME2="ocp-a-play-phy-worker-1-g1.cloud.svc.co.at" + ;; +esac + +NODENAME="$NODENAME1" +echo "$(date +'%Y-%m-%d %H:%M:%S') Move 0x to $NODENAME" +for INDEX in 0{0,1,2,3,4,5,6,7,8,9} ; do + oc patch deployments test-shell-$INDEX -n test-richph -p '{"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": "'$NODENAME'"}}}}}' >/dev/null +done + +NODENAME="$NODENAME2" +echo "$(date +'%Y-%m-%d %H:%M:%S') Move 1x to $NODENAME" +for INDEX in 1{0,1,2,3,4,5,6,7,8,9} ; do + oc patch deployments test-shell-$INDEX -n test-richph -p '{"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": "'$NODENAME'"}}}}}' >/dev/null +done + diff --git a/dell-publish-unpublish-test/test-block-00.yaml b/dell-publish-unpublish-test/test-block-00.yaml new file mode 100644 index 0000000..5caf2a9 --- /dev/null +++ b/dell-publish-unpublish-test/test-block-00.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: test-block-$INDEX + namespace: test-richph +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: powermax-x + volumeMode: Block diff --git a/dell-publish-unpublish-test/test-shell-00.yaml b/dell-publish-unpublish-test/test-shell-00.yaml new file mode 100644 index 0000000..c0d3401 --- /dev/null +++ b/dell-publish-unpublish-test/test-shell-00.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-shell-$INDEX + namespace: test-richph +spec: + progressDeadlineSeconds: 600 + replicas: 0 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: test-shell-$INDEX + template: + metadata: + labels: + app: test-shell-$INDEX + deploymentconfig: test-shell-$INDEX + spec: + containers: + - args: + - tail + - -f + - /dev/null + image: ocp-play-quay.cloud.svc.co.at/rhel8/support-tools:latest + imagePullPolicy: IfNotPresent + name: tools + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeDevices: + - devicePath: /dev/test-block + name: test-block + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + nodeSelector: + kubernetes.io/hostname: $NODENAME + volumes: + - name: test-block + persistentVolumeClaim: + claimName: test-block-$INDEX + strategy: + type: Recreate diff --git a/reboot-test/drain-test.sh b/reboot-test/drain-test.sh new file mode 100755 index 0000000..82de8dd --- /dev/null +++ b/reboot-test/drain-test.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +HOSTS=( +# "ocp-a-play-phy-infra-1-g1.cloud.svc.co.at" +# "ocp-b-play-phy-infra-1-g1.cloud.svc.co.at" +# "ocp-a-play-phy-infra-2-g1.cloud.svc.co.at" +# "ocp-b-play-phy-infra-2-g1.cloud.svc.co.at" + "ocp-b-play-phy-worker-1-g1.cloud.svc.co.at" + "ocp-a-play-phy-worker-1-g1.cloud.svc.co.at" +# "ocp-a-play-phy-worker-2-g1.cloud.svc.co.at" +# "ocp-b-play-phy-worker-2-g1.cloud.svc.co.at" +) + +for i in 1 2 ; do +for H in ${HOSTS[@]} ; do + echo "Draining Host $H ... (Run: $i)" + oc adm cordon $H + echo " - draining" + oc adm drain $H --ignore-daemonsets --delete-emptydir-data --force + sleep 10 + # echo " - reboot" + # ssh core@$H sudo systemctl reboot + # sleep 10 + # echo " - wait for not ready" + # oc wait --for=condition=Ready=Unknown --timeout=10m nodes $H + # sleep 5 + # echo " - wait for ready" + # oc wait --for=condition=Ready --timeout=10m nodes $H + # #echo " - rescan scsi bus" + # #ssh core@$H sudo rescan-scsi-bus.sh -r + # sleep 10 + echo " - set schedulable" + oc adm uncordon $H + echo " - sleep 2min" + sleep 120 +done +done diff --git a/reboot-test/reboot-test.sh b/reboot-test/reboot-test.sh new file mode 100755 index 0000000..7308392 --- /dev/null +++ b/reboot-test/reboot-test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +HOSTS=( +# "ocp-a-play-phy-infra-1-g1.cloud.svc.co.at" +# "ocp-b-play-phy-infra-1-g1.cloud.svc.co.at" +# "ocp-a-play-phy-infra-2-g1.cloud.svc.co.at" +# "ocp-b-play-phy-infra-2-g1.cloud.svc.co.at" + "ocp-a-play-phy-worker-1-g1.cloud.svc.co.at" + "ocp-b-play-phy-worker-1-g1.cloud.svc.co.at" + "ocp-a-play-phy-worker-2-g1.cloud.svc.co.at" + "ocp-b-play-phy-worker-2-g1.cloud.svc.co.at" +) + +for H in ${HOSTS[@]} ; do + echo "Rebooting Host $H ..." + oc adm cordon $H + echo " - draining" + oc adm drain $H --ignore-daemonsets --delete-emptydir-data --force + echo " - sleep 120" + sleep 120 + + oc get pods -o wide -A | grep $H > $H-pods.log + oc get volumeattachments > $H-volumeattachments.log + oc logs $( oc get pods -n powermax -l app=powermax-node --field-selector spec.nodeName=$H -o name ) -n powermax | xz > $H-powermax-node.log.xz + ssh core@$H "sudo multipath -ll" > $H-multipath_ll.log + + echo " - reboot" + ssh core@$H sudo systemctl reboot + sleep 10 + echo " - wait for not ready" + oc wait --for=condition=Ready=Unknown --timeout=10m nodes $H + sleep 5 + echo " - wait for ready" + oc wait --for=condition=Ready --timeout=10m nodes $H + sleep 10 + echo " - set schedulable" + oc adm uncordon $H + #sleep 30 + read -p "PAUSE PRESS ENTER" +done