diff options
Diffstat (limited to 'scripts/maintain')
| -rwxr-xr-x | scripts/maintain/gluster/gluster.sh | 177 | ||||
| -rw-r--r-- | scripts/maintain/gluster/opts.sh | 21 | ||||
| -rw-r--r-- | scripts/maintain/prunerc.sh | 60 |
3 files changed, 258 insertions, 0 deletions
diff --git a/scripts/maintain/gluster/gluster.sh b/scripts/maintain/gluster/gluster.sh new file mode 100755 index 0000000..a3ff186 --- /dev/null +++ b/scripts/maintain/gluster/gluster.sh @@ -0,0 +1,177 @@ +#! /bin/bash + +. opts.sh + +[ $? -ne 0 -o -z "$gpod" ] && { echo "No storage pods are running..." ; exit 1 ; } +[ -z "$1" ] && { echo "Usage: $0 <info|heal|migrate> <vol|all> [src] [dst]" ; exit 1 ; } +action=$1 +shift + + +function info { + vol=$1 + + status=$(gluster volume info "$vol" | grep -P 'Status' | awk '{ print $2 }' | tr -d '\r\n') + bricks=$(gluster volume info "$vol" | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n') + avail=$(gluster volume status "$vol" detail | grep Brick | wc -l) + online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l) + + echo "Volume $vol: $status (Bricks: $bricks, Available: $avail, Online: $online)" +} + +function heal { + vol=$1 + + distributed=0 + gluster volume info "$vol" | grep "Type:" | grep -i "Distribute" &> /dev/null + [ $? -eq 0 ] && distributed=1 + + echo "Healing volume $vol" + echo "-------------------" + gluster volume heal "$vol" full + gluster volume heal "$vol" info + + if [ $distributed -eq 1 ]; then + echo "Rebalancing distributed volume $vol" + gluster volume rebalance "$vol" fix-layout start + fi + + + gluster volume status "$vol" +} + +function migrate { + vol=$1 + src=$2 + dst=$3 + + [ -z "$src" -o -z "$dst" ] && { echo "Source and destination servers are required" ; exit 1 ; } + + src_brick=$(gluster volume info $vol | grep -P '^Brick\d+:' | awk '{ print $2 }' | grep -P "^$src" | tr -d '\r\n' ) + dst_brick=${src_brick/$src/$dst} + + [ -z "$src_brick" -o -z "$dst_brick" ] && return 0 + + echo "Volume $vol: migrating failed brick" + echo " from $src_brick" + echo " to $dst_brick" + echo "Press enter to continue" + read + [ $? -ne 0 ] && exit + + gluster volume replace-brick $vol "$src_brick" "$dst_brick" commit force + heal $vol +} + + +function transport { + vol=$1 + transport=${2:-tcp,rdma} + echo "Changing $vol to transport $transport" + gluster volume stop "$vol" + gluster volume set "$vol" config.transport "$transport" + gluster volume start "$vol" +} + + + +function restart { + vol=$1 + + echo $vol + bricks=$(gluster volume info "$vol" | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n') + online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l) + + if [ "$bricks" -ne "$online" ]; then + echo "Restarting $vol ($online bricks of $bricks are/is online)" + gluster --mode=script volume stop "$vol" + gluster --mode=script volume start "$vol" + fi +} + +function delete_failed { + vol=$1 + + bricks=$(gluster volume info "$vol" | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n') + online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l) + + if [ "$online" == "0" ]; then + echo "Deleting $vol ($online bricks of $bricks are/is online)" +# gluster --mode=script volume stop "$vol" + gluster --mode=script volume delete "$vol" + fi +} + + +function lvm_clean { + used_bricks=`gluster volume info | grep "/brick_" | sed -r -e 's/.*brick_(.*)\/brick/\1/'` + + for ip in $(seq 1 3); do + echo "Node $ip" + echo "========" + lvm_bricks=`node $ip lvscan | grep brick | sed -r -e 's/.*brick_([0-9a-z]*)[^0-9a-z].*/\1/'` + + diff=$(echo $used_bricks $lvm_bricks | tr -d '\r' | tr ' ' '\n' | sort | uniq -u) + remove=$(echo "$diff $lvm_bricks" | tr -d '\r' | tr ' ' '\n' | sort | uniq -d) + + for id in $remove; do + echo "Removing ---------------------------------------------" + node $ip lvs -o name,time,size -S "'name =~ $id'" + echo "Removing ---------------------------------------------" + node $ip lvremove -y -S "'name =~ $id'" + done + done + + +} + +function lvm_remove_today { + for ip in $(seq 1 3); do + node $ip hostname +#lvdisplay -o name,time -S 'time since "2018-03-16"' + done +} + + +function heketi_cmd { + heketi "$@" +} + +function heketi_clean { + heketi_vols=`heketi topology info | grep "Name: vol_" | sed -r -e 's/.*(vol_[0-9a-z]+)\s*$/\1/'` + gluster_vols=`gluster volume info | grep "Name: vol_" | sed -r -e 's/.*(vol_[0-9a-z]+)\s*$/\1/'` + echo $heketi_vols + + diff=$(echo $gluster_vols $heketi_vols | tr -d '\r' | tr ' ' '\n' | sort | uniq -u) + remove=$(echo "$diff $gluster_vols" | tr -d '\r' | tr ' ' '\n' | sort | uniq -d) + + for vol in $remove; do + echo "Stopping and deleting volume $vol" + echo "---------------------------------" + gluster --mode=script volume stop "$vol" + gluster --mode=script volume delete "$vol" + done +} + +if [[ "$action" =~ ^heketi ]]; then + eval "$action" "$@" +elif [[ "$action" =~ ^lvm ]]; then + eval "$action" "$@" +elif [[ -z "$1" || "$1" =~ ^all ]]; then + all=0 + [ "$1" == "all_heketi" ] && all=1 + [[ "$1" =~ ^all ]] && shift + + vols=$(gluster volume info | grep -P '^Volume Name' | awk '{ print $NF }' | tr '\r\n' ' ') + for vol in $vols; do + if [ $all -eq 0 ]; then + [[ "$vol" =~ [0-9] ]] && continue + [[ "$vol" =~ ^vol_ ]] && continue + [[ "$vol" =~ ^heketi ]] && continue + fi + + eval "$action" "$vol" "$@" + done +else + eval "$action" "$@" +fi diff --git a/scripts/maintain/gluster/opts.sh b/scripts/maintain/gluster/opts.sh new file mode 100644 index 0000000..2f76d8e --- /dev/null +++ b/scripts/maintain/gluster/opts.sh @@ -0,0 +1,21 @@ +function get_gluster_pod { + oc -n glusterfs get pods -l 'glusterfs=storage-pod' | grep Running | awk '{ print $1 }' | head -n 1 +} + +gpod=$(get_gluster_pod) + +function gluster { + oc -n glusterfs rsh po/$gpod gluster "$@" +} + + +function node { + ip=$1 + shift + + ssh -xq root@192.168.26.$ip "$@" +} + +function heketi { + node 1 heketi-cli -s http://heketi-storage.glusterfs.svc.cluster.local:8080 --user admin --secret "$(oc get secret heketi-storage-admin-secret -n glusterfs -o jsonpath='{.data.key}' | base64 -d)" "$@" +} diff --git a/scripts/maintain/prunerc.sh b/scripts/maintain/prunerc.sh new file mode 100644 index 0000000..166ede6 --- /dev/null +++ b/scripts/maintain/prunerc.sh @@ -0,0 +1,60 @@ +#! /bin/bash + +setup="" +commit=0 +prune_stopped=0 + +while [ -n "$1" ]; do + case "$1" in + -c | --commit ) commit=1; shift ;; + -s | --stopped ) prune_stopped=1; shift ;; + -n | --setup ) setup="-n $2"; shift 2;; + * ) echo "$0 [-n project] [--commit] [--stopped]"; break ;; + esac +done + + +dcs="$(oc $setup get dc | grep -v NAME)" + +while read -r dc; do + name=$(echo "$dc" | awk '{ print $1 }') + revision=$(echo "$dc" | awk '{ print $2 }') + dc_desired=$(echo "$dc" | awk '{ print $3 }') + dc_current=$(echo "$dc" | awk '{ print $4 }') + + rcname="$name-$revision" + rc="$(oc $setup get rc $rcname | grep -v NAME)" + desired=$(echo "$rc" | awk '{ print $2 }') + current=$(echo "$rc" | awk '{ print $3 }') + ready=$(echo "$rc" | awk '{ print $4 }') + + if [ $dc_desired -ne $dc_current ]; then + [ $commit -eq 1 ] && echo "Skipping faulty dc/$name" + continue + elif [ $desired -ne $current -o $desired -ne $ready ]; then + [ $commit -eq 1 ] && echo "Skipping not completely ready dc/$name" + continue + elif [ $desired -eq 0 -a $prune_stopped -ne 1 ]; then + [ $commit -eq 1 ] && echo "Skipping stopped dc/$name (last one could be faulty)" + continue + fi + + + rcs="$(oc $setup get rc -l openshift.io/deployment-config.name=$name | grep -v NAME)" + + while read -r oldrc; do + oldname=$(echo "$oldrc" | awk '{ print $1 }') + desired=$(echo "$oldrc" | awk '{ print $2 }') + + [ $oldname = $rcname ] && continue + [ $desired -ne 0 ] && continue + + if [ $commit -eq 1 ]; then + echo "Removing: $oldname (keeping $rcname[$ready])" + oc $setup delete rc "$oldname" + else + echo "Will remove: $oldname (keeping $rcname[$ready])" + fi + done <<< "$rcs" + +done <<< "$dcs" |