diff options
Diffstat (limited to 'lib/remote')
| -rw-r--r-- | lib/remote/README.txt | 34 | ||||
| -rw-r--r-- | lib/remote/print.sh | 168 | ||||
| -rw-r--r-- | lib/remote/report.sh | 51 | ||||
| -rw-r--r-- | lib/remote/run.sh | 13 | ||||
| -rw-r--r-- | lib/remote/status.sh | 196 |
5 files changed, 462 insertions, 0 deletions
diff --git a/lib/remote/README.txt b/lib/remote/README.txt new file mode 100644 index 0000000..c5998f8 --- /dev/null +++ b/lib/remote/README.txt @@ -0,0 +1,34 @@ +conky format +============ + - Server status + * ping works + * no output from 'check_server_status.sh' and additional listed checks + [*] Checks for additional listed services 'check_<service>_status.sh'. + - The optional status is in the beginning of first line (0 - red, 1 - green, 2 - yellow). Otherwise, 'red' if any output (also if first word is non-number on the first string). + - If first line starts from the status, the rest of the line is considered a header and reported on the status line. Everything else goes to textual section bellow. + [headers] + + ... Textual information reported by 'check_server_status.sh' and check_*_status.sh scripts. The format + server:port[:service] text.... + + - Extra info for DarkSoft servers + * VPN to the server is working (checked with the ping). This not included in the scripted version, instead server traffic reported directly here. + # # Traffic on VPN interface from/to + + - Server traffic + in/out/forw. Yellow/Red if above specified threshold. No additional errors are reported here. + + - Services (operated by local scripts in 'services' directory) + * Online + * Healthy + [headers] Information about service operation + [errors] Short information on critical errors are reported here (in conky, we push everything in the text section) + + ... Larger textual information about the problems. + + + Service format is following (status is on the last line and mandatory): + * Critical Lines starting with '*' contain critical errors and highlighted (remote) / reported in the service table (conky) + Text This is normal problem description reported in the textual area + # # [header] Two statuses (0 - error, 1 - ok, 2 - warning) and optional header with information about service operation +
\ No newline at end of file diff --git a/lib/remote/print.sh b/lib/remote/print.sh new file mode 100644 index 0000000..a0fdcbe --- /dev/null +++ b/lib/remote/print.sh @@ -0,0 +1,168 @@ +# underlines 4;30,... blinking 5:30,... +# backgrounds: 40,... + +declare -A colors=( + [black]='0;30' + [red]='0;31' + [green]='0;32' + [orange]='0;33' + [blue]='0;34' + [purple]='0;35' + [cyan]='0;36' + [lightgray]='0;37' + [gray]='1;30' + [lightred]='1;31' + [lightgreen]='1;32' + [yellow]='1;33' + [lightblue]='1;34' + [lightpurple]='1;35' + [lightcyan]='1;36' + [white]='1;37' +) + +background="" +foreground="" + + + + +# Fixing color map +#echo -ne "\e[44m" +#echo -ne '\x1B[K' + +function set_background { + if [ -n "$background" ]; then + echo -ne "$background" + echo -ne '\x1B[K' + fi +} + +function set_color { + local color=${1:-""} + local effect=${2:-""} + + c=${colors[$color]} + + + case "$effect" in + 'u') + c=$(echo $c | sed -r "s/^0/4/ ; s/^1/1;4/") + ;; + esac + + [ -n "$color" ] && echo -ne "\033[${c}m" + set_background +} + +function reset_color { + echo -ne "\033[0m" + + set_background + [ -n "$foreground" ] && set_color "$foreground" +} + +function finish { + echo -ne "\033[0m" + echo +} + +function configure_palete { + case $1 in + blue) + background="\e[1;44m" + foreground="white" + ;; + *) + # Lightgray and Yellow are almost invisible on white + colors[yellow]=${colors[orange]} + colors[white]=${colors[black]} + colors[lightgray]=${colors[gray]} + colors[cyan]=${colors[blue]} + ;; + esac + + set_background + reset_color +} + +function decorate { + local output="$1" + local color=${2:-""} + local effect=${3:-""} + + if [ -n "$color" ]; then + set_color $color $effect + echo -ne "${output}" + reset_color + else + echo -ne "$output" + fi +} + +function eol { + sed 's/$/\x1B[K/' +} + +function print_eol { + echo -e '\x1B[K' +} + + + +function print { + local output="$1" + local color=${2:-""} + local char_limit=${3:-$fold} + local line_limit=${4:-0} + + local cmd="decorate \"${output}\" \"$color\" | fold -s -w $char_limit" + [ $line_limit -gt 0 ] && cmd="$cmd | head -n $line_limit" + + eval "$cmd" | eol +} + + +function print_hline { + local color=${1:-"gray"} + local cols=${COLUMNS:-$(tput cols)} + + decorate $(printf '%*s' $cols '' | tr ' ' -) "$color" + echo | eol +} + +function print_table { + # The problem here is that all escapes for colors are treated as normal visible symbols + sed "s/ ::: /@/g" | column -t -s "@" -o " " | eol +} + +function decorate_table { + print_hline + cat - + print_hline +} + +function print_status { + local status="$1" + + if [[ $status =~ ^.$ ]]; then + case $status in + 0) + decorate "*" "red" + ;; + 1) + decorate "*" "green" + ;; + 2) + decorate "*" "yellow" + ;; + *) + decorate "$status" "red" + ;; + esac + else + decorate "?" "red" + fi + echo -n " " +} + +clear diff --git a/lib/remote/report.sh b/lib/remote/report.sh new file mode 100644 index 0000000..e6766a4 --- /dev/null +++ b/lib/remote/report.sh @@ -0,0 +1,51 @@ +function simple_report { + local generator="$1" + local filter="${2:-cat}" + { + eval "$generator 12>/dev/fd/12 | $filter" + report=$(cat<&12) + } 12<<EOF +EOF + wait +} + + +function short_report { + local generator="$1" + local filter="${2:-print_table | decorate_table}" + + configure_palete "$palete" + simple_report "$generator" "$filter" + finish +} + + +function standart_report { + local generator="$1" + local filter="${2:-print_table | decorate_table}" + + configure_palete "$palete" + simple_report "$generator" "$filter" + + print "$report" + echo + finish +} + + +function report { + # Eval breaks output for some reason + case $report in + simple_report) + simple_report "$@" + ;; + short_report) + short_report "$@" + ;; + standart_report) + standart_report "$@" + ;; + default) + echo "Report is not implemented" + esac +} diff --git a/lib/remote/run.sh b/lib/remote/run.sh new file mode 100644 index 0000000..fdd7311 --- /dev/null +++ b/lib/remote/run.sh @@ -0,0 +1,13 @@ +function run_ { + local output=$(eval source "$@") + + flock -x $0 echo -e "$output" +} + +function run { + if [ $parallel -gt 0 ]; then + run_ "$@" & + else + run_ "$@" + fi +} diff --git a/lib/remote/status.sh b/lib/remote/status.sh new file mode 100644 index 0000000..7bef6c8 --- /dev/null +++ b/lib/remote/status.sh @@ -0,0 +1,196 @@ +. setup/remote.sh +. lib/remote/print.sh +. lib/remote/report.sh +. lib/remote/run.sh + +function check_server_traffic { + local result="$1" + local yellow=${2:-0} + local red=${3:-0} + + local name=("in" "out" "fwd") + + local traf + read -ra traf <<< $result + if [ ${#traf[@]} -ne 3 ]; then + [ -z "$result" ] && result="No info received" + print "$result" "red" $fold 1 + return + fi + + local output="" + for i in $(seq 0 2); do + if ! [[ ${traf[$i]} =~ ^[0-9]+$ ]]; then + print "$result" "red" $fold 1 + return + fi + + local val=$(printf "% 4u" $((${traf[$i]} / 1024 / 1024 / 1024))) + if [ $red -gt 0 -a $val -gt $red ]; then + val=$(decorate "$val GB" "red") + elif [ $yellow -gt 0 -a $val -gt $yellow ]; then + val=$(decorate "$val GB" "yellow") + else + val="$val GB" + fi + +# output="$output${output:+, }${name[$i]}: $val" + output="$output${output:+, } $val" + done + +# print "$output" + print "in/out/fwd: $output" +} + +function check_server_ { + local service=$1 && shift + + local hopo + IFS=':' read -ra hopo <<< $1 && shift + local host=${hopo[0]} + local port=${hopo[1]:-22} + + local result=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_$service.sh 2>&1 ) + + case "$service" in + 'traffic') + check_server_traffic "$result" "$@" + ;; + *) + print "$result" + esac + +} + + +function check_server_status { + local hopo + IFS=':' read -ra hopo <<< $1 && shift + local host=${hopo[0]} + local port=${hopo[1]:-22} + local services="$@" + + local ping_output=$(scripts/ping.pl $host $port $timeout) + print_status "$ping_output" + + if [ "$services" == "-" ]; then return; fi + + if [ "$ping_output" == "1" ]; then + local output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_status.sh 2>&1) + + local status=1 + [ -n "$output" ] && status=0 && output="\n$(decorate "$host:$port" "cyan" "u")\n$output\n" + print_status $status + + # Check Additional services, error is reported if anything provides information + local headers="" + if [ -n "$services" ]; then + for service in "$services"; do + if [[ "$service" =~ ^vpn/(.*)$ ]]; then + local ip=${BASH_REMATCH[1]} + [ $(hostname) == "styx" ] && check_server_status "$ip" "-" + else + local service_status=1 + local service_output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_${service}_status.sh 2>&1 | sed 's/^[[:space:]]*//;/^$/d') + if [ -n "$service_output" ]; then + first_line=$(echo "$service_output" | head -n 1) + if [[ "$first_line" =~ ^([0-9]+)[[:space:]](.*)$ ]]; then + service_status=${BASH_REMATCH[1]} + service_output=$(sed "1 d" <<< "$service_output") + if [ -n "${BASH_REMATCH[2]}" ]; then + service_header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "${BASH_REMATCH[2]}") #" + headers="$headers $service_header" + fi + else + service_status=0 + fi + + if [ -n "$service_output" ]; then + [ -z "$output" ] && output="\n$(decorate "$host:$port" "cyan" "u")" + output="${output}\n $(decorate "$service" "gray" "u")\n$(echo ${service_output} | sed 's/^/ /')" + fi + fi + fi + + done + fi + + print " ::: ${headers}" + #report="$report<section>$output" + if [ -n "$output" ]; then + flock -x $0 echo "${output}" >&12 + fi + else + print_status "x" + for service in "$services"; do + print_status "x" + done + fi +} + +function check_service { + local service="$1" && shift + local id="$1" && shift + local args="$@" + + local output=$(service/check_${service}.sh ${id} "$@" 2>&1 | sed 's/^[[:space:]]*//;/^$/d') + + local info=$(echo "$output" | tail -n 1) + + local online=0 + local health=0 + local header="" + if [[ "$info" =~ ^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+(.*)$ ]]; then + online=${BASH_REMATCH[1]} + health=${BASH_REMATCH[2]} + header=${BASH_REMATCH[3]} + + output=$(sed '$ d' <<< "$output") + fi + + print_status "$online" + print_status "$health" + + header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "$header") # " + print " ::: ${header}" + + + important=$(grep "^\*" <<< "$output") + messages=$(grep -v "^\*" <<< "$output") + + if [ -n "$output" ]; then + output="\n$(decorate "$service:$id" "cyan" "u")\n$important\n$(set_color gray)$messages$(reset_color)\n" + flock -x $0 echo "${output}" >&12 + fi +} + + +function check__ { + local args + local title="$1" && shift + read -ra args <<< "$1" && shift + local host=${args[0]}; + + print "$(decorate $title "purple") ::: " | sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g; " + + local service + for service in "$@"; do + service=$(sed -r "s/<([0-9]+)>/\${args[\\1]}/g; s/<([^>]+)>/\${\\1}/g" <<< "$service") # " + ( eval "$service" ) + done + print_eol +} + +function check_ { + # Buffer the output + local output=$(check__ "$@") + echo -e "$output" +} + +function check { + if [ $parallel -gt 0 ]; then + check_ "$@" & + else + check_ "$@" + fi +} |