summaryrefslogtreecommitdiffstats
path: root/service
diff options
context:
space:
mode:
Diffstat (limited to 'service')
-rw-r--r--service/README11
-rwxr-xr-xservice/check_adei.sh115
-rwxr-xr-xservice/check_gluster.sh83
-rwxr-xr-xservice/check_kaas.sh37
-rw-r--r--service/opts.sh22
5 files changed, 268 insertions, 0 deletions
diff --git a/service/README b/service/README
new file mode 100644
index 0000000..51d5122
--- /dev/null
+++ b/service/README
@@ -0,0 +1,11 @@
+* Message (important or local service messages)
+Status messages to be printed bellow all services
+
+Online Healthy [Short Message]
+
+
+Statuses
+========
+ 0 - Bad
+ 1 - Good
+ * - Intermediate
diff --git a/service/check_adei.sh b/service/check_adei.sh
new file mode 100755
index 0000000..f08ab5f
--- /dev/null
+++ b/service/check_adei.sh
@@ -0,0 +1,115 @@
+#! /bin/bash
+
+MAX_SOURCE_OFFSET=3600
+MAX_MASTER_OFFSET=300
+MAX_SLAVE_OFFSET=300
+
+function query {
+ if [ "$2" == "ecode" ]; then
+ url="$1&mysql=master"
+ resp=$(curl -f "$url" 2>&1 | grep "returned error")
+ else
+ if [ "$2" == "slave" ]; then
+ url="$1&mysql=slave"
+ else
+ url="$1&mysql=master"
+ fi
+
+ resp=$(curl -sf "$url")
+ err=$?
+ [ $err -eq 0 ] || resp=""
+ fi
+ echo $resp
+}
+
+function format_time {
+ offset=$1
+ if [ $offset -ge 86400 ]; then
+ echo "$((offset / 86400))d"
+ elif [ $offset -ge 3600 ]; then
+ echo "$((offset / 3600))h"
+ elif [ $offset -ge 60 ]; then
+ echo "$((offset / 60))m"
+ else
+ echo "${offset}s"
+ fi
+}
+
+
+cd "$(dirname "$0")"
+id=$1
+
+auth=$(cat ../security/adei.txt | grep -P "^$id" | awk '{ print $2 }')
+[ -n $auth ] && auth="$auth@"
+
+host=$(echo $2 | cut -d '/' -f 1)
+url="http://$auth$2/services"
+
+src=$3
+
+# Check if online
+online=$(../scripts/ping.pl "$host")
+healthy=$online
+
+# Check if healthy and find version
+version=$(query "$url/info.php?target=version&encoding=text")
+if [ -z "$version" ]; then
+ err=$(query "$url/info.php?target=version&encoding=text" "ecode")
+ echo $err
+ healthy=0
+else
+ if [[ "$version" =~ "Error:" ]]; then
+ echo $version
+ healthy=0
+ version=""
+ else
+ version="ADEI $version"
+ fi
+fi
+
+# Get current database size
+size=$(query "$url/info.php?target=size&encoding=text")
+[[ "$size" =~ "Error:" ]] && size=""
+[ -n "$size" ] && msg="\${color gray}/ $((size / 1024 / 1024 / 1024)) GB"
+
+
+# Verify offset (for selected database)
+if [ $healthy -ne 0 -a -n "$src" ]; then
+ now=$(date +%s)
+ sdate=$(query "$url/getdata.php?$src&db_mask=0&format=csv&window=-1&rt=1&cache=1&time_format=U" slave | cut -d ',' -f 1 | cut -d '.' -f 1)
+ mdate=$(query "$url/getdata.php?$src&db_mask=0&format=csv&window=-1&rt=1&cache=1&time_format=U" | cut -d ',' -f 1 | cut -d '.' -f 1)
+
+ if [ -z "$sdate" -o -z "$mdate" ]; then
+ echo "Error querying data from '$src'"
+ healthy=2
+ else
+ master_offset=$(($now - $mdate))
+ slave_offset=$(($mdate - $sdate))
+
+ if [ $master_offset -gt $MAX_SOURCE_OFFSET ]; then
+ rdate=$(query "$url/getdata.php?$src&db_mask=0&format=csv&window=-1&rt=1&time_format=U" | cut -d ',' -f 1 | cut -d '.' -f 1)
+ cache_offset=$(($rdate - $mdate))
+ if [ $cache_offset -gt $MAX_MASTER_OFFSET ]; then
+ echo "ADEI cache is off by $(format_time $cache_offset) for '$src'"
+ healthy=2
+ else
+ offset=$(($now - $rdate))
+# echo "Source '$src' is not updated for $(format_time $offset)"
+ fi
+ fi
+
+ if [ $slave_offset -gt $MAX_SLAVE_OFFSET ]; then
+ echo "MySQL slave is off by $(format_time $slave_offset) for '$src'"
+ healthy=2
+ fi
+
+ [ -n "$msg" ] && msg="${msg}, "
+ [ -z "$msg" ] && msg="\${color gray}/ "
+ msg="${msg}cache $(format_time $master_offset)"
+ [ $slave_offset -gt 0 ] && msg="$msg & slave $(format_time $slave_offset)"
+ msg="$msg off"
+ fi
+
+fi
+
+echo "$online $healthy $version $msg"
diff --git a/service/check_gluster.sh b/service/check_gluster.sh
new file mode 100755
index 0000000..e591e7e
--- /dev/null
+++ b/service/check_gluster.sh
@@ -0,0 +1,83 @@
+#! /bin/bash
+
+cd "$(dirname "$0")"
+. opts.sh
+
+if [ -f "../security/$host.kubeconfig" ]; then
+ gpod=$(get_gluster_pod)
+
+ function gluster {
+ oc -n glusterfs rsh po/$gpod gluster "$@"
+ }
+
+# check if gluster pods are running
+ if [ -n "$gpod" ]; then
+ online=1
+ else
+ oc -n glusterfs get pods -l 'glusterfs=storage-pod' | sed 's/^/* /'
+ online=0
+ fi
+else
+ echo "0 0 Not supported"
+ exit
+fi
+
+function check {
+ vol=$1
+ vol_bricks=$(gluster volume info "$vol" | grep -P 'Number of Bricks' | awk '{ print $NF }' | tr -d '\r\n')
+ vol_online=$(gluster volume status "$vol" detail | grep Online | grep Y | wc -l)
+
+ if [ -z "$vol_bricks" -o -z "$vol_online" -o "$vol_bricks" -ne "$vol_online" ]; then
+ vol_status=$(gluster volume info "$vol" | grep -P 'Status' | awk '{ print $2 }' | tr -d '\r\n')
+ vol_avail=$(gluster volume status "$vol" detail | grep Brick | wc -l)
+ echo "* Volume $vol: $vol_status (Bricks: $vol_bricks, Available: $vol_avail, Online: $vol_online)"
+
+ if [ "$vol_status" == "Started" -a "$vol_online" -ge 0 ]; then
+ return 2
+ else
+ return 0
+ fi
+ else
+ return 1
+ fi
+}
+
+version=$(gluster --version | head -n 1 | awk '{ print $2 }' | tr -d '\r')
+if [ -z "$version" ]; then
+ online=0
+else
+ version="GlusterFS $version"
+fi
+
+volumes=0
+partial=0
+failed=0
+healthy=$online
+
+if [ $online -eq 1 ]; then
+ vols=$(gluster volume info | grep -P '^Volume Name' | awk '{ print $NF }' | tr '\r\n' ' ')
+ for vol in $vols; do
+ [[ "$vol" =~ [0-9] ]] && continue
+ [[ "$vol" =~ ^vol_ ]] && continue
+ [[ "$vol" =~ ^heketi ]] && continue
+ check ${vol}
+ ret=$?
+
+ volumes=$((volumes + 1))
+ if [ $ret -eq 0 ]; then
+ healthy=0
+ failed=$((failed + 1))
+ elif [ $ret -ne 1 ]; then
+ [ $healthy -gt 0 ] && healthy=$ret
+ partial=$((partial + 1))
+ fi
+ done
+fi
+
+if [ $healthy -eq 1 ]; then
+ msg="\${color gray}/ $volumes volumes"
+else
+ msg="\${color gray}/ $volumes volumes, $failed failed, $partial bricks missing"
+fi
+
+echo "$online $healthy $version $msg"
diff --git a/service/check_kaas.sh b/service/check_kaas.sh
new file mode 100755
index 0000000..d6e7300
--- /dev/null
+++ b/service/check_kaas.sh
@@ -0,0 +1,37 @@
+#! /bin/bash
+
+cd "$(dirname "$0")"
+. opts.sh
+
+e_nodes=$2
+
+
+online=$(../scripts/ping.pl "$host")
+healthy=$online
+
+version=$(oc version | head -n 1 | awk '{ print $2 }')
+if [ -z "$version" ]; then
+ healthy=0
+else
+ version="OpenShift $version"
+fi
+
+etcd=$(oc get cs | grep etcd | grep "Healthy" | wc -l)
+if [ -z "$etcd" -o "$etcd" -lt 3 ]; then
+ healthy=2
+ oc get cs | grep etcd | grep "Healthy" | sed -r -e 's/\s+/ /g' | awk '{ print $1, $2 }' | sed 's/^/* /'
+fi
+
+if [ $healthy -ne 0 ]; then
+ nodes=$(oc get nodes | grep Ready | wc -l)
+ if [ $nodes -ge $e_nodes ]; then
+ nodes=" / \${color gray}$etcd etcd, $nodes nodes"
+ else
+ offline=$(oc get nodes | grep -v "STATUS" | grep -v "Ready" | wc -l)
+ nodes=" / \${color gray}$etcd etcd, $nodes ready, $offline offline"
+ healthy=2
+ oc get nodes | grep -v "STATUS" | grep -v "Ready" | awk '{ print $1, $2 }' | sed 's/^/* /'
+ fi
+fi
+
+echo "$online $healthy $version $nodes"
diff --git a/service/opts.sh b/service/opts.sh
new file mode 100644
index 0000000..3628708
--- /dev/null
+++ b/service/opts.sh
@@ -0,0 +1,22 @@
+host=$1
+
+function oc {
+ cfg="../security/$host.kubeconfig"
+ /usr/bin/oc --config "$cfg" "$@"
+}
+
+function node {
+ ip=$1
+ shift
+
+ ssh -xq root@192.168.26.$ip "$@"
+}
+
+function get_gluster_pod {
+ oc -n glusterfs get pods -l 'glusterfs=storage-pod' | grep Running | awk '{ print $1 }' | head -n 1
+}
+
+function heketi {
+ node 1 heketi-cli -s http://heketi-storage.glusterfs.svc.cluster.local:8080 --user admin --secret "$(oc get secret heketi-storage-admin-secret -n glusterfs -o jsonpath='{.data.key}' | base64 -d)" "$@"
+}
+