summaryrefslogtreecommitdiffstats
path: root/service/check_adei.sh
blob: 4edad93fc4211385c94f7f129ad6860f20c7f273 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
#! /bin/bash

MAX_SOURCE_OFFSET=3600
MAX_MASTER_OFFSET=300
MAX_SLAVE_OFFSET=300

function query {
    if [ "$2" == "ecode" ]; then
        url="$1&mysql=master"
	resp=$(curl -f "$url" 2>&1 | grep "returned error")
    else
        if [ "$2" == "slave" ]; then
            url="$1&mysql=slave"
        else
            url="$1&mysql=master"
        fi
        
	resp=$(curl -sf "$url")
	err=$?
	[ $err -eq 0 ] || resp=""
    fi
    echo $resp
}

function format_time {
    offset=$1
    if [ $offset -ge 86400 ]; then
        echo "$((offset / 86400))d"
    elif [ $offset -ge 3600 ]; then
        echo "$((offset / 3600))h"
    elif [ $offset -ge 60 ]; then
        echo "$((offset / 60))m"
    else
        echo "${offset}s"
    fi
}


cd "$(dirname "$0")"
id=$1

auth=$(cat ../security/adei.txt  | grep -P "^$id" | awk '{ print $2 }')
[ -n $auth ] && auth="$auth@"

host=$(echo $2 | cut -d '/' -f 1)
url="http://$auth$2/services"

src=$3

# Check if online
online=$(../scripts/ping.pl "$host")
healthy=$online

# Check if healthy and find version
version=$(query "$url/info.php?target=version&encoding=text")
if [ -z "$version" ]; then
    err=$(query "$url/info.php?target=version&encoding=text" "ecode")
    echo $err
    healthy=0
else
    if [[ "$version" =~ "Error:" ]]; then
	echo $version
	healthy=0
	version=""
    else
	version="ADEI $version"
    fi
fi

# Get current database size
size=$(query "$url/info.php?target=size&encoding=text")
[[ "$size" =~ "Error:" ]] && size=""
[ -n "$size" ] && msg="\${color gray}/ $((size / 1024 / 1024 / 1024)) GB"

# Check pending administrative scripts
if [ $healthy -ne 0 ]; then
    scripts=$(query "$url/info.php?target=scripts")
    waiting=$(echo $scripts | xmllint --format - | grep "Value" | sed -e "s/^\(.*mtime=\"\([^\"]*\)\".*\)$/\\2\\1/" | awk -v date="$(date +%s)" '{duration=date - $1} duration > 3600 { print duration }' | sort -rn)
    num_waiting=$(echo $waiting | tr ' ' '\n' | wc -l)
    long_waiting=$(echo $waiting | cut -d ' ' -f 1)
    if [ $num_waiting -gt 0 ]; then
        healthy=2
        echo "$num_waiting pending scripts, longest for $(format_time $long_waiting)"
    fi
fi


# Verify offset (for selected database)
if [ $healthy -ne 0 -a -n "$src" ]; then
    now=$(date +%s)
    sdate=$(query "$url/getdata.php?$src&db_mask=0&format=csv&window=-1&rt=1&cache=1&time_format=U" slave | cut -d ',' -f 1 | cut -d '.' -f 1)
    mdate=$(query "$url/getdata.php?$src&db_mask=0&format=csv&window=-1&rt=1&cache=1&time_format=U" | cut -d ',' -f 1 | cut -d '.' -f 1)

    if [ -z "$sdate" -o -z "$mdate" ]; then
        echo "Error querying data from '$src'"
        healthy=2
    else
        master_offset=$(($now - $mdate))
        slave_offset=$(($mdate - $sdate))
    
        if [ $master_offset -gt $MAX_SOURCE_OFFSET ]; then
            rdate=$(query "$url/getdata.php?$src&db_mask=0&format=csv&window=-1&rt=1&time_format=U" | cut -d ',' -f 1 | cut -d '.' -f 1)
            cache_offset=$(($rdate - $mdate))
            if [ $cache_offset -gt $MAX_MASTER_OFFSET ]; then
                echo "ADEI cache is off by $(format_time $cache_offset) for '$src'"
                healthy=2
            else
                offset=$(($now - $rdate))
#                echo "Source '$src' is not updated for $(format_time $offset)"
            fi
        fi
        
        if [ $slave_offset -gt $MAX_SLAVE_OFFSET ]; then
            echo "MySQL slave is off by $(format_time $slave_offset) for '$src'"
            healthy=2
        fi

        [ -n "$msg" ] && msg="${msg}, "
        [ -z "$msg" ] && msg="\${color gray}/ "
        msg="${msg}cache $(format_time $master_offset)"
        [ $slave_offset -gt 0 ] && msg="$msg & slave $(format_time $slave_offset)"
        msg="$msg off"
    fi

fi

echo "$online $healthy $version $msg"