You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
57 lines
1.4 KiB
57 lines
1.4 KiB
#!/usr/bin/env bash
|
|
|
|
# Pagerduty constants
|
|
LOGFILE="/home/tmp_log/log-20190628.153354/*.log"
|
|
LOG=$(tail -n 1000 $logfile)
|
|
SHARD_NUM=$(echo "$log" | grep -oE -m 1 "Shard\":.?" | rev | cut -c 1)
|
|
IP=$(dig -4 @resolver1.opendns.com ANY myip.opendns.com +short)
|
|
KEY="5d11f6173899423689b6fc98691930de"
|
|
TRIGGER="\
|
|
{\
|
|
\"routing_key\": \"$KEY\",\
|
|
\"event_action\": \"trigger\",\
|
|
\"dedup_key\": \"$IP\",\
|
|
\"payload\": {\
|
|
\"summary\": \"Node NOT validating: Shard-$SHARD_NUM: $IP\",\
|
|
\"source\": \"$IP\",\
|
|
\"severity\": \"critical\"\
|
|
}\
|
|
}"
|
|
RESOLVE="\
|
|
{\
|
|
\"routing_key\": \"$KEY\",\
|
|
\"event_action\": \"resolve\",\
|
|
\"dedup_key\": \"$IP\"\
|
|
}"
|
|
JSON_HEADER="Content-Type: application/json"
|
|
POSTURL="https://events.pagerduty.com/v2/enqueue"
|
|
|
|
# Validation check constants
|
|
if [[ -f "triggered" ]]
|
|
then
|
|
triggered=$(< triggered)
|
|
else
|
|
triggered=false
|
|
fi
|
|
validation=$(tail -n 1000 $LOGFILE | tac | grep -ai -m 1 "bingo\|hooray")
|
|
timestamp=$(echo $validation | cut -f 3 -d 't' | cut -c 4-22 | tr T \ )
|
|
latest=$(date -d "$timestamp" +%s)
|
|
curtime=$(date +%s)
|
|
delay=60
|
|
|
|
# Pagerduty alert
|
|
if [[ $(($curtime - $latest)) -gt $delay ]]
|
|
then
|
|
cat "$TRIGGER"
|
|
curl -X POST -H "$JSON_HEADER" -d "$TRIGGER" "$POSTURL"
|
|
echo
|
|
triggered=true
|
|
else
|
|
if [[ $triggered = true ]]
|
|
then
|
|
curl -X POST -H "$JSON_HEADER" -d "$RESOLVE" "$POSTURL"
|
|
echo
|
|
triggered=false
|
|
fi
|
|
fi
|
|
echo "$triggered" > triggered
|
|
|