blob: d668287db9a298a3535da85d80d72305383cada8 [file] [log] [blame]
tom82d6bde2014-09-23 17:33:58 -07001#!/bin/bash
Pavlin Radoslavov91413792014-10-15 11:00:32 -07002# -----------------------------------------------------------------------------
tom82d6bde2014-09-23 17:33:58 -07003# Checks the logs of the remote ONOS instance and makes sure they are clean.
Pavlin Radoslavov91413792014-10-15 11:00:32 -07004# -----------------------------------------------------------------------------
tom82d6bde2014-09-23 17:33:58 -07005
Ayaka Koshibebad156d2015-09-18 17:30:14 -07006function __usage() {
7cat << _EOM_
Thomas Vachuska27ece8f2016-11-11 10:03:11 -08008usage: $(basename $0) [--old|--ignore-store-exceptions] [node]
Ayaka Koshibebad156d2015-09-18 17:30:14 -07009
10options:
Thomas Vachuska27ece8f2016-11-11 10:03:11 -080011 --ignore-store-exceptions
12 if specified, any store service exceptions are ignored
13 --old
14 if specified, the entire logs are searched for errors and exceptions;
15 otherwise logs are scanned only from the last server start-up
16
17 node the cluster node whose logs to inspect; default is \$OCI
Ayaka Koshibebad156d2015-09-18 17:30:14 -070018
19summary:
20 Checks the logs of the remote ONOS instance and makes sure they are clean.
21
22_EOM_
23}
24
tom82d6bde2014-09-23 17:33:58 -070025[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
26. $ONOS_ROOT/tools/build/envDefaults
27
Thomas Vachuska27ece8f2016-11-11 10:03:11 -080028# Process options
29while [ -z $opts ] ; do
30 case "$1" in
31 --ignore-store-exceptions) export ise=1; shift;;
32 --old) export allLogs=1; shift;;
33 --*) __usage && exit 1;;
34 -h) __usage && exit 0;;
35 *) export opts=1;;
36 esac
37done
38
tom82d6bde2014-09-23 17:33:58 -070039remote=$ONOS_USER@${1:-$OCI}
40
Thomas Vachuska77a9a442015-05-19 17:35:52 -070041LOG=$ONOS_INSTALL_DIR/log/karaf.log*
tom82d6bde2014-09-23 17:33:58 -070042
Thomas Vachuska77a9a442015-05-19 17:35:52 -070043aux=/tmp/log.$$
44
Thomas Vachuska27ece8f2016-11-11 10:03:11 -080045if [ -n "$allLogs" ]; then
Thomas Vachuska77a9a442015-05-19 17:35:52 -070046 ssh $remote "egrep 'ERROR|Exception|Error' $LOG"
47
48else
49 ssh $remote "
Ayaka Koshibebad156d2015-09-18 17:30:14 -070050 [ "'`uname`'" != "'"Linux"'" ] && alias tac='tail -r'
Thomas Vachuska27ece8f2016-11-11 10:03:11 -080051 tac $LOG | tr -d '\000' | awk -v ignoreStoreExceptions=${ise:-0} '
Yuta HIGUCHIdf5eeb12017-05-05 21:19:02 -070052 BEGIN { off = 0; fail = 0; exclusion = 0; expected = 0; trace = \"\";}
Thomas Vachuska77a9a442015-05-19 17:35:52 -070053 / org.apache.karaf.main.lock.SimpleFileLock lock/ {
Thomas Vachuska77a9a442015-05-19 17:35:52 -070054 exit fail;
55 }
56
Yuta HIGUCHIdf5eeb12017-05-05 21:19:02 -070057 # note: we are parsing log in reverse from the end
58 / EXPECTING_EXCEPTION_BEGIN/ {
59 print \$0;
60 expected = 0;
61 }
62
63 / EXPECTING_EXCEPTION_END/ {
64 print \$0;
65 expected = 1;
66 }
67
Thomas Vachuska77a9a442015-05-19 17:35:52 -070068 / ERROR / {
Thomas Vachuska128cffb2016-10-24 13:04:46 -070069 if (exception && !exclusion) {
70 print \$0;
71 exception = 0;
Yuta HIGUCHIdf5eeb12017-05-05 21:19:02 -070072
73 if (!expected) {
74 fail = 1;
75 }
Thomas Vachuska128cffb2016-10-24 13:04:46 -070076 }
77 exclusion = 0;
Thomas Vachuska77a9a442015-05-19 17:35:52 -070078 }
79 / WARN / {
Thomas Vachuska128cffb2016-10-24 13:04:46 -070080 if (exception && !exclusion) {
Thomas Vachuska77a9a442015-05-19 17:35:52 -070081 print \$0;
Thomas Vachuska128cffb2016-10-24 13:04:46 -070082 exception = 0;
Thomas Vachuska77a9a442015-05-19 17:35:52 -070083 }
Thomas Vachuska128cffb2016-10-24 13:04:46 -070084 exclusion = 0;
Thomas Vachuska77a9a442015-05-19 17:35:52 -070085 }
Thomas Vachuska2ec5aa92016-08-31 13:30:31 -070086 / INFO / {
Thomas Vachuska128cffb2016-10-24 13:04:46 -070087 exception = 0;
88 exclusion = 0;
Thomas Vachuska2ec5aa92016-08-31 13:30:31 -070089 }
Thomas Vachuska77a9a442015-05-19 17:35:52 -070090
Thomas Vachuska27ece8f2016-11-11 10:03:11 -080091 # Sanctioned exclusions for exceptions in the distributed stores; one pattern per exclusion
92 /org\.onosproject\.store\.service\..*Exception/ { exclusion = ignoreStoreExceptions; }
93
Thomas Vachuska77a9a442015-05-19 17:35:52 -070094 /^[a-zA-Z0-9.]*(Exception|Error)/ {
Thomas Vachuska128cffb2016-10-24 13:04:46 -070095 if (!exclusion) {
96 print trace;
97 print \$0;
98 exception = 1;
Yuta HIGUCHIdf5eeb12017-05-05 21:19:02 -070099 if (!expected) {
100 fail = 1;
101 }
Thomas Vachuska128cffb2016-10-24 13:04:46 -0700102 trace = \"\";
103 }
Thomas Vachuska77a9a442015-05-19 17:35:52 -0700104 }
105
Thomas Vachuska2ec5aa92016-08-31 13:30:31 -0700106 /( at|Caused by:) / {
Thomas Vachuska128cffb2016-10-24 13:04:46 -0700107 trace = trace \"\n\" \$0;
Thomas Vachuska77a9a442015-05-19 17:35:52 -0700108 }
Thomas Vachuska128cffb2016-10-24 13:04:46 -0700109
110 # Sanctioned exclusions for exceptions in third-party code; one pattern per exclusion
Thomas Vachuska09fee5f2018-10-31 15:23:20 -0700111 /at org\.apache\.felix\.scr\.impl\.ComponentRegistry\.getComponentHolders\(ComponentRegistry\.java:356\)/ { exclusion = 1; }
Thomas Vachuska128cffb2016-10-24 13:04:46 -0700112 /at org\.apache\.karaf\.service\.guard\.impl\.GuardProxyCatalog.1.run\(GuardProxyCatalog\.java:253\)/ { exclusion = 1; }
Ray Milkeyc7e31632019-01-02 20:33:34 +0000113 /at org\.apache\.sshd\.server\.SshServer\.start/ { exclusion = 1; }
Thomas Vachuska128cffb2016-10-24 13:04:46 -0700114
Thomas Vachuska77a9a442015-05-19 17:35:52 -0700115 END { exit fail; }
116 ' > $aux
117 status=\$?
118 tac $aux && rm $aux
119 exit \$status
120 "
121fi