Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | |
| 3 | # |
| 4 | # Copyright 2015-present Open Networking Foundation |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
| 17 | # |
| 18 | |
| 19 | # ----------------------------------------------------------------------------- |
| 20 | # Tool to collect cluster-wide diagnostics into a single tar stream. |
| 21 | # ----------------------------------------------------------------------------- |
| 22 | function usage() { |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 23 | echo "usage: $(basename $0) [-x] [-j] [-n name] [-P port] [-u user] [-p password] [ip1 ip2...]" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 24 | echo "" |
| 25 | echo "Environment Variables:" |
| 26 | echo " ONOS_INSTANCES IPs or hostnames of ONOS cluster machines" |
| 27 | echo " ONOS_WEB_USER username for REST API" |
| 28 | echo " ONOS_WEB_PASS password for REST API" |
| 29 | echo "" |
| 30 | echo "Example Usages:" |
| 31 | echo " # Collect compressed diagnostics for the cluster." |
| 32 | echo " # REST API user and password are drawn from environment variables." |
| 33 | echo " # Collection archive will be named /tmp/onos-diags.tar.gz" |
| 34 | echo " # The cluster node IPs will be drawn from ONOS_INSTANCES variable." |
| 35 | echo " > $(basename $0) " |
| 36 | echo "" |
| 37 | echo " # Collect diagnostics for the cluster and leave them extracted. " |
| 38 | echo " # Collection directory will be named /tmp/prague-diags/" |
| 39 | echo " # Collection archive will be named /tmp/prague-diags.tar.gz." |
| 40 | echo " # REST API user name is 'onos' and password is 'rules'." |
| 41 | echo " # The cluster node IPs will be drawn from ONOS_INSTANCES variable." |
| 42 | echo " > $(basename $0) -x -n prague -u onos -p rules" |
| 43 | echo "" |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 44 | echo " # Collect diagnostics for the cluster and store them in JSON files." |
| 45 | echo " # JSON_CLI_COMMANDS below lists JSON-supported diagnostics commands." |
| 46 | echo " # Collection archive will be named /tmp/onos-diags.tar.gz" |
| 47 | echo " > $(basename $0) -j" |
| 48 | echo "" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 49 | echo " # Collect compressed diagnostics for a cluster." |
| 50 | echo " # REST API user name is 'onos' and password is 'rules'." |
| 51 | echo " # Collection archive will be named /tmp/onos-diags.tar.gz" |
| 52 | echo " # The cluster node IPs are listed explicitly." |
| 53 | echo " > $(basename $0) -u onos -p rules 172.17.0.11 172.17.0.12 172.17.0.13" |
| 54 | |
| 55 | exit 1 |
| 56 | } |
| 57 | |
Thomas Vachuska | a7be50d | 2018-04-16 14:02:18 -0700 | [diff] [blame] | 58 | ONOS_WEB_USER=${ONOS_WEB_USER:-onos} # ONOS WEB User defaults to 'onos' |
| 59 | ONOS_WEB_PASS=${ONOS_WEB_PASS:-rocks} # ONOS WEB Password defaults to 'rocks' |
| 60 | ONOS_WEB_PORT=${ONOS_WEB_PORT:-8181} # REST API port defaults to '8181' |
| 61 | |
| 62 | . $(dirname $0)/_find-node |
| 63 | |
Charles Chan | c6576fa | 2018-03-08 14:59:42 -0800 | [diff] [blame] | 64 | # TODO We should make SR commands optional |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 65 | CLI_COMMANDS=( |
Thomas Vachuska | a40aa0b | 2018-02-01 15:52:10 -0800 | [diff] [blame] | 66 | "feature:repo-list" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 67 | "feature:list" |
| 68 | "bundle:list" |
Charles Chan | dc77af5 | 2019-02-12 17:47:41 -0800 | [diff] [blame] | 69 | "scr-list" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 70 | |
| 71 | "summary" |
| 72 | "nodes" |
| 73 | "apps -s" |
| 74 | "netcfg" |
| 75 | "cfg get" |
| 76 | |
| 77 | "devices" |
Seyeon Jeong | 1e24910 | 2020-03-10 17:41:14 -0700 | [diff] [blame] | 78 | "device-drivers" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 79 | "links" |
| 80 | "hosts" |
Pier | 4099f05 | 2018-04-05 20:45:40 +0200 | [diff] [blame] | 81 | "interfaces" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 82 | |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 83 | "ports" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 84 | "portstats -nz" |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 85 | "edge-ports" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 86 | |
Thomas Vachuska | 7f2a356 | 2018-02-28 10:02:16 -0800 | [diff] [blame] | 87 | "packet-processors" |
| 88 | "packet-requests" |
| 89 | |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 90 | "intents" |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 91 | "flows" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 92 | "groups" |
| 93 | |
| 94 | "roles" |
| 95 | "masters" |
| 96 | |
Pier | b185858 | 2018-05-02 13:31:12 +0200 | [diff] [blame] | 97 | "maps" |
| 98 | |
Charles Chan | c6576fa | 2018-03-08 14:59:42 -0800 | [diff] [blame] | 99 | "fpm-connections" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 100 | "routes" |
| 101 | "obj-next-ids" |
| 102 | "obj-pending-nexts" |
Charles Chan | 8b7df41 | 2018-04-24 10:23:21 -0700 | [diff] [blame] | 103 | "obj-queues" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 104 | |
Charles Chan | c6576fa | 2018-03-08 14:59:42 -0800 | [diff] [blame] | 105 | "sr-device-subnets" |
| 106 | "sr-ecmp-spg" |
Charles Chan | 8bc75ee | 2018-04-17 18:56:53 -0700 | [diff] [blame] | 107 | "sr-should-program" |
Charles Chan | c6576fa | 2018-03-08 14:59:42 -0800 | [diff] [blame] | 108 | "sr-link-state" |
Charles Chan | c6576fa | 2018-03-08 14:59:42 -0800 | [diff] [blame] | 109 | "sr-mcast-tree" |
Pier | 954b368 | 2018-04-18 08:50:29 +0200 | [diff] [blame] | 110 | "sr-mcast-leader" |
Pier | b185858 | 2018-05-02 13:31:12 +0200 | [diff] [blame] | 111 | "sr-mcast-role" |
Andreas Pantelopoulos | ff691b7 | 2018-03-12 16:30:20 -0700 | [diff] [blame] | 112 | "sr-pw-list" |
Charles Chan | 3fe60be | 2018-10-26 15:54:32 -0700 | [diff] [blame] | 113 | "sr-next-mcast" |
Harshada Chaundkar | 72f3b9d | 2019-07-02 16:01:24 +0000 | [diff] [blame] | 114 | "sr-filt-mcast" |
Charles Chan | 3fe60be | 2018-10-26 15:54:32 -0700 | [diff] [blame] | 115 | "sr-next-dst" |
| 116 | "sr-next-port" |
| 117 | "sr-next-vlan" |
| 118 | "sr-next-pw" |
| 119 | "sr-next-xconnect" |
Ruchi Sahota | ef0761c | 2019-01-28 01:08:18 +0000 | [diff] [blame] | 120 | "sr-next-mac-vlan" |
Charles Chan | 4d73add | 2020-02-14 13:23:57 -0800 | [diff] [blame^] | 121 | "sr-pr-list" |
Charles Chan | c6576fa | 2018-03-08 14:59:42 -0800 | [diff] [blame] | 122 | "dhcp-relay" |
Pier | 4099f05 | 2018-04-05 20:45:40 +0200 | [diff] [blame] | 123 | |
| 124 | "mcast-host-routes" |
| 125 | "mcast-host-show" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 126 | ) |
| 127 | |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 128 | JSON_CLI_COMMANDS=( |
| 129 | "netcfg -j" |
| 130 | "devices -j" |
| 131 | "device-drivers -j" |
| 132 | "links -j" |
| 133 | "hosts -j" |
| 134 | "ports -j" |
| 135 | "edge-ports -j" |
| 136 | "flows -j" |
| 137 | "groups -j" |
| 138 | "masters -j" |
| 139 | "routes -j" |
| 140 | "mcast-host-show -j" |
| 141 | ) |
| 142 | |
Thomas Vachuska | a7be50d | 2018-04-16 14:02:18 -0700 | [diff] [blame] | 143 | port=${ONOS_WEB_PORT} |
| 144 | user=${ONOS_WEB_USER} |
| 145 | password=${ONOS_WEB_PASS} |
| 146 | |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 147 | # Scan arguments for user/password or other options... |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 148 | while getopts n:P:u:p:x?j?h o; do |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 149 | case "$o" in |
| 150 | n) name=$OPTARG;; |
Thomas Vachuska | a7be50d | 2018-04-16 14:02:18 -0700 | [diff] [blame] | 151 | P) port=$OPTARG;; |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 152 | u) user=$OPTARG;; |
| 153 | p) password=$OPTARG;; |
| 154 | x) extract=true;; |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 155 | j) json=true;; |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 156 | *) usage;; |
| 157 | esac |
| 158 | done |
Thomas Vachuska | 7f2a356 | 2018-02-28 10:02:16 -0800 | [diff] [blame] | 159 | |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 160 | let OPC=$OPTIND-1 |
| 161 | shift $OPC |
| 162 | |
| 163 | [ $# -lt 1 -a -z "$ONOS_INSTANCES" ] && usage; |
| 164 | |
| 165 | diags=/tmp/${name:-onos}-diags |
| 166 | rm -fr $diags $diags.tar.gz; mkdir -p $diags |
| 167 | |
| 168 | [ -z $1 ] && nodes=$ONOS_INSTANCES || nodes=$* |
| 169 | |
| 170 | # Collect diagnostics from each cluster node |
| 171 | for node in $nodes; do |
| 172 | printf "Collecting diagnostics on $node..." |
| 173 | |
| 174 | # Prepare a clean place for collecting the node diagnostic data |
| 175 | cd $diags; rm -fr $node; mkdir -p $node; cd $node; |
| 176 | |
| 177 | # Acquire locally obtained diagnostics via REST API and extract them |
| 178 | printf "logs " |
| 179 | curl -sS --fail --user $user:$password \ |
Thomas Vachuska | a7be50d | 2018-04-16 14:02:18 -0700 | [diff] [blame] | 180 | http://$node:$port/onos/v1/diagnostics > ../$node.tar.gz |
Thomas Vachuska | 7f2a356 | 2018-02-28 10:02:16 -0800 | [diff] [blame] | 181 | tar zxf ../$node.tar.gz && rm ../$node.tar.gz |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 182 | |
| 183 | # Acquire remotely obtained diagnostics via ssh CLI |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 184 | [ ! -z $json ] && CLI_COMMANDS=("${JSON_CLI_COMMANDS[@]}") |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 185 | for cmd in "${CLI_COMMANDS[@]}"; do |
| 186 | cmdLog="$(echo $cmd | cut -d\ -f1 | sed 's/:/-/g').txt" |
Seyeon Jeong | 357bcec | 2020-02-28 01:17:34 -0800 | [diff] [blame] | 187 | [ ! -z $json ] && cmdLog="${cmdLog%.txt}.json" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 188 | printf "$cmdLog " |
| 189 | onos $node $cmd 2>/dev/null >$cmdLog |
| 190 | done |
| 191 | |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 192 | printf " Done.\n" |
Thomas Vachuska | 15370d2 | 2018-01-30 13:57:38 -0800 | [diff] [blame] | 193 | done |
| 194 | |
| 195 | # Tar-up diagnostics from all the nodes |
| 196 | cd $diags |
| 197 | tar zcf $diags.tar.gz * |
| 198 | [ -z $extract ] && rm -fr $diags |