blob: f3b60e39d07f36995d757cec23bf580ec377d284 [file] [log] [blame]
pier1fece882020-06-12 18:48:37 +02001#!/bin/bash
2
3#
4# Copyright 2020-present Open Networking Foundation
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17#
18
19# --------------------------------------------------------------------------------
20# Tool to collect cluster-wide diagnostics into a single tar stream using kubectl
21# --------------------------------------------------------------------------------
22function usage() {
pierventrebaffa992020-09-28 14:15:23 +020023 echo "usage: $(basename $0) [-x] [-j] [-l] [-n name] [-k karaf_home] [-s namespace] [pod1 pod2...]"
pier1fece882020-06-12 18:48:37 +020024 echo ""
25 echo "Environment Variables:"
26 echo " DIAGS_PROFILE Profile to be used to collect diags."
27 echo " Availables profiles in onos-diagnostics-profile"
28 echo " KARAF_HOME KARAF_HOME inside the ONOS pod (path from the mount point)"
pierventrebaffa992020-09-28 14:15:23 +020029 echo " NAMESPACE k8s namespace"
pier1fece882020-06-12 18:48:37 +020030 echo " ONOS_PODS ONOS pods composing the cluster"
31 echo ""
32 echo "Example Usages:"
33 echo " # Collect compressed diagnostics for the cluster."
34 echo " # Karaf home is drawn from environment variable."
35 echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
36 echo " # ONOS pods names will be drawn from ONOS_PODS variable."
37 echo " # Diags profile is drawn from environment variable."
38 echo " > $(basename $0) "
39 echo ""
40 echo " # Collect diagnostics for the cluster and leave them extracted. "
41 echo " # Collection directory will be named /tmp/prague-diags/"
42 echo " # Collection archive will be named /tmp/prague-diags.tar.gz."
43 echo " # Karaf home is '/root/foo/karaf'."
44 echo " # ONOS pods names will be drawn from ONOS_PODS variable."
45 echo " > $(basename $0) -x -n prague -k karaf"
46 echo ""
47 echo " # Collect diagnostics for the cluster and store them in JSON files."
48 echo " # JSON_CLI_COMMANDS below lists JSON-supported diagnostics commands."
49 echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
50 echo " > $(basename $0) -j"
51 echo ""
pierventrebaffa992020-09-28 14:15:23 +020052 echo " # Collect ONOS logs from the Karaf home."
53 echo " > $(basename $0) -l"
54 echo ""
pier1fece882020-06-12 18:48:37 +020055 echo " # Collect compressed diagnostics for a cluster."
56 echo " # Karaf home is 'karaf'."
57 echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
58 echo " # The pods names are listed explicitly."
59 echo " > $(basename $0) -k karaf onos-0 onos-1 onos-2"
60
61 exit 1
62}
63
64command -v kubectl >/dev/null 2>&1 || usage;
65
66# Let's source the different profiles
67. onos-diagnostics-profile
68
69# By default the ONOS base profile will be used
70[ -z "$DIAGS_PROFILE" ] && DIAGS_PROFILE=ONOS_PROFILE;
71
72# Scan arguments for user/password or other options...
pierventrebaffa992020-09-28 14:15:23 +020073while getopts n:k:s:x?j?l?h o; do
pier1fece882020-06-12 18:48:37 +020074 case "$o" in
75 n) name=$OPTARG;;
76 k) KARAF_HOME=$OPTARG;;
pierventrebaffa992020-09-28 14:15:23 +020077 s) NAMESPACE=$OPTARG;;
pier1fece882020-06-12 18:48:37 +020078 x) extract=true;;
79 j) json=true;;
pierventrebaffa992020-09-28 14:15:23 +020080 l) KARAF_LOG=true;;
pier1fece882020-06-12 18:48:37 +020081 *) usage;;
82 esac
83done
84
85let OPC=$OPTIND-1
86shift $OPC
87
88[ $# -lt 1 -a -z "$ONOS_PODS" ] && usage;
89
90[ -z "$KARAF_HOME" ] && usage;
91
pierventrebaffa992020-09-28 14:15:23 +020092[ -z "$NAMESPACE" ] && usage;
93
pier1fece882020-06-12 18:48:37 +020094diags=/tmp/${name:-onos}-diags
95rm -fr $diags $diags.tar.gz; mkdir -p $diags
96
97[ -z $1 ] && nodes=$ONOS_PODS || nodes=$*
98
99# -j option will activate T3_OFFLINE_PROFILE
100[ ! -z $json ] && DIAGS_PROFILE=T3_OFFLINE_PROFILE
101
102# Collect diagnostics from each cluster node
103for node in $nodes; do
104 printf "Collecting diagnostics on $node..."
105
106 # Prepare a clean place for collecting the node diagnostic data
107 cd $diags; rm -fr $node; mkdir -p $node; cd $node;
108
109 if [ -z $json ]; then
110 # Acquire locally obtained diagnostics via kubectl
111 printf "logs "
pierventrebaffa992020-09-28 14:15:23 +0200112 if [ -z "$KARAF_LOG" ]; then
113 kubectl -n $NAMESPACE logs $node > karaf.log
114 else
115 kubectl -n $NAMESPACE cp $node:$KARAF_HOME/data/log .
116 fi
117 kubectl -n $NAMESPACE exec -it $node -- bash -c "ls -l apps/*" > apps-dir.txt
pier1fece882020-06-12 18:48:37 +0200118 fi
119
120 # Acquire apps info through onos cli
121 eval CLI_COMMANDS=\${${DIAGS_PROFILE}[@]}
122 for cmd in $CLI_COMMANDS; do
123 # @ is used as workaround for the array expansion
124 cmd="$(echo $cmd | sed 's/@/ /g')"
125 cmdLog="$(echo $cmd | cut -d\ -f1 | sed 's/:/-/g').txt"
126 printf "$cmdLog "
pierventrebaffa992020-09-28 14:15:23 +0200127 CMD="kubectl -n $NAMESPACE exec -it $node -- bash $KARAF_HOME/bin/client"
pier1fece882020-06-12 18:48:37 +0200128 $CMD "$cmd" 2>/dev/null | sed -n '1!p' >$cmdLog
129 done
130
131 printf " Done.\n"
132done
133
134# Tar-up diagnostics from all the nodes
135cd $diags
136tar zcf $diags.tar.gz *
137[ -z $extract ] && rm -fr $diags