pier | 724037b | 2020-06-12 18:48:37 +0200 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | |
| 3 | # |
| 4 | # Copyright 2020-present Open Networking Foundation |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
| 17 | # |
| 18 | |
| 19 | # -------------------------------------------------------------------------------- |
| 20 | # Tool to collect cluster-wide diagnostics into a single tar stream using kubectl |
| 21 | # -------------------------------------------------------------------------------- |
| 22 | function usage() { |
| 23 | echo "usage: $(basename $0) [-x] [-j] [-n name] [-k karaf_home] [pod1 pod2...]" |
| 24 | echo "" |
| 25 | echo "Environment Variables:" |
| 26 | echo " DIAGS_PROFILE Profile to be used to collect diags." |
| 27 | echo " Availables profiles in onos-diagnostics-profile" |
| 28 | echo " KARAF_HOME KARAF_HOME inside the ONOS pod (path from the mount point)" |
| 29 | echo " ONOS_PODS ONOS pods composing the cluster" |
| 30 | echo "" |
| 31 | echo "Example Usages:" |
| 32 | echo " # Collect compressed diagnostics for the cluster." |
| 33 | echo " # Karaf home is drawn from environment variable." |
| 34 | echo " # Collection archive will be named /tmp/onos-diags.tar.gz" |
| 35 | echo " # ONOS pods names will be drawn from ONOS_PODS variable." |
| 36 | echo " # Diags profile is drawn from environment variable." |
| 37 | echo " > $(basename $0) " |
| 38 | echo "" |
| 39 | echo " # Collect diagnostics for the cluster and leave them extracted. " |
| 40 | echo " # Collection directory will be named /tmp/prague-diags/" |
| 41 | echo " # Collection archive will be named /tmp/prague-diags.tar.gz." |
| 42 | echo " # Karaf home is '/root/foo/karaf'." |
| 43 | echo " # ONOS pods names will be drawn from ONOS_PODS variable." |
| 44 | echo " > $(basename $0) -x -n prague -k karaf" |
| 45 | echo "" |
| 46 | echo " # Collect diagnostics for the cluster and store them in JSON files." |
| 47 | echo " # JSON_CLI_COMMANDS below lists JSON-supported diagnostics commands." |
| 48 | echo " # Collection archive will be named /tmp/onos-diags.tar.gz" |
| 49 | echo " > $(basename $0) -j" |
| 50 | echo "" |
| 51 | echo " # Collect compressed diagnostics for a cluster." |
| 52 | echo " # Karaf home is 'karaf'." |
| 53 | echo " # Collection archive will be named /tmp/onos-diags.tar.gz" |
| 54 | echo " # The pods names are listed explicitly." |
| 55 | echo " > $(basename $0) -k karaf onos-0 onos-1 onos-2" |
| 56 | |
| 57 | exit 1 |
| 58 | } |
| 59 | |
| 60 | command -v kubectl >/dev/null 2>&1 || usage; |
| 61 | |
| 62 | # Let's source the different profiles |
| 63 | . onos-diagnostics-profile |
| 64 | |
| 65 | # By default the ONOS base profile will be used |
| 66 | [ -z "$DIAGS_PROFILE" ] && DIAGS_PROFILE=ONOS_PROFILE; |
| 67 | |
| 68 | # Scan arguments for user/password or other options... |
| 69 | while getopts n:k:x?j?h o; do |
| 70 | case "$o" in |
| 71 | n) name=$OPTARG;; |
| 72 | k) KARAF_HOME=$OPTARG;; |
| 73 | x) extract=true;; |
| 74 | j) json=true;; |
| 75 | *) usage;; |
| 76 | esac |
| 77 | done |
| 78 | |
| 79 | let OPC=$OPTIND-1 |
| 80 | shift $OPC |
| 81 | |
| 82 | [ $# -lt 1 -a -z "$ONOS_PODS" ] && usage; |
| 83 | |
| 84 | [ -z "$KARAF_HOME" ] && usage; |
| 85 | |
| 86 | diags=/tmp/${name:-onos}-diags |
| 87 | rm -fr $diags $diags.tar.gz; mkdir -p $diags |
| 88 | |
| 89 | [ -z $1 ] && nodes=$ONOS_PODS || nodes=$* |
| 90 | |
| 91 | # -j option will activate T3_OFFLINE_PROFILE |
| 92 | [ ! -z $json ] && DIAGS_PROFILE=T3_OFFLINE_PROFILE |
| 93 | |
| 94 | # Collect diagnostics from each cluster node |
| 95 | for node in $nodes; do |
| 96 | printf "Collecting diagnostics on $node..." |
| 97 | |
| 98 | # Prepare a clean place for collecting the node diagnostic data |
| 99 | cd $diags; rm -fr $node; mkdir -p $node; cd $node; |
| 100 | |
| 101 | if [ -z $json ]; then |
| 102 | # Acquire locally obtained diagnostics via kubectl |
| 103 | printf "logs " |
| 104 | kubectl cp $node:$KARAF_HOME/data/log . |
| 105 | kubectl exec -it $node -- bash -c "ls -l apps/*" > apps-dir.txt |
| 106 | fi |
| 107 | |
| 108 | # Acquire apps info through onos cli |
| 109 | eval CLI_COMMANDS=\${${DIAGS_PROFILE}[@]} |
| 110 | for cmd in $CLI_COMMANDS; do |
| 111 | # @ is used as workaround for the array expansion |
| 112 | cmd="$(echo $cmd | sed 's/@/ /g')" |
| 113 | cmdLog="$(echo $cmd | cut -d\ -f1 | sed 's/:/-/g').txt" |
| 114 | printf "$cmdLog " |
| 115 | CMD="kubectl exec -it $node -- bash $KARAF_HOME/bin/client" |
| 116 | $CMD "$cmd" 2>/dev/null | sed -n '1!p' >$cmdLog |
| 117 | done |
| 118 | |
| 119 | printf " Done.\n" |
| 120 | done |
| 121 | |
| 122 | # Tar-up diagnostics from all the nodes |
| 123 | cd $diags |
| 124 | tar zcf $diags.tar.gz * |
| 125 | [ -z $extract ] && rm -fr $diags |