tools/package/runtime/bin/onos-diagnostics-k8s

#!/bin/bash

#
# Copyright 2020-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# --------------------------------------------------------------------------------
# Tool to collect cluster-wide diagnostics into a single tar stream using kubectl
# --------------------------------------------------------------------------------
function usage() {
    echo "usage: $(basename $0) [-x] [-j] [-l] [-n name] [-k karaf_home] [-s namespace] [pod1 pod2...]"
    echo ""
    echo "Environment Variables:"
    echo "    DIAGS_PROFILE     Profile to be used to collect diags."
    echo "                      Availables profiles in onos-diagnostics-profile"
    echo "    KARAF_HOME        KARAF_HOME inside the ONOS pod (path from the mount point)"
    echo "    NAMESPACE         k8s namespace"
    echo "    ONOS_PODS         ONOS pods composing the cluster"
    echo ""
    echo "Example Usages:"
    echo "    # Collect compressed diagnostics for the cluster."
    echo "    # Karaf home is drawn from environment variable."
    echo "    # Collection archive will be named /tmp/onos-diags.tar.gz"
    echo "    # ONOS pods names will be drawn from ONOS_PODS variable."
    echo "    # Diags profile is drawn from environment variable."
    echo "    > $(basename $0) "
    echo ""
    echo "    # Collect diagnostics for the cluster and leave them extracted. "
    echo "    # Collection directory will be named /tmp/prague-diags/"
    echo "    # Collection archive will be named /tmp/prague-diags.tar.gz."
    echo "    # Karaf home is '/root/foo/karaf'."
    echo "    # ONOS pods names will be drawn from ONOS_PODS variable."
    echo "    > $(basename $0) -x -n prague -k karaf"
    echo ""
    echo "    # Collect diagnostics for the cluster and store them in JSON files."
    echo "    # JSON_CLI_COMMANDS below lists JSON-supported diagnostics commands."
    echo "    # Collection archive will be named /tmp/onos-diags.tar.gz"
    echo "    > $(basename $0) -j"
    echo ""
    echo "    # Collect ONOS logs from the Karaf home."
    echo "    > $(basename $0) -l"
    echo ""
    echo "    # Collect compressed diagnostics for a cluster."
    echo "    # Karaf home is 'karaf'."
    echo "    # Collection archive will be named /tmp/onos-diags.tar.gz"
    echo "    # The pods names are listed explicitly."
    echo "    > $(basename $0) -k karaf onos-0 onos-1 onos-2"

    exit 1
}

command -v kubectl >/dev/null 2>&1 || usage;

# Let's source the different profiles
. onos-diagnostics-profile

# By default the ONOS base profile will be used
[ -z "$DIAGS_PROFILE" ] && DIAGS_PROFILE=ONOS_PROFILE;

# Scan arguments for user/password or other options...
while getopts n:k:s:x?j?l?h o; do
    case "$o" in
        n) name=$OPTARG;;
        k) KARAF_HOME=$OPTARG;;
        s) NAMESPACE=$OPTARG;;
        x) extract=true;;
        j) json=true;;
        l) KARAF_LOG=true;;
        *) usage;;
    esac
done

let OPC=$OPTIND-1
shift $OPC

[ $# -lt 1 -a -z "$ONOS_PODS" ] && usage;

[ -z "$KARAF_HOME" ] && usage;

[ -z "$NAMESPACE" ] && usage;

diags=/tmp/${name:-onos}-diags
rm -fr $diags $diags.tar.gz; mkdir -p $diags

[ -z $1 ] && nodes=$ONOS_PODS || nodes=$*

# -j option will activate T3_OFFLINE_PROFILE
[ ! -z $json ] && DIAGS_PROFILE=T3_OFFLINE_PROFILE

# Collect diagnostics from each cluster node
for node in $nodes; do
    printf "Collecting diagnostics on $node..."

    # Prepare a clean place for collecting the node diagnostic data
    cd $diags; rm -fr $node; mkdir -p $node; cd $node;

    if [ -z $json ]; then
        # Acquire locally obtained diagnostics via kubectl
        printf "logs "
        if [ -z "$KARAF_LOG" ]; then
          kubectl -n $NAMESPACE logs $node > karaf.log
        else
          kubectl -n $NAMESPACE cp $node:$KARAF_HOME/data/log .
        fi
        kubectl -n $NAMESPACE exec -it $node -- bash -c "ls -l apps/*" > apps-dir.txt
    fi

    # Acquire apps info through onos cli
    eval CLI_COMMANDS=\${${DIAGS_PROFILE}[@]}
    for cmd in $CLI_COMMANDS; do
        # @ is used as workaround for the array expansion
        cmd="$(echo $cmd | sed 's/@/ /g')"
        cmdLog="$(echo $cmd | cut -d\  -f1 | sed 's/:/-/g').txt"
        printf "$cmdLog "
        CMD="kubectl -n $NAMESPACE exec -it $node -- bash $KARAF_HOME/bin/client"
        $CMD "$cmd" 2>/dev/null | sed -n '1!p' >$cmdLog
    done

    printf " Done.\n"
done

# Tar-up diagnostics from all the nodes
cd $diags
tar zcf $diags.tar.gz *
[ -z $extract ] && rm -fr $diags