[AETHER-77] onos-diagnostics-k8s
- Adds onos-diagnostics-k8s
- Introduces profile abstraction to make some cmds optional
- Refactors onos-diagnostics to make use of the profiles
- Optimizes T3_OFFLINE profile
Change-Id: I615f03971018526b174894b39b4255e6f9ce4e3e
diff --git a/tools/package/runtime/bin/onos-diagnostics b/tools/package/runtime/bin/onos-diagnostics
index 655bb40..ec80974 100755
--- a/tools/package/runtime/bin/onos-diagnostics
+++ b/tools/package/runtime/bin/onos-diagnostics
@@ -23,6 +23,8 @@
echo "usage: $(basename $0) [-x] [-j] [-n name] [-P port] [-u user] [-p password] [ip1 ip2...]"
echo ""
echo "Environment Variables:"
+ echo " DIAGS_PROFILE Profile to be used to collect diags."
+ echo " Availables profiles in onos-diagnostics-profile"
echo " ONOS_INSTANCES IPs or hostnames of ONOS cluster machines"
echo " ONOS_WEB_USER username for REST API"
echo " ONOS_WEB_PASS password for REST API"
@@ -32,6 +34,7 @@
echo " # REST API user and password are drawn from environment variables."
echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
echo " # The cluster node IPs will be drawn from ONOS_INSTANCES variable."
+ echo " # Diags profile is drawn from environment variable."
echo " > $(basename $0) "
echo ""
echo " # Collect diagnostics for the cluster and leave them extracted. "
@@ -55,91 +58,18 @@
exit 1
}
+# Let's source the different profiles
+. onos-diagnostics-profile
+
+# By default the ONOS base profile will be used
+[ -z "$DIAGS_PROFILE" ] && DIAGS_PROFILE=ONOS_PROFILE;
+
ONOS_WEB_USER=${ONOS_WEB_USER:-onos} # ONOS WEB User defaults to 'onos'
ONOS_WEB_PASS=${ONOS_WEB_PASS:-rocks} # ONOS WEB Password defaults to 'rocks'
ONOS_WEB_PORT=${ONOS_WEB_PORT:-8181} # REST API port defaults to '8181'
. $(dirname $0)/_find-node
-# TODO We should make SR commands optional
-CLI_COMMANDS=(
- "feature:repo-list"
- "feature:list"
- "bundle:list"
- "scr-list"
-
- "summary"
- "nodes"
- "apps -s"
- "netcfg"
- "cfg get"
-
- "devices"
- "device-drivers"
- "links"
- "hosts"
- "interfaces"
-
- "ports"
- "portstats -nz"
- "edge-ports"
-
- "packet-processors"
- "packet-requests"
-
- "intents"
- "flows"
- "groups"
-
- "roles"
- "masters"
-
- "maps"
-
- "fpm-connections"
- "routes"
- "obj-next-ids"
- "obj-pending-nexts"
- "obj-queues"
-
- "sr-device-subnets"
- "sr-ecmp-spg"
- "sr-should-program"
- "sr-link-state"
- "sr-mcast-tree"
- "sr-mcast-leader"
- "sr-mcast-role"
- "sr-pw-list"
- "sr-next-mcast"
- "sr-filt-mcast"
- "sr-next-dst"
- "sr-next-port"
- "sr-next-vlan"
- "sr-next-pw"
- "sr-next-xconnect"
- "sr-next-mac-vlan"
- "sr-pr-list"
- "dhcp-relay"
-
- "mcast-host-routes"
- "mcast-host-show"
-)
-
-JSON_CLI_COMMANDS=(
- "netcfg -j"
- "devices -j"
- "device-drivers -j"
- "links -j"
- "hosts -j"
- "ports -j"
- "edge-ports -j"
- "flows -j"
- "groups -j"
- "masters -j"
- "routes -j"
- "mcast-host-show -j"
-)
-
port=${ONOS_WEB_PORT}
user=${ONOS_WEB_USER}
password=${ONOS_WEB_PASS}
@@ -167,6 +97,9 @@
[ -z $1 ] && nodes=$ONOS_INSTANCES || nodes=$*
+# -j option will activate T3_OFFLINE_PROFILE
+[ ! -z $json ] && DIAGS_PROFILE=T3_OFFLINE_PROFILE
+
# Collect diagnostics from each cluster node
for node in $nodes; do
printf "Collecting diagnostics on $node..."
@@ -174,15 +107,19 @@
# Prepare a clean place for collecting the node diagnostic data
cd $diags; rm -fr $node; mkdir -p $node; cd $node;
- # Acquire locally obtained diagnostics via REST API and extract them
- printf "logs "
- curl -sS --fail --user $user:$password \
- http://$node:$port/onos/v1/diagnostics > ../$node.tar.gz
- tar zxf ../$node.tar.gz && rm ../$node.tar.gz
+ if [ -z $json ]; then
+ # Acquire locally obtained diagnostics via REST API and extract them
+ printf "logs "
+ curl -sS --fail --user $user:$password \
+ http://$node:$port/onos/v1/diagnostics > ../$node.tar.gz
+ tar zxf ../$node.tar.gz && rm ../$node.tar.gz
+ fi
# Acquire remotely obtained diagnostics via ssh CLI
- [ ! -z $json ] && CLI_COMMANDS=("${JSON_CLI_COMMANDS[@]}")
- for cmd in "${CLI_COMMANDS[@]}"; do
+ eval CLI_COMMANDS=\${${DIAGS_PROFILE}[@]}
+ for cmd in $CLI_COMMANDS; do
+ # @ is used as workaround for the array expansion
+ cmd="$(echo $cmd | sed 's/@/ /g')"
cmdLog="$(echo $cmd | cut -d\ -f1 | sed 's/:/-/g').txt"
[ ! -z $json ] && cmdLog="${cmdLog%.txt}.json"
printf "$cmdLog "
diff --git a/tools/package/runtime/bin/onos-diagnostics-k8s b/tools/package/runtime/bin/onos-diagnostics-k8s
new file mode 100755
index 0000000..e5b9aa7
--- /dev/null
+++ b/tools/package/runtime/bin/onos-diagnostics-k8s
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+#
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# --------------------------------------------------------------------------------
+# Tool to collect cluster-wide diagnostics into a single tar stream using kubectl
+# --------------------------------------------------------------------------------
+function usage() {
+ echo "usage: $(basename $0) [-x] [-j] [-n name] [-k karaf_home] [pod1 pod2...]"
+ echo ""
+ echo "Environment Variables:"
+ echo " DIAGS_PROFILE Profile to be used to collect diags."
+ echo " Availables profiles in onos-diagnostics-profile"
+ echo " KARAF_HOME KARAF_HOME inside the ONOS pod (path from the mount point)"
+ echo " ONOS_PODS ONOS pods composing the cluster"
+ echo ""
+ echo "Example Usages:"
+ echo " # Collect compressed diagnostics for the cluster."
+ echo " # Karaf home is drawn from environment variable."
+ echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
+ echo " # ONOS pods names will be drawn from ONOS_PODS variable."
+ echo " # Diags profile is drawn from environment variable."
+ echo " > $(basename $0) "
+ echo ""
+ echo " # Collect diagnostics for the cluster and leave them extracted. "
+ echo " # Collection directory will be named /tmp/prague-diags/"
+ echo " # Collection archive will be named /tmp/prague-diags.tar.gz."
+ echo " # Karaf home is '/root/foo/karaf'."
+ echo " # ONOS pods names will be drawn from ONOS_PODS variable."
+ echo " > $(basename $0) -x -n prague -k karaf"
+ echo ""
+ echo " # Collect diagnostics for the cluster and store them in JSON files."
+ echo " # JSON_CLI_COMMANDS below lists JSON-supported diagnostics commands."
+ echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
+ echo " > $(basename $0) -j"
+ echo ""
+ echo " # Collect compressed diagnostics for a cluster."
+ echo " # Karaf home is 'karaf'."
+ echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
+ echo " # The pods names are listed explicitly."
+ echo " > $(basename $0) -k karaf onos-0 onos-1 onos-2"
+
+ exit 1
+}
+
+command -v kubectl >/dev/null 2>&1 || usage;
+
+# Let's source the different profiles
+. onos-diagnostics-profile
+
+# By default the ONOS base profile will be used
+[ -z "$DIAGS_PROFILE" ] && DIAGS_PROFILE=ONOS_PROFILE;
+
+# Scan arguments for user/password or other options...
+while getopts n:k:x?j?h o; do
+ case "$o" in
+ n) name=$OPTARG;;
+ k) KARAF_HOME=$OPTARG;;
+ x) extract=true;;
+ j) json=true;;
+ *) usage;;
+ esac
+done
+
+let OPC=$OPTIND-1
+shift $OPC
+
+[ $# -lt 1 -a -z "$ONOS_PODS" ] && usage;
+
+[ -z "$KARAF_HOME" ] && usage;
+
+diags=/tmp/${name:-onos}-diags
+rm -fr $diags $diags.tar.gz; mkdir -p $diags
+
+[ -z $1 ] && nodes=$ONOS_PODS || nodes=$*
+
+# -j option will activate T3_OFFLINE_PROFILE
+[ ! -z $json ] && DIAGS_PROFILE=T3_OFFLINE_PROFILE
+
+# Collect diagnostics from each cluster node
+for node in $nodes; do
+ printf "Collecting diagnostics on $node..."
+
+ # Prepare a clean place for collecting the node diagnostic data
+ cd $diags; rm -fr $node; mkdir -p $node; cd $node;
+
+ if [ -z $json ]; then
+ # Acquire locally obtained diagnostics via kubectl
+ printf "logs "
+ kubectl cp $node:$KARAF_HOME/data/log .
+ kubectl exec -it $node -- bash -c "ls -l apps/*" > apps-dir.txt
+ fi
+
+ # Acquire apps info through onos cli
+ eval CLI_COMMANDS=\${${DIAGS_PROFILE}[@]}
+ for cmd in $CLI_COMMANDS; do
+ # @ is used as workaround for the array expansion
+ cmd="$(echo $cmd | sed 's/@/ /g')"
+ cmdLog="$(echo $cmd | cut -d\ -f1 | sed 's/:/-/g').txt"
+ printf "$cmdLog "
+ CMD="kubectl exec -it $node -- bash $KARAF_HOME/bin/client"
+ $CMD "$cmd" 2>/dev/null | sed -n '1!p' >$cmdLog
+ done
+
+ printf " Done.\n"
+done
+
+# Tar-up diagnostics from all the nodes
+cd $diags
+tar zcf $diags.tar.gz *
+[ -z $extract ] && rm -fr $diags
diff --git a/tools/package/runtime/bin/onos-diagnostics-profile b/tools/package/runtime/bin/onos-diagnostics-profile
new file mode 100644
index 0000000..fecefae
--- /dev/null
+++ b/tools/package/runtime/bin/onos-diagnostics-profile
@@ -0,0 +1,145 @@
+#!/bin/bash
+
+#
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# ------------------------------------------------
+# Defines profiles for different ONF projects
+# ------------------------------------------------
+
+# Base commands that are always useful.
+ONOS_PROFILE=(
+ "feature:repo-list"
+ "feature:list"
+ "bundle:list"
+ "scr-list"
+
+ "summary"
+ "nodes"
+ "apps@-s"
+ "netcfg"
+ "cfg@get"
+
+ "devices"
+ "device-drivers"
+ "links"
+ "hosts"
+ "interfaces"
+
+ "ports"
+ "portstats@-nz"
+ "edge-ports"
+
+ "packet-processors"
+ "packet-requests"
+
+ "intents"
+ "flows"
+ "groups"
+ "meters"
+
+ "roles"
+ "masters"
+
+ "maps"
+
+ "obj-next-ids"
+ "obj-pending-nexts"
+ "obj-queues"
+)
+
+# Trellis related commands. Includes also the base commands.
+TRELLIS_PROFILE=(
+ "${ONOS_PROFILE[@]}"
+
+ "fpm-connections"
+ "routes"
+ "sr-device-subnets"
+ "sr-ecmp-spg"
+ "sr-should-program"
+ "sr-link-state"
+ "sr-mcast-tree"
+ "sr-mcast-leader"
+ "sr-mcast-role"
+ "sr-pw-list"
+ "sr-next-mcast"
+ "sr-filt-mcast"
+ "sr-next-dst"
+ "sr-next-port"
+ "sr-next-vlan"
+ "sr-next-pw"
+ "sr-next-xconnect"
+ "sr-next-mac-vlan"
+ "dhcp-relay"
+
+ "mcast-host-routes"
+ "mcast-host-show"
+)
+
+# T3 offline related commands.
+T3_OFFLINE_PROFILE=(
+ "netcfg@-j"
+ "devices@-j"
+ "device-drivers@-j"
+ "links@-j"
+ "hosts@-j"
+ "ports@-j"
+ "edge-ports@-j"
+ "flows@-j"
+ "groups@-j"
+ "masters@-j"
+ "routes@-j"
+ "mcast-host-show@-j"
+)
+
+# VOLTHA related commands. Includes also the base commands.
+VOLTHA_PROFILE=(
+ "${ONOS_PROFILE[@]}"
+
+ "aaa-users"
+
+ "dhcpl2relay-allocations"
+
+ "volt-programmed-subscribers"
+ "volt-programmed-meters"
+ "volt-bpmeter-mappings"
+ "volt-olts"
+
+ "sr-device-subnets"
+ "sr-ecmp-spg"
+ "sr-should-program"
+ "sr-link-state"
+ "sr-xconnect"
+ "sr-mcast-tree"
+ "sr-mcast-leader"
+ "sr-mcast-role"
+ "sr-next-mcast"
+ "sr-filt-mcast"
+ "sr-next-dst"
+ "sr-next-port"
+ "sr-next-vlan"
+ "sr-next-xconnect"
+ "sr-next-mac-vlan"
+
+ "mcast-host-routes"
+ "mcast-host-show"
+
+)
+
+# SEBA related commands. Includes also the base commands.
+SEBA_PROFILE=(
+ "${VOLTHA_PROFILE[@]}"
+)
\ No newline at end of file