Adding a tool for remote collection of node and cluster diagnostics.
- Includes 'onos-diagnostics', 'onos-node-diagnostics'
- Includes REST API /onos/v1/diagnostics
Change-Id: Ife0a15627b14238d0fce52b01b72d56e6a1fe40f
(cherry picked from commit 15370d2a16fb9577a20103801611b004fcb56a6e)
diff --git a/tools/package/bin/onos-form-cluster b/tools/package/bin/onos-form-cluster
index 9286d0c..e677873 100755
--- a/tools/package/bin/onos-form-cluster
+++ b/tools/package/bin/onos-form-cluster
@@ -2,8 +2,9 @@
# -----------------------------------------------------------------------------
# Forms ONOS cluster using REST API of each separate instance.
# -----------------------------------------------------------------------------
-
-[ $# -lt 2 ] && echo "usage: $(basename $0) ip1 ip2..." && exit 1
+function usage() {
+ echo "usage: $(basename $0)[-x] [-u user] [-p password] [-s partitionSize] ip1 ip2..." && exit 1
+}
# Scan arguments for user/password or other options...
while getopts u:p:s: o; do
@@ -11,6 +12,7 @@
u) user=$OPTARG;;
p) password=$OPTARG;;
s) partitionsize=$OPTARG;;
+ *) usage;;
esac
done
ONOS_WEB_USER=${ONOS_WEB_USER:-onos} # ONOS WEB User defaults to 'onos'
@@ -20,6 +22,8 @@
let OPC=$OPTIND-1
shift $OPC
+[ $# -lt 2 ] && usage
+
ip=$1
shift
nodes=$*
diff --git a/tools/package/runtime/bin/onos-diagnostics b/tools/package/runtime/bin/onos-diagnostics
new file mode 100755
index 0000000..51b3f8c
--- /dev/null
+++ b/tools/package/runtime/bin/onos-diagnostics
@@ -0,0 +1,137 @@
+#!/bin/bash
+
+#
+# Copyright 2015-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -----------------------------------------------------------------------------
+# Tool to collect cluster-wide diagnostics into a single tar stream.
+# -----------------------------------------------------------------------------
+function usage() {
+ echo "usage: $(basename $0) [-x] [-n name] [-u user] [-p password] [ip1 ip2...]"
+ echo ""
+ echo "Environment Variables:"
+ echo " ONOS_INSTANCES IPs or hostnames of ONOS cluster machines"
+ echo " ONOS_WEB_USER username for REST API"
+ echo " ONOS_WEB_PASS password for REST API"
+ echo ""
+ echo "Example Usages:"
+ echo " # Collect compressed diagnostics for the cluster."
+ echo " # REST API user and password are drawn from environment variables."
+ echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
+ echo " # The cluster node IPs will be drawn from ONOS_INSTANCES variable."
+ echo " > $(basename $0) "
+ echo ""
+ echo " # Collect diagnostics for the cluster and leave them extracted. "
+ echo " # Collection directory will be named /tmp/prague-diags/"
+ echo " # Collection archive will be named /tmp/prague-diags.tar.gz."
+ echo " # REST API user name is 'onos' and password is 'rules'."
+ echo " # The cluster node IPs will be drawn from ONOS_INSTANCES variable."
+ echo " > $(basename $0) -x -n prague -u onos -p rules"
+ echo ""
+ echo " # Collect compressed diagnostics for a cluster."
+ echo " # REST API user name is 'onos' and password is 'rules'."
+ echo " # Collection archive will be named /tmp/onos-diags.tar.gz"
+ echo " # The cluster node IPs are listed explicitly."
+ echo " > $(basename $0) -u onos -p rules 172.17.0.11 172.17.0.12 172.17.0.13"
+
+ exit 1
+}
+
+CLI_COMMANDS=(
+ "feature:list"
+ "bundle:list"
+ "scr:list"
+
+ "summary"
+ "nodes"
+ "apps -s"
+ "netcfg"
+ "cfg get"
+
+ "devices"
+ "links"
+ "hosts"
+
+ "ports -e"
+ "portstats -nz"
+
+ "intents"
+ "flows -s"
+ "groups"
+
+ "roles"
+ "masters"
+
+ "routes"
+ "obj-next-ids"
+ "obj-pending-nexts"
+
+ "log:display -l WARN"
+)
+
+# Scan arguments for user/password or other options...
+while getopts n:u:p:x?h o; do
+ case "$o" in
+ n) name=$OPTARG;;
+ u) user=$OPTARG;;
+ p) password=$OPTARG;;
+ x) extract=true;;
+ *) usage;;
+ esac
+done
+ONOS_WEB_USER=${ONOS_WEB_USER:-onos} # ONOS WEB User defaults to 'onos'
+ONOS_WEB_PASS=${ONOS_WEB_PASS:-rocks} # ONOS WEB Password defaults to 'rocks'
+user=${user:-$ONOS_WEB_USER}
+password=${password:-$ONOS_WEB_PASS}
+let OPC=$OPTIND-1
+shift $OPC
+
+[ $# -lt 1 -a -z "$ONOS_INSTANCES" ] && usage;
+
+diags=/tmp/${name:-onos}-diags
+rm -fr $diags $diags.tar.gz; mkdir -p $diags
+
+[ -z $1 ] && nodes=$ONOS_INSTANCES || nodes=$*
+
+# Collect diagnostics from each cluster node
+for node in $nodes; do
+ printf "Collecting diagnostics on $node..."
+
+ # Prepare a clean place for collecting the node diagnostic data
+ cd $diags; rm -fr $node; mkdir -p $node; cd $node;
+
+ # Acquire locally obtained diagnostics via REST API and extract them
+ printf "logs "
+ curl -sS --fail --user $user:$password \
+ http://$node:8181/onos/v1/diagnostics > ../$node.tar.gz
+ tar zxf ../$node.tar.gz
+
+ # Acquire remotely obtained diagnostics via ssh CLI
+ for cmd in "${CLI_COMMANDS[@]}"; do
+ cmdLog="$(echo $cmd | cut -d\ -f1 | sed 's/:/-/g').txt"
+ printf "$cmdLog "
+ onos $node $cmd 2>/dev/null >$cmdLog
+ done
+
+ # Tar-up local and remote diagnostics together
+ printf " Done.\n"
+ tar zcf ../$node.tar.gz *
+done
+
+# Tar-up diagnostics from all the nodes
+cd $diags
+tar zcf $diags.tar.gz *
+[ -z $extract ] && rm -fr $diags
diff --git a/tools/package/runtime/bin/onos-node-diagnostics b/tools/package/runtime/bin/onos-node-diagnostics
new file mode 100755
index 0000000..1cb18b5
--- /dev/null
+++ b/tools/package/runtime/bin/onos-node-diagnostics
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#
+# Copyright 2015-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -----------------------------------------------------------------------------
+# Tool to collect node-specific diagnostics into a single tar stream.
+# -----------------------------------------------------------------------------
+
+# If ONOS_HOME is set, respect its value.
+# If ONOS_HOME is not set (e.g. in the init or service environment),
+# set it based on this script's path.
+ONOS_HOME=${ONOS_HOME:-$(cd $(dirname $0)/.. >/dev/null 2>&1 && pwd)}
+
+diag=/tmp/diag$$
+mkdir -p $diag; cd $diag
+
+# Collect application state
+ls -l $ONOS_HOME/apps/* > apps-dir.txt
+
+# Collect the karaf logs
+cp $ONOS_HOME/apache-karaf-*/data/log/* .
+
+# Tar-up the collected information
+tar zcf ../onos-node-diags.tar.gz *
+rm -fr $diag
+
+
diff --git a/web/api/src/main/java/org/onosproject/rest/resources/CoreWebApplication.java b/web/api/src/main/java/org/onosproject/rest/resources/CoreWebApplication.java
index 6cd7884..4677844 100644
--- a/web/api/src/main/java/org/onosproject/rest/resources/CoreWebApplication.java
+++ b/web/api/src/main/java/org/onosproject/rest/resources/CoreWebApplication.java
@@ -52,7 +52,8 @@
VirtualNetworkWebResource.class,
MastershipWebResource.class,
InvalidConfigExceptionMapper.class,
- DpisWebResource.class
+ DpisWebResource.class,
+ DiagnosticsWebResource.class
);
}
}
diff --git a/web/api/src/main/java/org/onosproject/rest/resources/DiagnosticsWebResource.java b/web/api/src/main/java/org/onosproject/rest/resources/DiagnosticsWebResource.java
new file mode 100644
index 0000000..6b951e0
--- /dev/null
+++ b/web/api/src/main/java/org/onosproject/rest/resources/DiagnosticsWebResource.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.rest.resources;
+
+import org.onosproject.rest.AbstractWebResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import static com.google.common.io.ByteStreams.toByteArray;
+
+/**
+ * Provides stream of diagnostic information.
+ */
+@Path("diagnostics")
+public class DiagnosticsWebResource extends AbstractWebResource {
+
+ private static final String COMMAND = "../bin/onos-node-diagnostics";
+ private static final String DIAGS = "/tmp/onos-node-diags.tar.gz";
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ /**
+ * Get tar.gz stream of node diagnostic information.
+ *
+ * @return 200 OK with a tar.gz stream of diagnostic data
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ public Response getDiagnostics() {
+ try {
+ execute(COMMAND);
+ return ok(new FileInputStream(DIAGS)).build();
+ } catch (IOException e) {
+ return Response.serverError().build();
+ }
+ }
+
+ // Executes the given command arguments as a system command.
+ private void execute(String command) throws IOException {
+ try {
+ Process process = Runtime.getRuntime().exec(command);
+ byte[] output = toByteArray(process.getInputStream());
+ byte[] error = toByteArray(process.getErrorStream());
+ int code = process.waitFor();
+ if (code != 0) {
+ log.info("Command failed: status={}, output={}, error={}",
+ code, new String(output), new String(error));
+ }
+ } catch (InterruptedException e) {
+ log.error("Interrupted executing command {}", command, e);
+ }
+ }
+}