Enhancing diagnostic and test facilities for troubleshooting

- added new CLI to list driver providers
- added ability to power on/off borrowed cells
- enabled ONOS service to restart on boot/death

Change-Id: Ifc889cdbc0740e5d3286c9836dfecc38d458bbc5
diff --git a/tools/test/bin/onos-install b/tools/test/bin/onos-install
index 65b7a42..ea2ab68 100755
--- a/tools/test/bin/onos-install
+++ b/tools/test/bin/onos-install
@@ -101,8 +101,8 @@
     echo 'log4j.appender.out.layout.ConversionPattern=%d{ISO8601} | %-5.5p | %-16.16t | %-32.32c{1} | %X{bundle.id} | %m%n' \
         >> $ONOS_INSTALL_DIR/$KARAF_DIST/etc/org.ops4j.pax.logging.cfg
 
-    # Set up the ONOS service on systemd-based systems
-    sudo systemctl daemon-reload || true
+    # Set up and enable the ONOS service on systemd-based systems
+    sudo systemctl daemon-reload && sudo systemctl enable onos.service || true
 
 "
 
diff --git a/tools/test/bin/onos-power b/tools/test/bin/onos-power
new file mode 100755
index 0000000..8b37c5d
--- /dev/null
+++ b/tools/test/bin/onos-power
@@ -0,0 +1,34 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Provides a unioform way to simulate power on/off control over an ONOS node.
+# If environment variable HARD_POWER_OFF is set and the ONOS_CELL value
+# indicates that the cell is a borrowed shared cell LXC machine, it will use
+# a REST call to warden, which then uses lxc-stop/lxc-start command to
+# simulate a power control operations. Otherwise, it will just use the soft
+# 'onos-die' command.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+node=${1:-$OCI}
+cmd=${2:-on}
+
+if [ "$cmd" = "on" ]; then
+    if [ -n "$HARD_POWER_OFF" -a $ONOS_CELL = "borrow" ]; then
+        curl -sS -X PUT "http://$CELL_WARDEN:4321/?cmd=powerOn&nodeIp=$node&user=$(id -un)"
+    else
+        onos-service $node start
+    fi
+
+elif [ "$cmd" = "off" ]; then
+    if [ -n "$HARD_POWER_OFF" -a $ONOS_CELL = "borrow" ]; then
+        curl -sS -X PUT "http://$CELL_WARDEN:4321/?cmd=powerOff&nodeIp=$node&user=$(id -un)"
+    else
+        onos-die $node start
+    fi
+
+else
+    echo "usage: $(basename $0) node-ip {on|off}" >&2 && exit 1
+fi
+
diff --git a/tools/test/scenarios/ha-single-node.xml b/tools/test/scenarios/ha-single-node.xml
index bf69e96..27a0e4e 100644
--- a/tools/test/scenarios/ha-single-node.xml
+++ b/tools/test/scenarios/ha-single-node.xml
@@ -17,8 +17,7 @@
 <scenario name="ha-single-node"
           description="ONOS cluster single node failure and recovery">
     <group name="Single-Node-Failure" if="${OC2}">
-        <step name="Node-Death" exec="onos-die ${OC1}"/>
-
+        <step name="Node-Death" exec="onos-power ${OC1} off"/>
         <step name="Validate-Death" requires="Node-Death"
               exec="onos-check-node-status ${OC2} ${OC1} INACTIVE"/>
 
@@ -36,10 +35,8 @@
                   exec="onos-check-flows ${OC2}"/>
         </group>
 
-        <group name="Recover" requires="Validate-Normal-Operation">
-            <step name="Node-Start"
-                  exec="onos-service ${OC1} start"/>
-
+        <group name="Recover" requires="~Validate-Normal-Operation">
+            <step name="Node-Start" exec="onos-power ${OC1} on"/>
             <step name="Wait-for-Start" requires="~Node-Start"
                   exec="onos-wait-for-start ${OC1}"/>
         </group>