Fix STC scripts for checking ONOS nodes

- Fix onos-check-nodes to return an error code if  onos is up but nodes
  aren't in the correct states
- Add onos-check-storage-nodes to check if all atomix core nodes are
  connected
- Add a step for onos-check-storage-nodes step to stc setup

Change-Id: If61c2316653b6dc359fc84f18a65b2edbdd4acd8
diff --git a/tools/test/bin/onos-check-nodes b/tools/test/bin/onos-check-nodes
index 0c48a6a..6151a12 100755
--- a/tools/test/bin/onos-check-nodes
+++ b/tools/test/bin/onos-check-nodes
@@ -20,4 +20,5 @@
     # Check for differences
     diff $aux.1 $aux.2 && exit 0
     sleep 1
-done
\ No newline at end of file
+done
+exit 1
diff --git a/tools/test/bin/onos-check-storage-nodes b/tools/test/bin/onos-check-storage-nodes
new file mode 100755
index 0000000..37bb60e
--- /dev/null
+++ b/tools/test/bin/onos-check-storage-nodes
@@ -0,0 +1,24 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Checks whether all and only the configured ONOS core storeage instances are active.
+# -----------------------------------------------------------------------------
+
+aux=/tmp/stc/stc-$$.log
+trap "rm -f $aux $aux.1 $aux.2 2>/dev/null" EXIT
+
+for attempt in {1..5}; do
+    onos ${1:-$OCI} "onos:storage-nodes" > $aux
+    cat $aux
+
+    # Normalize the nodes
+    cut -d= -f3 $aux | cut -d: -f1 | sort > $aux.1
+
+    # Normalize the expected nodes
+    nodes=${2:-$ONOS_CORES}
+    (for node in $nodes; do echo $node; done) | sort > $aux.2
+
+    # Check for differences
+    diff $aux.1 $aux.2 && exit 0
+    sleep 1
+done
+exit 1
diff --git a/tools/test/scenarios/setup.xml b/tools/test/scenarios/setup.xml
index 10073e0..a830100 100644
--- a/tools/test/scenarios/setup.xml
+++ b/tools/test/scenarios/setup.xml
@@ -110,6 +110,8 @@
 
                 <step name="Check-Nodes-${#}" exec="onos-check-nodes ${OC#}" delay="3"
                       requires="~Wait-for-Start-${#}"/>
+                <step name="Check-Storage-Nodes-${#}" exec="onos-check-storage-nodes ${OC#}" delay="3"
+                      requires="~Wait-for-Start-${#}"/>
                 <step name="Check-Components-${#}" delay="5"
                       exec="onos-check-components ${OC#}"
                       requires="~Wait-for-Start-${#}"/>