Make STC less sensitive to timing problems with node and app startup

Change-Id: Ia5d7d7e2301e3c94b7aa2c388831a51561bc95fa
diff --git a/tools/test/bin/onos-wait-for-command b/tools/test/bin/onos-wait-for-command
new file mode 100755
index 0000000..ec4e015
--- /dev/null
+++ b/tools/test/bin/onos-wait-for-command
@@ -0,0 +1,21 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Waits for a CLI command to be runnable.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+. $ONOS_ROOT/tools/test/bin/find-node.sh
+
+node=$(find_node ${1})
+cmd="$2"
+
+remote=$ONOS_USER@$node
+
+for i in {1..10}; do
+    $ONOS_ROOT/tools/test/bin/onos $node  ${cmd} 2>/dev/null | \
+        grep -q -v 'Command not found:' && exit 0 || sleep 3
+done
+
+exit 1
+
diff --git a/tools/test/bin/onos-wait-for-start b/tools/test/bin/onos-wait-for-start
index 2cc5002..aad891c 100755
--- a/tools/test/bin/onos-wait-for-start
+++ b/tools/test/bin/onos-wait-for-start
@@ -19,7 +19,7 @@
 
 ssh -t -t $remote "
     # Wait until ApplicationManager is available
-    for i in {1..10}; do
+    for i in {1..30}; do
         grep -q \" ApplicationManager .* Started\" \
             $ONOS_INSTALL_DIR/log/karaf.log* && break || sleep 1
     done
diff --git a/tools/test/scenarios/dist-setup.xml b/tools/test/scenarios/dist-setup.xml
index 85bce01..e28926a 100644
--- a/tools/test/scenarios/dist-setup.xml
+++ b/tools/test/scenarios/dist-setup.xml
@@ -22,6 +22,9 @@
             <step name="Distributed-Primitives-Setup.Check-Apps-${#}"
                   requires="Distributed-Primitives-Setup.Activate-Distributed-Primitives-Test-App"
                   exec="onos-check-apps ${OC#} distributedprimitives includes"/>
+            <step name="Distributed-Primitives-Setup.Wait-Apps-Cli-${#}"
+                  requires="Distributed-Primitives-Setup.Check-Apps-${#}"
+                  exec="onos-wait-for-command ${#} onos:set-test-get"/>
         </parallel>
     </group>
 </scenario>