Added tool to verify that ONOS server has stopped; added HA cluster restart scenario.
Change-Id: I534eb75003a88c03ec2fe74d29fd56e64ffacc35
diff --git a/tools/test/bin/onos-wait-for-stop b/tools/test/bin/onos-wait-for-stop
new file mode 100755
index 0000000..a684c23
--- /dev/null
+++ b/tools/test/bin/onos-wait-for-stop
@@ -0,0 +1,21 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Waits for ONOS java process to terminate.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+. $ONOS_ROOT/tools/test/bin/find-node.sh
+
+node=$(find_node ${1:-$OCI})
+
+remote=$ONOS_USER@$node
+
+ssh -t $remote "
+ # Wait until ONOS Java process terminates
+ for i in \$(seq 1 45); do
+ ps -ef | egrep 'java .*/onos/.* org\.apache\.karaf\.main\.Main' | grep -qv grep || exit 0
+ sleep 1
+ done
+ exit 1
+"
diff --git a/tools/test/scenarios/ha-cluster-restart.xml b/tools/test/scenarios/ha-cluster-restart.xml
new file mode 100644
index 0000000..16b4b27
--- /dev/null
+++ b/tools/test/scenarios/ha-cluster-restart.xml
@@ -0,0 +1,62 @@
+<!--
+ ~ Copyright 2016-present Open Networking Laboratory
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<scenario name="ha-cluster-restart"
+ description="ONOS cluster restart and recovery">
+ <group name="Cluster-Restart" if="${OC2}">
+ <group name="Fail">
+ <parallel var="${OC#}">
+ <step name="Node-Stop-${#}" exec="onos-service ${OC#} stop"/>
+ <step name="Wait-for-Stop-${#}" requires="~Node-Stop-${#}"
+ exec="onos-wait-for-stop ${OC#}"/>
+ </parallel>
+ </group>
+
+ <group name="Recover" requires="~Fail">
+ <parallel var="${OC#}">
+ <step name="Node-Start-${#}" exec="onos-service ${OC#} start"/>
+ <step name="Wait-for-Start-${#}" requires="~Node-Start-${#}"
+ exec="onos-wait-for-start ${OC#}"/>
+ </parallel>
+ </group>
+
+ <group name="Validate-Recovery" requires="~Recover">
+ <parallel var="${OC#}">
+ <step name="Check-Nodes-${#}"
+ exec="onos-check-nodes ${OC#}"/>
+
+ <step name="Check-Components-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-components ${OC#}"/>
+
+ <step name="Check-Logs-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-logs --ignore-store-exceptions ${OC#}"/>
+ <step name="Check-Apps-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-apps ${OC#} ${ONOS_APPS} includes"/>
+
+ <step name="Check-Summary-${#}" requires="~Check-Nodes-${#},~Check-Apps-${#}"
+ exec="onos-check-summary ${OC#} [0-9]* ${OTD} ${OTL} ${OTH}"/>
+ </parallel>
+ </group>
+
+ <step name="Balance-Masters-Again" requires="~Validate-Recovery"
+ exec="onos ${OC1} balance-masters"/>
+ </group>
+
+ <group name="Cluster-Restart-NA" unless="${OC2}">
+ <step name="No-Op" exec="echo Scenario not supported for this cell size"/>
+ </group>
+
+</scenario>
diff --git a/tools/test/scenarios/shutdown.xml b/tools/test/scenarios/shutdown.xml
index b52475b..935145e 100644
--- a/tools/test/scenarios/shutdown.xml
+++ b/tools/test/scenarios/shutdown.xml
@@ -17,8 +17,10 @@
<group name="Shutdown">
<parallel var="${OC#}">
<step name="Stop-Service-${#}" exec="onos-service ${OC#} stop"/>
- <step name="Check-Logs-${#}" exec="onos-check-logs ${OC#}"
+ <step name="Wait-for-Stop-${#}" exec="onos-wait-for-stop ${OC#}"
requires="~Stop-Service-${#}"/>
+ <step name="Check-Logs-${#}" exec="onos-check-logs ${OC#}"
+ requires="~Wait-for-Stop-${#}"/>
</parallel>
</group>
</scenario>