Added tool to verify that ONOS server has stopped; added HA cluster restart scenario.
Change-Id: I534eb75003a88c03ec2fe74d29fd56e64ffacc35
diff --git a/tools/test/scenarios/ha-cluster-restart.xml b/tools/test/scenarios/ha-cluster-restart.xml
new file mode 100644
index 0000000..16b4b27
--- /dev/null
+++ b/tools/test/scenarios/ha-cluster-restart.xml
@@ -0,0 +1,62 @@
+<!--
+ ~ Copyright 2016-present Open Networking Laboratory
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<scenario name="ha-cluster-restart"
+ description="ONOS cluster restart and recovery">
+ <group name="Cluster-Restart" if="${OC2}">
+ <group name="Fail">
+ <parallel var="${OC#}">
+ <step name="Node-Stop-${#}" exec="onos-service ${OC#} stop"/>
+ <step name="Wait-for-Stop-${#}" requires="~Node-Stop-${#}"
+ exec="onos-wait-for-stop ${OC#}"/>
+ </parallel>
+ </group>
+
+ <group name="Recover" requires="~Fail">
+ <parallel var="${OC#}">
+ <step name="Node-Start-${#}" exec="onos-service ${OC#} start"/>
+ <step name="Wait-for-Start-${#}" requires="~Node-Start-${#}"
+ exec="onos-wait-for-start ${OC#}"/>
+ </parallel>
+ </group>
+
+ <group name="Validate-Recovery" requires="~Recover">
+ <parallel var="${OC#}">
+ <step name="Check-Nodes-${#}"
+ exec="onos-check-nodes ${OC#}"/>
+
+ <step name="Check-Components-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-components ${OC#}"/>
+
+ <step name="Check-Logs-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-logs --ignore-store-exceptions ${OC#}"/>
+ <step name="Check-Apps-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-apps ${OC#} ${ONOS_APPS} includes"/>
+
+ <step name="Check-Summary-${#}" requires="~Check-Nodes-${#},~Check-Apps-${#}"
+ exec="onos-check-summary ${OC#} [0-9]* ${OTD} ${OTL} ${OTH}"/>
+ </parallel>
+ </group>
+
+ <step name="Balance-Masters-Again" requires="~Validate-Recovery"
+ exec="onos ${OC1} balance-masters"/>
+ </group>
+
+ <group name="Cluster-Restart-NA" unless="${OC2}">
+ <step name="No-Op" exec="echo Scenario not supported for this cell size"/>
+ </group>
+
+</scenario>