Add test scenario for crashing all nodes in sequence
Change-Id: Idd3c87ff8155a3c2b612b0447428e0af7d78a756
diff --git a/tools/test/scenarios/bin/check-masters.py b/tools/test/scenarios/bin/check-masters.py
new file mode 100755
index 0000000..dfe4df6
--- /dev/null
+++ b/tools/test/scenarios/bin/check-masters.py
@@ -0,0 +1,34 @@
+#! /usr/bin/env python
+
+import json
+import subprocess
+import sys
+import time
+
+if len(sys.argv) != 2:
+ print "usage: check-masters {node}"
+ sys.exit(1)
+
+node = sys.argv[1]
+
+def check_masters():
+ nodes_json = json.loads(subprocess.check_output(["onos", node, "nodes", "-j"]))
+ inactive_nodes = [n['id'] for n in nodes_json if n['state'] != 'READY']
+
+ masters_json = json.loads(subprocess.check_output(["onos", node, "masters", "-j"]))
+ masters = dict([(m['id'], m['devices']) for m in masters_json])
+
+ for inactive_node in inactive_nodes:
+ if inactive_node in masters:
+ devices = masters[inactive_node]
+ if len(devices) > 0:
+ return False
+ return True
+
+for i in range(10):
+ if not check_masters():
+ time.sleep(1)
+ else:
+ sys.exit(0)
+
+sys.exit(1)
diff --git a/tools/test/scenarios/ha-multi-node.xml b/tools/test/scenarios/ha-multi-node.xml
new file mode 100644
index 0000000..6b0ecae
--- /dev/null
+++ b/tools/test/scenarios/ha-multi-node.xml
@@ -0,0 +1,71 @@
+<!--
+ ~ Copyright 2017-present Open Networking Foundation
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<scenario name="ha-multi-node"
+ description="ONOS cluster multiple node failure and recovery">
+ <group name="Multi-Node-Failure" if="${OC2}">
+ <sequential var="${OC#}"
+ starts="Node-Death-${#}"
+ ends="Check-All-Logs-${#-1}">
+ <step name="Node-Death-${#}"
+ exec="onos-power ${OC#} off"/>
+
+ <step name="Validate-Death-${#}"
+ requires="Node-Death-${#}"
+ exec="onos-check-node-status ${OC#} INACTIVE"/>
+
+ <step name="Breathe-${#}"
+ requires="Validate-Death-${#}"
+ exec="sleep 5"/>
+
+ <step name="Node-Start-${#}"
+ requires="Breathe-${#}"
+ exec="onos-power ${OC#} on"/>
+
+ <step name="Wait-for-Start-${#}"
+ requires="~Node-Start-${#}"
+ exec="onos-wait-for-start ${OC#}"/>
+
+ <step name="Check-Nodes-${#}"
+ requires="Wait-for-Start-${#}"
+ exec="onos-check-nodes ${OC#}"/>
+
+ <step name="Check-Masters-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="check-masters.py ${OC#}"/>
+
+ <step name="Check-Components-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="onos-check-components ${OC#}"/>
+
+ <step name="Check-Apps-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="onos-check-apps ${OC#} ${ONOS_APPS} includes"/>
+
+ <step name="Check-Flows-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="onos-check-flows ${OC#}"/>
+
+ <step name="Check-All-Logs-${#}"
+ requires="~Check-Nodes-${#},~Check-Masters-${#},~Check-Components-${#},~Check-Apps-${#},~Check-Flows-${#}"
+ exec="onos-check-all-logs --ignore-store-exceptions"/>
+ </sequential>
+ </group>
+
+ <group name="Multi-Node-NA" unless="${OC2}">
+ <step name="No-Op" exec="echo Scenario not supported for this cell size"/>
+ </group>
+</scenario>
diff --git a/tools/test/scenarios/ha-single-node.xml b/tools/test/scenarios/ha-single-node.xml
index 27a0e4e..b05d6e3 100644
--- a/tools/test/scenarios/ha-single-node.xml
+++ b/tools/test/scenarios/ha-single-node.xml
@@ -19,7 +19,7 @@
<group name="Single-Node-Failure" if="${OC2}">
<step name="Node-Death" exec="onos-power ${OC1} off"/>
<step name="Validate-Death" requires="Node-Death"
- exec="onos-check-node-status ${OC2} ${OC1} INACTIVE"/>
+ exec="onos-check-node-status ${OC1} INACTIVE"/>
<group name="Validate-Normal-Operation" requires="Validate-Death">
<step name="Check-Summary"