Add test scenario for crashing all nodes in sequence
Change-Id: Idd3c87ff8155a3c2b612b0447428e0af7d78a756
diff --git a/tools/test/bin/onos-check-all-logs b/tools/test/bin/onos-check-all-logs
new file mode 100755
index 0000000..7648710
--- /dev/null
+++ b/tools/test/bin/onos-check-all-logs
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+function onos_nodes() {
+ echo $(env | sort | egrep "^OC[0-9]+" | cut -d= -f2)
+}
+
+nodes=$(onos_nodes)
+for node in ${nodes}; do
+ onos-check-logs $@ $node || exit 1
+done
\ No newline at end of file
diff --git a/tools/test/bin/onos-check-node-status b/tools/test/bin/onos-check-node-status
index 0a76f08..0cd4688 100755
--- a/tools/test/bin/onos-check-node-status
+++ b/tools/test/bin/onos-check-node-status
@@ -6,15 +6,23 @@
aux=/tmp/stc/stc-$$.log
trap "rm -f $aux 2>/dev/null" EXIT
-for attempt in {1..10}; do
- onos ${1:-$OCI} "onos:nodes" > $aux
- cat $aux
+function onos_nodes() {
+ echo $(env | sort | egrep "^OC[0-9]+" | cut -d= -f2)
+}
- # Normalize the node status
- state=$(grep ${2:-$OC2} $aux | cut -d, -f3 | cut -d= -f2)
+nodes=$(onos_nodes)
- [ "$state" = "${3:-READY}" ] && exit 0
- sleep 1
+for attempt in {1..5}; do
+ for node in $nodes; do
+ onos $node "onos:nodes" > $aux
+ cat $aux
+
+ # Normalize the node status
+ state=$(grep ${1} $aux | cut -d, -f3 | cut -d= -f2)
+
+ [ "$state" = "${2:-READY}" ] && exit 0
+ sleep 1
+ done
done
exit 1
\ No newline at end of file
diff --git a/tools/test/scenarios/bin/check-masters.py b/tools/test/scenarios/bin/check-masters.py
new file mode 100755
index 0000000..dfe4df6
--- /dev/null
+++ b/tools/test/scenarios/bin/check-masters.py
@@ -0,0 +1,34 @@
+#! /usr/bin/env python
+
+import json
+import subprocess
+import sys
+import time
+
+if len(sys.argv) != 2:
+ print "usage: check-masters {node}"
+ sys.exit(1)
+
+node = sys.argv[1]
+
+def check_masters():
+ nodes_json = json.loads(subprocess.check_output(["onos", node, "nodes", "-j"]))
+ inactive_nodes = [n['id'] for n in nodes_json if n['state'] != 'READY']
+
+ masters_json = json.loads(subprocess.check_output(["onos", node, "masters", "-j"]))
+ masters = dict([(m['id'], m['devices']) for m in masters_json])
+
+ for inactive_node in inactive_nodes:
+ if inactive_node in masters:
+ devices = masters[inactive_node]
+ if len(devices) > 0:
+ return False
+ return True
+
+for i in range(10):
+ if not check_masters():
+ time.sleep(1)
+ else:
+ sys.exit(0)
+
+sys.exit(1)
diff --git a/tools/test/scenarios/ha-multi-node.xml b/tools/test/scenarios/ha-multi-node.xml
new file mode 100644
index 0000000..6b0ecae
--- /dev/null
+++ b/tools/test/scenarios/ha-multi-node.xml
@@ -0,0 +1,71 @@
+<!--
+ ~ Copyright 2017-present Open Networking Foundation
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<scenario name="ha-multi-node"
+ description="ONOS cluster multiple node failure and recovery">
+ <group name="Multi-Node-Failure" if="${OC2}">
+ <sequential var="${OC#}"
+ starts="Node-Death-${#}"
+ ends="Check-All-Logs-${#-1}">
+ <step name="Node-Death-${#}"
+ exec="onos-power ${OC#} off"/>
+
+ <step name="Validate-Death-${#}"
+ requires="Node-Death-${#}"
+ exec="onos-check-node-status ${OC#} INACTIVE"/>
+
+ <step name="Breathe-${#}"
+ requires="Validate-Death-${#}"
+ exec="sleep 5"/>
+
+ <step name="Node-Start-${#}"
+ requires="Breathe-${#}"
+ exec="onos-power ${OC#} on"/>
+
+ <step name="Wait-for-Start-${#}"
+ requires="~Node-Start-${#}"
+ exec="onos-wait-for-start ${OC#}"/>
+
+ <step name="Check-Nodes-${#}"
+ requires="Wait-for-Start-${#}"
+ exec="onos-check-nodes ${OC#}"/>
+
+ <step name="Check-Masters-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="check-masters.py ${OC#}"/>
+
+ <step name="Check-Components-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="onos-check-components ${OC#}"/>
+
+ <step name="Check-Apps-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="onos-check-apps ${OC#} ${ONOS_APPS} includes"/>
+
+ <step name="Check-Flows-${#}"
+ requires="~Check-Nodes-${#}"
+ exec="onos-check-flows ${OC#}"/>
+
+ <step name="Check-All-Logs-${#}"
+ requires="~Check-Nodes-${#},~Check-Masters-${#},~Check-Components-${#},~Check-Apps-${#},~Check-Flows-${#}"
+ exec="onos-check-all-logs --ignore-store-exceptions"/>
+ </sequential>
+ </group>
+
+ <group name="Multi-Node-NA" unless="${OC2}">
+ <step name="No-Op" exec="echo Scenario not supported for this cell size"/>
+ </group>
+</scenario>
diff --git a/tools/test/scenarios/ha-single-node.xml b/tools/test/scenarios/ha-single-node.xml
index 27a0e4e..b05d6e3 100644
--- a/tools/test/scenarios/ha-single-node.xml
+++ b/tools/test/scenarios/ha-single-node.xml
@@ -19,7 +19,7 @@
<group name="Single-Node-Failure" if="${OC2}">
<step name="Node-Death" exec="onos-power ${OC1} off"/>
<step name="Validate-Death" requires="Node-Death"
- exec="onos-check-node-status ${OC2} ${OC1} INACTIVE"/>
+ exec="onos-check-node-status ${OC1} INACTIVE"/>
<group name="Validate-Normal-Operation" requires="Validate-Death">
<step name="Check-Summary"