Adding an STC scenario to test basic HA functionality
Change-Id: Ic70b592bf932dcdcd4eb88606d0b0aa9ef9c4a59
diff --git a/tools/dev/bash_profile b/tools/dev/bash_profile
index 79f80f2..a288765 100644
--- a/tools/dev/bash_profile
+++ b/tools/dev/bash_profile
@@ -105,6 +105,10 @@
alias sshctl='onos-ssh'
alias sshnet='onos-ssh $OCN'
+# Runs a sequence of STC scenarios until the first failure.
+function stcs {
+ for s in "$@"; do stc $s || return 1; done
+}
# Applies the settings in the specified topology recipe file or lists current
# topo recipe definition if no topo recipe file is given.
diff --git a/tools/test/bin/ogroup-opts b/tools/test/bin/ogroup-opts
index cdc623f..b15ced9 100644
--- a/tools/test/bin/ogroup-opts
+++ b/tools/test/bin/ogroup-opts
@@ -22,6 +22,7 @@
complete -F _stc-opts stc
complete -F _stc-opts stw
+complete -F _stc-opts stcs
# Tab completion settings for stl
diff --git a/tools/test/bin/onos-check-node-status b/tools/test/bin/onos-check-node-status
new file mode 100755
index 0000000..ba86703
--- /dev/null
+++ b/tools/test/bin/onos-check-node-status
@@ -0,0 +1,20 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Checks whether the specified ONOS cluster node has the desired state.
+# -----------------------------------------------------------------------------
+
+aux=/tmp/stc-$$.log
+trap "rm -f $aux 2>/dev/null" EXIT
+
+for attempt in {1..10}; do
+ onos ${1:-$OCI} "onos:nodes" | grep -v /bin/client > $aux
+ cat $aux
+
+ # Normalize the node status
+ state=$(grep ${2:-$OC2} $aux | cut -d, -f3 | cut -d= -f2)
+
+ [ "$state" = "${3:-READY}" ] && exit 0
+ sleep 1
+done
+
+exit 1
\ No newline at end of file
diff --git a/tools/test/bin/onos-mininet b/tools/test/bin/onos-mininet
index 9079e88..7f1d276 100755
--- a/tools/test/bin/onos-mininet
+++ b/tools/test/bin/onos-mininet
@@ -65,6 +65,8 @@
$mininet -X "stuff \"^C\\n\"" 2>/dev/null && \
$mininet -X "stuff \"^C\\n\"" 2>/dev/null && \
$mininet -X "stuff \"exit\\n\"" 2>/dev/null
+ ssh -t -t $remote sudo mn -c
+
;;
cleanup)
ssh -t -t $remote sudo mn -c
diff --git a/tools/test/scenarios/ha-single-node.xml b/tools/test/scenarios/ha-single-node.xml
new file mode 100644
index 0000000..554dcac
--- /dev/null
+++ b/tools/test/scenarios/ha-single-node.xml
@@ -0,0 +1,74 @@
+<!--
+ ~ Copyright 2016-present Open Networking Laboratory
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<scenario name="ha-single-node"
+ description="ONOS cluster single node failure and recovery">
+ <group name="Single-Node-Failure" if="${OC2}">
+ <step name="Node-Death" exec="onos-die ${OC1}"/>
+
+ <step name="Validate-Death" requires="Node-Death"
+ exec="onos-check-node-status ${OC2} ${OC1} INACTIVE"/>
+
+ <group name="Validate-Normal-Operation">
+ <step name="Check-Summary" requires="~Validate-Death"
+ exec="onos-check-summary ${OC2} [0-9]* ${OTD} ${OTL} ${OTH}"/>
+
+ <step name="Balance-Masters" requires="~Check-Summary"
+ exec="onos ${OC2} balance-masters"/>
+
+ <step name="Check-Summary-Again" requires="~Balance-Masters" delay="5"
+ exec="onos-check-summary ${OC2} [0-9]* ${OTD} ${OTL} ${OTH}"/>
+
+ <step name="Check-Flows" requires="~Balance-Masters" delay="5"
+ exec="onos-check-flows ${OC2}"/>
+ </group>
+
+ <group name="Recover" requires="~Validate-Normal-Operation">
+ <step name="Node-Start"
+ exec="onos-service ${OC1} start"/>
+
+ <step name="Wait-for-Start" requires="~Node-Start"
+ exec="onos-wait-for-start ${OC1}"/>
+ </group>
+
+ <group name="Validate-Recovery" requires="~Recover">
+ <parallel var="${OC#}">
+ <step name="Check-Nodes-${#}"
+ exec="onos-check-nodes ${OC#}"/>
+
+ <step name="Check-Components-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-components ${OC#}"/>
+
+ <step name="Check-Logs-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-logs ${OC#}"/>
+ <step name="Check-Apps-${#}" requires="~Check-Nodes-${#}"
+ exec="onos-check-apps ${OC#} ${ONOS_APPS} includes"/>
+
+ <step name="Check-Summary-${#}" requires="~Check-Nodes-${#},~Check-Apps-${#}"
+ exec="onos-check-summary ${OC#} [0-9]* ${OTD} ${OTL} ${OTH}"/>
+ </parallel>
+ </group>
+
+ <step name="Balance-Masters-Again" requires="~Validate-Recovery"
+ exec="onos ${OC1} balance-masters"/>
+ </group>
+
+ <group name="Single-Node-NA" unless="${OC2}">
+ <step name="No-Op" exec="echo Scenario not supported for this cell size"/>
+ </group>
+
+
+</scenario>