[ONOS-7221] Upgrade state tests
Change-Id: I0e9214ddcb272dc98bb8f54fa600704c38e3562d
diff --git a/tools/test/bin/onos-execute-expect b/tools/test/bin/onos-execute-expect
index 4b9270e..0413522 100755
--- a/tools/test/bin/onos-execute-expect
+++ b/tools/test/bin/onos-execute-expect
@@ -1,27 +1,46 @@
#!/bin/bash
# -----------------------------------------------------------------------------
# Executes a command on the given ONOS instance and matches the output
-# to the passed one.
+# to the passed after X retries.
# First argument is the IP address of the machine to run the command on,
# then you pass the command and it's arguments if needed, then --expect and
-# after it the string of what the output should be.
+# after it the string of what the output should be or --retry and the maximum
+# retries with 5 seconds delay between them
# Example:
-# onos-execute-expect 1.1.1.1 fooCommand fooParamenter --expect fooOutputString
+# onos-execute-expect 1.1.1.1 fooCommand fooParamenter --retry 5 --expect fooOutputString
# -----------------------------------------------------------------------------
[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
. $ONOS_ROOT/tools/build/envDefaults
-
aux=/tmp/stc/stc-$$.log
trap "rm -f $aux 2>/dev/null" EXIT
-ip=$1
+
+retry=0
cmd=""
-for a in ${*:2}; do shift; if [ "$a" = "--expect" ]; then break; fi; cmd="$cmd $a"; done
-expect="${@: -1}"
-onos $ip $cmd > $aux
-cat $aux
-grep -q $expect $aux && echo "expected value found" && exit 0
+expct=""
+
+for ((i=1; i<=$#; i++))
+do
+ if [ "${!i}" = "--expect" ]; then
+ next=$((i+1))
+ expct="${!next}"
+ i=$i+1
+ elif [ "${!i}" = "--retry" ]; then
+ next=$((i+1))
+ retry="${!next}"
+ i=$i+1
+ else
+ cmd="$cmd ${!i}";
+ fi
+done
+
+until [ $retry -eq -1 ]
+do
+ onos $cmd > $aux
+ cat $aux
+ grep -q $expct $aux && echo "expected value found" && exit 0
+ retry=$[$retry-1]
+ sleep 5
+done
exit 1
-
-
diff --git a/tools/test/scenarios/upgrade-rollback.xml b/tools/test/scenarios/upgrade-rollback.xml
index 05841bb..13f4403 100644
--- a/tools/test/scenarios/upgrade-rollback.xml
+++ b/tools/test/scenarios/upgrade-rollback.xml
@@ -23,6 +23,10 @@
<step name="Initialize-ECM"
exec="onos ${OC1} ec-map-test foo put a b"/>
+ <step name="Initialize-CM"
+ exec="onos ${OC1} map-test foo put a b"
+ requires="Initialize-ECM"/>
+
<step name="Initialize-Upgrade"
exec="onos ${OC1} issu init"
requires="Initialize-ECM"/>
@@ -65,8 +69,12 @@
requires="Phase-One-Wait-for-Start-${#}"/>
<step name="Phase-One-Check-ECM-${#}"
- exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect b"
+ exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect b"
requires="Phase-One-Distributed-Primitives-Check-Apps-${#}"/>
+
+ <step name="Phase-One-Check-CM-${#}"
+ exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect b"
+ requires="Phase-One-Check-ECM-${#}"/>
</sequential>
</group>
@@ -74,19 +82,31 @@
<step name="Change-Majority-ECM"
exec="onos ${OCMA1} ec-map-test foo put a d"/>
+ <step name="Change-Majority-CM"
+ exec="onos ${OCMA1} map-test foo put a d"/>
+
<step name="Change-Minority-ECM"
exec="onos ${OCMI1} ec-map-test foo put a c"/>
+
+ <step name="Change-Minority-CM"
+ exec="onos ${OCMI1} map-test foo put a c"/>
</group>
<group name="Check-ECM-values" requires="Change-ECM-values">
<parallel var="${OCMI#}">
<step name="Parallel-Check-Minority-ECM-${#}"
- exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect c"/>
+ exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect c"/>
+
+ <step name="Parallel-Check-Minority-CM-${#}"
+ exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect c"/>
</parallel>
<parallel var="${OCMA#}">
<step name="Parallel-Check-Majority-ECM-${#}"
- exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --expect d"/>
+ exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --retry 5 --expect d"/>
+
+ <step name="Parallel-Check-Majority-CM-${#}"
+ exec="onos-execute-expect ${OCMA#} map-test foo get a --retry 5 --expect d"/>
</parallel>
</group>
@@ -136,8 +156,12 @@
requires="Phase-Two-Wait-for-Start-${#}"/>
<step name="Phase-Two-Check-ECM-${#}"
- exec="onos-execute-expect ${OC#} ec-map-test foo get a --expect d"
+ exec="onos-execute-expect ${OC#} ec-map-test foo get a --retry 5 --expect d"
requires="Phase-Two-Distributed-Primitives-Check-Apps-${#}"/>
+
+ <step name="Phase-Two-Check-CM-${#}"
+ exec="onos-execute-expect ${OC#} map-test foo get a --retry 5 --expect d"
+ requires="Phase-Two-Check-ECM-${#}"/>
</sequential>
</group>
@@ -149,8 +173,7 @@
<parallel var="${OC#}">
<step name="Check-Nodes-${#}"
exec="onos-check-nodes ${OC#}"
- delay="3"
- requires="Verify-EC-Map-${#}"/>
+ delay="3"/>
<step name="Check-Components-${#}"
exec="onos-check-components ${OC#}"
diff --git a/tools/test/scenarios/upgrade.xml b/tools/test/scenarios/upgrade.xml
index 16fc627..0890ded 100644
--- a/tools/test/scenarios/upgrade.xml
+++ b/tools/test/scenarios/upgrade.xml
@@ -23,9 +23,13 @@
<step name="Initialize-ECM"
exec="onos ${OC1} ec-map-test foo put a b"/>
+ <step name="Initialize-CM"
+ exec="onos ${OC1} map-test foo put a b"
+ requires="Initialize-ECM"/>
+
<step name="Initialize-Upgrade"
exec="onos ${OC1} issu init"
- requires="Initialize-ECM"/>
+ requires="Initialize-CM"/>
<group name="Phase-1">
<sequential var="${OCMI#}"
@@ -65,8 +69,12 @@
requires="Phase-One-Wait-for-Start-${#}"/>
<step name="Phase-One-Check-ECM-${#}"
- exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect b"
+ exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect b"
requires="Phase-One-Distributed-Primitives-Check-Apps-${#}"/>
+
+ <step name="Phase-One-Check-CM-${#}"
+ exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect b"
+ requires="Phase-One-Check-ECM-${#}"/>
</sequential>
</group>
@@ -74,19 +82,31 @@
<step name="Change-Majority-ECM"
exec="onos ${OCMA1} ec-map-test foo put a d"/>
+ <step name="Change-Majority-CM"
+ exec="onos ${OCMA1} map-test foo put a d"/>
+
<step name="Change-Minority-ECM"
exec="onos ${OCMI1} ec-map-test foo put a c"/>
+
+ <step name="Change-Minority-CM"
+ exec="onos ${OCMI1} map-test foo put a c"/>
</group>
<group name="Check-ECM-values" requires="Change-ECM-values">
<parallel var="${OCMI#}">
<step name="Parallel-Check-Minority-ECM-${#}"
- exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect c"/>
+ exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect c"/>
+
+ <step name="Parallel-Check-Minority-CM-${#}"
+ exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect c"/>
</parallel>
<parallel var="${OCMA#}">
<step name="Parallel-Check-Majority-ECM-${#}"
- exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --expect d"/>
+ exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --retry 5 --expect d"/>
+
+ <step name="Parallel-Check-Majority-CM-${#}"
+ exec="onos-execute-expect ${OCMA#} map-test foo get a --retry 5 --expect d"/>
</parallel>
</group>
@@ -132,8 +152,12 @@
requires="Phase-Two-Wait-for-Start-${#}"/>
<step name="Phase-Two-Check-ECM-${#}"
- exec="onos-execute-expect ${OC#} ec-map-test foo get a --expect c"
+ exec="onos-execute-expect ${OC#} ec-map-test foo get a --retry 5 --expect c"
requires="Phase-Two-Distributed-Primitives-Check-Apps-${#}"/>
+
+ <step name="Phase-Two-Check-CM-${#}"
+ exec="onos-execute-expect ${OC#} map-test foo get a --retry 5 --expect c"
+ requires="Phase-Two-Check-ECM-${#}"/>
</sequential>
</group>