Update HA Tests
- Add configuration for number of nodes to backup flow rules to
- Add some more conditions to exit the test early when failing
- Turn of flow objective intent compiler for more stability
Change-Id: Ie250c8e214b9f44d55abec95b0ef29839f0a4579
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params
index a77e100..71c964d 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params
@@ -22,8 +22,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
index 6ff14ce..94d5515 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
@@ -1858,6 +1858,8 @@
output = cli.summary()
if not output:
ready = False
+ if ready:
+ break
time.sleep( 30 )
utilities.assert_equals( expect=True, actual=ready,
onpass="ONOS summary command succeded",
@@ -2627,6 +2629,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params
index eb758d2..f79b913 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params
@@ -24,8 +24,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
index 1b83edf..0da776c 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
@@ -1793,6 +1793,8 @@
main.log.debug( "Checking logs for errors on " + node.name + ":" )
main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+ main.log.debug( main.CLIs[0].roles( jsonFormat=False ) )
+
n = len( main.nodes ) # Number of nodes
p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
main.partition = [ 0 ] # ONOS node to partition, listed by index in main.nodes
@@ -1870,6 +1872,27 @@
main.cleanup()
main.exit()
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ sleep=15,
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for i in main.activeNodes:
+ cli = main.CLIs[i]
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ cli.name,
+ cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
+
def CASE7( self, main ):
"""
Check state after ONOS failure
@@ -2611,6 +2634,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down
diff --git a/TestON/tests/HA/HAkillNodes/HAkillNodes.params b/TestON/tests/HA/HAkillNodes/HAkillNodes.params
index dd035f5..a83e878 100644
--- a/TestON/tests/HA/HAkillNodes/HAkillNodes.params
+++ b/TestON/tests/HA/HAkillNodes/HAkillNodes.params
@@ -24,8 +24,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAkillNodes/HAkillNodes.py b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
index c6aa669..c8df1f9 100644
--- a/TestON/tests/HA/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
@@ -1829,6 +1829,27 @@
onpass="ONOS nodes killed successfully",
onfail="ONOS nodes NOT successfully killed" )
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ sleep=15,
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for i in main.activeNodes:
+ cli = main.CLIs[i]
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ cli.name,
+ cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
+
def CASE62( self, main ):
"""
The bring up stopped nodes
@@ -1888,7 +1909,27 @@
main.restartTime = time.time() - restartTime
main.log.debug( "Restart time: " + str( main.restartTime ) )
# TODO: MAke this configurable. Also, we are breaking the above timer
- time.sleep( 60 )
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ sleep=15,
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for i in main.activeNodes:
+ cli = main.CLIs[i]
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ cli.name,
+ cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
+
node = main.activeNodes[0]
main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
@@ -2646,6 +2687,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down
diff --git a/TestON/tests/HA/HAsanity/HAsanity.params b/TestON/tests/HA/HAsanity/HAsanity.params
index 8c8b0f5..1821ff1 100644
--- a/TestON/tests/HA/HAsanity/HAsanity.params
+++ b/TestON/tests/HA/HAsanity/HAsanity.params
@@ -23,8 +23,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAsanity/HAsanity.py b/TestON/tests/HA/HAsanity/HAsanity.py
index b18cce8..c8ae7d9 100644
--- a/TestON/tests/HA/HAsanity/HAsanity.py
+++ b/TestON/tests/HA/HAsanity/HAsanity.py
@@ -586,6 +586,9 @@
"one or more ping pairs failed" )
main.log.info( "Time for pingall: %2f seconds" %
( time2 - time1 ) )
+ if not pingResult:
+ main.cleanup()
+ main.exit()
# timeout for fwd flows
time.sleep( 11 )
# uninstall onos-app-fwd
@@ -2547,6 +2550,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down
diff --git a/TestON/tests/HA/HAscaling/HAscaling.params b/TestON/tests/HA/HAscaling/HAscaling.params
index 388f432..74adfaa 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.params
+++ b/TestON/tests/HA/HAscaling/HAscaling.params
@@ -27,8 +27,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAscaling/HAscaling.py b/TestON/tests/HA/HAscaling/HAscaling.py
index 04000ba..b0bc126 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.py
+++ b/TestON/tests/HA/HAscaling/HAscaling.py
@@ -2686,6 +2686,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down
diff --git a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params
index 4e47832..d7ac493 100644
--- a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params
+++ b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.params
@@ -19,8 +19,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index c13b910..bd581ae 100644
--- a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -1854,6 +1854,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.params b/TestON/tests/HA/HAstopNodes/HAstopNodes.params
index dd035f5..a83e878 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.params
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.params
@@ -24,8 +24,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index 7617a7a..9b0f7a7 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -1808,6 +1808,27 @@
onpass="ONOS nodes stopped successfully",
onfail="ONOS nodes NOT successfully stopped" )
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ sleep=15,
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for i in main.activeNodes:
+ cli = main.CLIs[i]
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ cli.name,
+ cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
+
def CASE62( self, main ):
"""
The bring up stopped nodes
@@ -1866,8 +1887,27 @@
# protocol has had time to work
main.restartTime = time.time() - restartTime
main.log.debug( "Restart time: " + str( main.restartTime ) )
- # TODO: MAke this configurable. Also, we are breaking the above timer
- time.sleep( 60 )
+
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ sleep=15,
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for i in main.activeNodes:
+ cli = main.CLIs[i]
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ cli.name,
+ cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
node = main.activeNodes[0]
main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
@@ -2623,6 +2663,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.params b/TestON/tests/HA/HAswapNodes/HAswapNodes.params
index d3729a4..6cee74a 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.params
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.params
@@ -17,7 +17,7 @@
#CASE15: Check that Leadership Election is still functional
#CASE16: Install Distributed Primitives app
#CASE17: Check for basic functionality with distributed primitives
- <testcases>1,[2,8,21,3,8,4,5,14,16,17]*1,6,8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,[2,8,21,3,8,4,5,14,16,17]*1,6,[8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4]*1,13</testcases>
<server>
<port>8000</port>
@@ -26,8 +26,11 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>true</useFlowObjectives>
+ <useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.DistributedFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.DistributedFlowRuleStore>
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.py b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
index 1ced5a2..0065ecf 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.py
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
@@ -2682,6 +2682,10 @@
main.CLIs[i].name,
main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
def CASE9( self, main ):
"""
Link s3-s28 down