[ONOS-7594] Collect onos-diags when ping fails in SR test and other
minor fixes
Change-Id: I5ed5ab44264f7429bca746daad8c637e02094f41
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py b/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
index d1a8ee3..9280ca8 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/dependencies/SRBridgingTest.py
@@ -73,8 +73,4 @@
run.checkFlowsByDpid( main, dpid, self.topo[ topology ][ 4 ], sleep=5 )
run.pingAll( main )
- if hasattr( main, 'Mininet1' ):
- run.cleanup( main )
- else:
- # TODO: disconnect TestON from the physical network
- pass
+ run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params
index 778370e..a991004 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params
@@ -35,6 +35,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
+ <OnosDiscovery>12</OnosDiscovery>
</timers>
<kill>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/dependencies/SRDhcprelayTest.py b/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/dependencies/SRDhcprelayTest.py
index 1bbb0f9..9693add 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/dependencies/SRDhcprelayTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/dependencies/SRDhcprelayTest.py
@@ -72,8 +72,4 @@
run.verifyOnosHostIp( main )
run.verifyNetworkHostIp( main )
- if hasattr( main, 'Mininet1' ):
- run.cleanup( main )
- else:
- # TODO: disconnect TestON from the physical network
- pass
+ run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.params b/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.params
index 8beeccd..f15f207 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.params
@@ -35,6 +35,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
+ <OnosDiscovery>12</OnosDiscovery>
</timers>
<kill>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py
index 08ad86c..0341046 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/dependencies/SRDynamicConfTest.py
@@ -342,11 +342,7 @@
run.checkFlows( main, minFlowCount=minFlowCountPerLeaf * topo[ topology ][ 1 ], sleep=5, dumpflows=False )
run.pingAll( main, '%s_After' % TAG, retryAttempts=2 )
- if hasattr( main, 'Mininet1' ):
- run.cleanup( main )
- else:
- # TODO: disconnect TestON from the physical network
- pass
+ run.cleanup( main )
@staticmethod
def updateIntfCfg( main, portNum, dualHomed, ips=[], untagged=0, tagged=[], native=0 ):
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params
index 0870eab..577edac 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params
@@ -35,6 +35,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
+ <OnosDiscovery>12</OnosDiscovery>
</timers>
<switches>
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
index 778370e..a991004 100755
--- a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
@@ -35,6 +35,7 @@
<timers>
<LinkDiscovery>12</LinkDiscovery>
<SwitchDiscovery>12</SwitchDiscovery>
+ <OnosDiscovery>12</OnosDiscovery>
</timers>
<kill>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
index 399152d..947666d 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
@@ -34,8 +34,13 @@
</CTRL>
<timers>
- <LinkDiscovery>12</LinkDiscovery>
- <SwitchDiscovery>12</SwitchDiscovery>
+ <LinkDiscovery>30</LinkDiscovery>
+ <SwitchDiscovery>30</SwitchDiscovery>
+ <OnosDiscovery>30</OnosDiscovery>
+ <loadNetcfgSleep>5</loadNetcfgSleep>
+ <startMininetSleep>25</startMininetSleep>
+ <dhcpSleep>60</dhcpSleep>
+ <balanceMasterSleep>10</balanceMasterSleep>
</timers>
<SLEEP>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index df7c87d..0c65c1e 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -82,7 +82,7 @@
run.loadLinkFailureChart( main )
# wait some time
- time.sleep( 5 )
+ time.sleep( float( main.params[ 'timers' ][ 'loadNetcfgSleep' ] ) )
if hasattr( main, 'Mininet1' ):
# Run the test with Mininet
@@ -94,9 +94,9 @@
pass
# wait some time for onos to install the rules!
- time.sleep( 25 )
+ time.sleep( float( main.params[ 'timers' ][ 'startMininetSleep' ] ) )
if ( dhcp ):
- time.sleep( 60 )
+ time.sleep( float( main.params[ 'timers' ][ 'dhcpSleep' ] ) )
SRRoutingTest.runChecks( main, test_idx, countFlowsGroups )
@@ -131,24 +131,18 @@
for ctrl in xrange( numCtrls ):
# Kill node
run.killOnos( main, [ ctrl ], switches, links, ( numCtrls - 1 ) )
- time.sleep( float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) )
main.Cluster.active(0).CLI.balanceMasters()
- time.sleep( float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) )
+ time.sleep( float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) )
SRRoutingTest.runChecks( main, test_idx, countFlowsGroups )
# Recover node
run.recoverOnos( main, [ ctrl ], switches, links, numCtrls )
- time.sleep( float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) )
main.Cluster.active(0).CLI.balanceMasters()
- time.sleep( float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) )
+ time.sleep( float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) )
SRRoutingTest.runChecks( main, test_idx, countFlowsGroups )
# Cleanup
- if hasattr( main, 'Mininet1' ):
- run.cleanup( main )
- else:
- # TODO: disconnect TestON from the physical network
- pass
+ run.cleanup( main, copyKarafLog=False )
@staticmethod
def runChecks( main, test_idx, countFlowsGroups ):
@@ -159,4 +153,4 @@
if countFlowsGroups:
run.checkFlowsGroupsFromFile( main )
# ping hosts
- run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ run.pingAll( main, 'CASE%03d' % test_idx, False, acceptableFailed=5, basedOnIp=True, skipOnFail=True )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 980db4d..62982c3 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -128,7 +128,7 @@
@staticmethod
def loadCount( main ):
with open("%s/count/%s.count" % (main.configPath, main.cfgName)) as count:
- main.count = json.load(count)
+ main.count = json.load(count)
@staticmethod
def loadJson( main ):
@@ -237,6 +237,17 @@
onfail="Failed to assign switches to controllers" )
@staticmethod
+ def saveOnosDiagnostics( main ):
+ """
+ Get onos-diags.tar.gz and save it to the log directory.
+ suffix: suffix string of the file name. E.g. onos-diags-case1.tar.gz
+ """
+ main.log.info( "Collecting onos-diags..." )
+ main.ONOSbench.onosDiagnostics( [ctrl.ipAddress for ctrl in main.Cluster.runningNodes],
+ main.logdir,
+ "-CASE%d" % main.CurrentTestCaseNumber )
+
+ @staticmethod
def config( main, cfgName ):
main.spines = []
@@ -408,7 +419,7 @@
return
@staticmethod
- def pingAll( main, tag="", dumpflows=True, acceptableFailed=0, basedOnIp=False, sleep=10, retryAttempts=1 ):
+ def pingAll( main, tag="", dumpflows=True, acceptableFailed=0, basedOnIp=False, sleep=10, retryAttempts=1, skipOnFail=False ):
'''
Verify connectivity between hosts according to the ping chart
acceptableFailed: max number of acceptable failed pings.
@@ -472,6 +483,10 @@
utilities.assert_equals( expect=expect, actual=pa,
onpass="IP connectivity successfully tested",
onfail="IP connectivity failed" )
+ if skipOnFail and pa != expect:
+ Testcaselib.saveOnosDiagnostics( main )
+ Testcaselib.cleanup( main )
+ main.skipCase()
if dumpflows:
main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
@@ -650,7 +665,7 @@
onfail="Failed to recover switch?" )
@staticmethod
- def cleanup( main, physical=False):
+ def cleanup( main, copyKarafLog=True ):
"""
Stop Onos-cluster.
Stops Mininet
@@ -666,10 +681,11 @@
except ( NameError, AttributeError ):
main.utils = Utils()
- if not physical:
+ if hasattr( main, 'Mininet1' ):
main.utils.mininetCleanup( main.Mininet1 )
- main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
+ if copyKarafLog:
+ main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
for ctrl in main.Cluster.active():
main.ONOSbench.onosStop( ctrl.ipAddress )
@@ -682,6 +698,7 @@
Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
"""
main.step( "Killing ONOS instances with index(es): {}".format( nodes ) )
+ main.onosSleep = float( main.params[ 'timers' ][ 'OnosDiscovery' ] )
for i in nodes:
killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
@@ -689,7 +706,7 @@
onpass="ONOS instance Killed",
onfail="Error killing ONOS instance" )
main.Cluster.runningNodes[ i ].active = False
- time.sleep( 12 )
+ time.sleep( main.onosSleep )
if len( nodes ) < main.Cluster.numCtrls:
@@ -730,6 +747,7 @@
"""
main.step( "Recovering ONOS instances with index(es): {}".format( nodes ) )
[ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
+ time.sleep( main.onosSleep )
for i in nodes:
isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )
utilities.assert_equals( expect=main.TRUE, actual=isUp,