Port stability fixes to all tests
Change-Id: I1e712ad830a76f80ed9d6f3792786ed9d1ca8194
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HAclusterRestart/HAclusterRestart.py
index 8c5b1bb..234c995 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.py
@@ -48,6 +48,7 @@
start tcpdump
"""
import imp
+ import time
main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
@@ -250,6 +251,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
+ time.sleep(60)
appCheck = main.TRUE
threads = []
for i in range( main.numCtrls ):
@@ -1567,7 +1569,7 @@
if hosts[ controller ]:
for host in hosts[ controller ]:
if not host.get( 'ipAddresses', [ ] ):
- main.log.error( "DEBUG:Error with host ips on controller" +
+ main.log.error( "Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
utilities.assert_equals(
diff --git a/TestON/tests/HAkillNodes/HAkillNodes.py b/TestON/tests/HAkillNodes/HAkillNodes.py
index aa49941..396e08a 100644
--- a/TestON/tests/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HAkillNodes/HAkillNodes.py
@@ -49,6 +49,7 @@
start tcpdump
"""
import imp
+ import time
import pexpect
main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
"initialization" )
@@ -281,6 +282,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
+ time.sleep(60)
appCheck = main.TRUE
threads = []
for i in main.activeNodes:
@@ -1515,7 +1517,7 @@
# FIXME: better handling of this, print which node
# Maybe use thread name?
main.log.exception( "Error parsing json output of hosts" )
- # FIXME: should this be an empty json object instead?
+ main.log.warn( repr( t.result ) )
hosts.append( None )
ports = []
@@ -1561,7 +1563,7 @@
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
- if "Error" not in hosts[ controller ]:
+ if hosts[ controller ] and "Error" not in hosts[ controller ]:
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
@@ -1588,11 +1590,12 @@
ipResult = main.TRUE
for controller in range( 0, len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
- for host in hosts[ controller ]:
- if not host.get( 'ipAddresses', [ ] ):
- main.log.error( "Error with host ips on controller" +
- controllerStr + ": " + str( host ) )
- ipResult = main.FALSE
+ if hosts[ controller ]:
+ for host in hosts[ controller ]:
+ if not host.get( 'ipAddresses', [ ] ):
+ main.log.error( "Error with host ips on controller" +
+ controllerStr + ": " + str( host ) )
+ ipResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
actual=ipResult,
@@ -2188,13 +2191,13 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
- hosts.append( [] )
+ hosts.append( None )
for controller in range( 0, len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
for host in hosts[ controller ]:
if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ipAddresses on controller" +
+ "Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2389,7 +2392,7 @@
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
- if "Error" not in hosts[ controller ]:
+ if hosts[ controller ] or "Error" not in hosts[ controller ]:
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
diff --git a/TestON/tests/HAsanity/HAsanity.py b/TestON/tests/HAsanity/HAsanity.py
index 859d582..517d53e 100644
--- a/TestON/tests/HAsanity/HAsanity.py
+++ b/TestON/tests/HAsanity/HAsanity.py
@@ -49,6 +49,7 @@
start tcpdump
"""
import imp
+ import time
main.log.info( "ONOS HA Sanity test - initialization" )
main.case( "Setting up test environment" )
main.caseExplanation = "Setup the test environment including " +\
@@ -246,6 +247,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
+ time.sleep(60)
appCheck = main.TRUE
threads = []
for i in range( main.numCtrls ):
@@ -1471,7 +1473,7 @@
# FIXME: better handling of this, print which node
# Maybe use thread name?
main.log.exception( "Error parsing json output of hosts" )
- # FIXME: should this be an empty json object instead?
+ main.log.warn( repr( t.result ) )
hosts.append( None )
ports = []
@@ -1517,7 +1519,7 @@
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
- if "Error" not in hosts[ controller ]:
+ if hosts[ controller ] and "Error" not in hosts[ controller ]:
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
@@ -1544,11 +1546,12 @@
ipResult = main.TRUE
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
- for host in hosts[ controller ]:
- if not host.get( 'ipAddresses', [ ] ):
- main.log.error( "DEBUG:Error with host ips on controller" +
- controllerStr + ": " + str( host ) )
- ipResult = main.FALSE
+ if hosts[ controller ]:
+ for host in hosts[ controller ]:
+ if not host.get( 'ipAddresses', [ ] ):
+ main.log.error( "Error with host ips on controller" +
+ controllerStr + ": " + str( host ) )
+ ipResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
actual=ipResult,
@@ -2084,13 +2087,13 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
- hosts.append( [] )
+ hosts.append( None )
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ipAddresses on controller" +
+ "Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2286,7 +2289,7 @@
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
- if "Error" not in hosts[ controller ]:
+ if hosts[ controller ] or "Error" not in hosts[ controller ]:
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
diff --git a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index 728b9ab..2f2b01b 100644
--- a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -48,6 +48,7 @@
start tcpdump
"""
import imp
+ import time
main.log.info( "ONOS Single node cluster restart " +
"HA test - initialization" )
main.case( "Setting up test environment" )
@@ -219,6 +220,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
+ time.sleep(60)
appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( main.CLIs[0].apps() )
diff --git a/TestON/tests/HAstopNodes/HAstopNodes.py b/TestON/tests/HAstopNodes/HAstopNodes.py
index 47689af..84b714d 100644
--- a/TestON/tests/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HAstopNodes/HAstopNodes.py
@@ -50,6 +50,7 @@
"""
import imp
import pexpect
+ import time
main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
@@ -274,6 +275,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
+ time.sleep(60)
appCheck = main.TRUE
threads = []
for i in main.activeNodes:
@@ -1504,7 +1506,7 @@
# FIXME: better handling of this, print which node
# Maybe use thread name?
main.log.exception( "Error parsing json output of hosts" )
- # FIXME: should this be an empty json object instead?
+ main.log.warn( repr( t.result ) )
hosts.append( None )
ports = []
@@ -1550,7 +1552,7 @@
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
- if "Error" not in hosts[ controller ]:
+ if hosts[ controller ] and "Error" not in hosts[ controller ]:
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
@@ -1577,11 +1579,12 @@
ipResult = main.TRUE
for controller in range( 0, len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
- for host in hosts[ controller ]:
- if not host.get( 'ipAddresses', [ ] ):
- main.log.error( "Error with host ips on controller" +
- controllerStr + ": " + str( host ) )
- ipResult = main.FALSE
+ if hosts[ controller ]:
+ for host in hosts[ controller ]:
+ if not host.get( 'ipAddresses', [ ] ):
+ main.log.error( "Error with host ips on controller" +
+ controllerStr + ": " + str( host ) )
+ ipResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
actual=ipResult,
@@ -2177,13 +2180,13 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
- hosts.append( [] )
+ hosts.append( None )
for controller in range( 0, len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
for host in hosts[ controller ]:
if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ipAddresses on controller" +
+ "Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2378,7 +2381,7 @@
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
- if "Error" not in hosts[ controller ]:
+ if hosts[ controller ] or "Error" not in hosts[ controller ]:
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent