Fix some stability issues in HA Tests
- Add some more output when things fail
- Protect against some type errors when function calls fail
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 2933c93..ff38cfe 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -119,7 +119,7 @@
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
- main.ONOSbench.getVersion( report=True )
+ main.ONOSbench.getVersion( report=True )
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
@@ -439,19 +439,36 @@
str( hex( i )[ 2: ] ).zfill( 2 ).upper()
host2 = "00:00:00:00:00:" + \
str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
- host1Id = main.ONOScli1.getHost( host1 )[ 'id' ]
- host2Id = main.ONOScli1.getHost( host2 )[ 'id' ]
- # NOTE: get host can return None
+ # NOTE: getHost can return None
+ host1Dict = main.ONOScli1.getHost( host1 )
+ host2Dict = main.ONOScli1.getHost( host2 )
+ host1Id = None
+ host2Id = None
+ if host1Dict and host2Dict:
+ host1Id = host1Dict.get( 'id', None )
+ host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
tmpResult = main.ONOScli1.addHostIntent(
host1Id,
host2Id )
else:
main.log.error( "Error, getHost() failed" )
+ main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
tmpResult = main.FALSE
intentAddResult = bool( pingResult and intentAddResult
and tmpResult )
# TODO Check that intents were added?
+ # Print the intent states
+ intents = main.ONOScli1.intents( )
+ intentStates = []
+ for intent in json.loads( intents ): # Iter through intents of a node
+ intentStates.append( intent.get( 'state', None ) )
+ out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
+ main.log.info( dict( out ) )
+
utilities.assert_equals(
expect=True,
actual=intentAddResult,
@@ -1293,6 +1310,24 @@
actual=intentCheck,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ # Print the intent states
+ intents = []
+ intents.append( ONOS1Intents )
+ intents.append( ONOS2Intents )
+ intents.append( ONOS3Intents )
+ intents.append( ONOS4Intents )
+ intents.append( ONOS5Intents )
+ intents.append( ONOS6Intents )
+ intents.append( ONOS7Intents )
+ intentStates = []
+ for node in intents: # Iter through ONOS nodes
+ nodeStates = []
+ for intent in json.loads( node ): # Iter through intents of a node
+ nodeStates.append( intent[ 'state' ] )
+ intentStates.append( nodeStates )
+ out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
+ main.log.info( dict( out ) )
+
# NOTE: Hazelcast has no durability, so intents are lost across system
# restarts
@@ -1300,7 +1335,8 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- if intentState == ONOS1Intents:
+ sameIntents = main.TRUE
+ if intentState and intentState == ONOS1Intents:
sameIntents = main.TRUE
main.log.report( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
@@ -1489,7 +1525,6 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- host
if host[ 'ips' ] == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
@@ -1642,7 +1677,7 @@
note = "note it takes about " + str( int( cliTime ) ) + \
" seconds for the test to make all the cli calls to fetch " +\
"the topology from each ONOS instance"
- main.log.report(
+ main.log.info(
"Very crass estimate for topology discovery/convergence( " +
str( note ) + " ): " + str( elapsed ) + " seconds, " +
str( count ) + " tries" )
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index d23dcda..3d9adc7 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -437,19 +437,37 @@
str( hex( i )[ 2: ] ).zfill( 2 ).upper()
host2 = "00:00:00:00:00:" + \
str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
- host1Id = main.ONOScli1.getHost( host1 )[ 'id' ]
- host2Id = main.ONOScli1.getHost( host2 )[ 'id' ]
- # NOTE: get host can return None
+ # NOTE: getHost can return None
+ host1Dict = main.ONOScli1.getHost( host1 )
+ host2Dict = main.ONOScli1.getHost( host2 )
+ host1Id = None
+ host2Id = None
+ if host1Dict and host2Dict:
+ host1Id = host1Dict.get( 'id', None )
+ host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
- tmpResult = main.ONOScli1.addHostIntent(
+ #Changed onos node to test something
+ tmpResult = main.ONOScli4.addHostIntent(
host1Id,
host2Id )
else:
main.log.error( "Error, getHost() failed" )
+ main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
tmpResult = main.FALSE
intentAddResult = bool( pingResult and intentAddResult
and tmpResult )
# TODO Check that intents were added?
+ # Print the intent states
+ intents = main.ONOScli1.intents( )
+ intentStates = []
+ for intent in json.loads( intents ): # Iter through intents of a node
+ intentStates.append( intent.get( 'state', None ) )
+ out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
+ main.log.info( dict( out ) )
+
utilities.assert_equals(
expect=True,
actual=intentAddResult,
@@ -909,12 +927,6 @@
hosts.append( main.ONOScli5.hosts() )
hosts.append( main.ONOScli6.hosts() )
hosts.append( main.ONOScli7.hosts() )
- for controller in range( 0, len( hosts ) ):
- controllerStr = str( controller + 1 )
- for host in hosts[ controller ]:
- if host[ 'ips' ] == []:
- main.log.error( "DEBUG:Error with host ips on controller" +
- controllerStr + ": " + str( host ) )
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -1290,13 +1302,32 @@
actual=intentCheck,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ # Print the intent states
+ intents = []
+ intents.append( ONOS1Intents )
+ intents.append( ONOS2Intents )
+ intents.append( ONOS3Intents )
+ intents.append( ONOS4Intents )
+ intents.append( ONOS5Intents )
+ intents.append( ONOS6Intents )
+ intents.append( ONOS7Intents )
+ intentStates = []
+ for node in intents: # Iter through ONOS nodes
+ nodeStates = []
+ for intent in json.loads( node ): # Iter through intents of a node
+ nodeStates.append( intent[ 'state' ] )
+ intentStates.append( nodeStates )
+ out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
+ main.log.info( dict( out ) )
+
# NOTE: Hazelcast has no durability, so intents are lost across system
# restarts
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- if intentState == ONOS1Intents:
+ sameIntents = main.TRUE
+ if intentState and intentState == ONOS1Intents:
sameIntents = main.TRUE
main.log.report( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
@@ -1490,7 +1521,6 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- host
if host[ 'ips' ] == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
@@ -1643,7 +1673,7 @@
note = "note it takes about " + str( int( cliTime ) ) + \
" seconds for the test to make all the cli calls to fetch " +\
"the topology from each ONOS instance"
- main.log.report(
+ main.log.info(
"Very crass estimate for topology discovery/convergence( " +
str( note ) + " ): " + str( elapsed ) + " seconds, " +
str( count ) + " tries" )
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index e205a02..58f9954 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -437,19 +437,36 @@
str( hex( i )[ 2: ] ).zfill( 2 ).upper()
host2 = "00:00:00:00:00:" + \
str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
- host1Id = main.ONOScli1.getHost( host1 )[ 'id' ]
- host2Id = main.ONOScli1.getHost( host2 )[ 'id' ]
- # NOTE: get host can return None
+ # NOTE: getHost can return None
+ host1Dict = main.ONOScli1.getHost( host1 )
+ host2Dict = main.ONOScli1.getHost( host2 )
+ host1Id = None
+ host2Id = None
+ if host1Dict and host2Dict:
+ host1Id = host1Dict.get( 'id', None )
+ host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
tmpResult = main.ONOScli1.addHostIntent(
host1Id,
host2Id )
else:
main.log.error( "Error, getHost() failed" )
+ main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
tmpResult = main.FALSE
intentAddResult = bool( pingResult and intentAddResult
and tmpResult )
# TODO Check that intents were added?
+ # Print the intent states
+ intents = main.ONOScli1.intents( )
+ intentStates = []
+ for intent in json.loads( intents ): # Iter through intents of a node
+ intentStates.append( intent.get( 'state', None ) )
+ out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
+ main.log.info( dict( out ) )
+
utilities.assert_equals(
expect=True,
actual=intentAddResult,
@@ -1261,13 +1278,32 @@
actual=intentCheck,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ # Print the intent states
+ intents = []
+ intents.append( ONOS1Intents )
+ intents.append( ONOS2Intents )
+ intents.append( ONOS3Intents )
+ intents.append( ONOS4Intents )
+ intents.append( ONOS5Intents )
+ intents.append( ONOS6Intents )
+ intents.append( ONOS7Intents )
+ intentStates = []
+ for node in intents: # Iter through ONOS nodes
+ nodeStates = []
+ for intent in json.loads( node ): # Iter through intents of a node
+ nodeStates.append( intent[ 'state' ] )
+ intentStates.append( nodeStates )
+ out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
+ main.log.info( dict( out ) )
+
# NOTE: Hazelcast has no durability, so intents are lost across system
# restarts
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- if intentState == ONOS1Intents:
+ sameIntents = main.TRUE
+ if intentState and intentState == ONOS1Intents:
sameIntents = main.TRUE
main.log.report( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
@@ -1457,7 +1493,6 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- host
if host[ 'ips' ] == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
@@ -1611,7 +1646,7 @@
note = "note it takes about " + str( int( cliTime ) ) + \
" seconds for the test to make all the cli calls to fetch " +\
"the topology from each ONOS instance"
- main.log.report(
+ main.log.info(
"Very crass estimate for topology discovery/convergence( " +
str( note ) + " ): " + str( elapsed ) + " seconds, " +
str( count ) + " tries" )
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index f312e23..4b9b287 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -239,19 +239,36 @@
str( hex( i )[ 2: ] ).zfill( 2 ).upper()
host2 = "00:00:00:00:00:" + \
str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
- host1Id = main.ONOScli1.getHost( host1 )[ 'id' ]
- host2Id = main.ONOScli1.getHost( host2 )[ 'id' ]
- # NOTE: get host can return None
+ # NOTE: getHost can return None
+ host1Dict = main.ONOScli1.getHost( host1 )
+ host2Dict = main.ONOScli1.getHost( host2 )
+ host1Id = None
+ host2Id = None
+ if host1Dict and host2Dict:
+ host1Id = host1Dict.get( 'id', None )
+ host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
tmpResult = main.ONOScli1.addHostIntent(
host1Id,
host2Id )
else:
main.log.error( "Error, getHost() failed" )
+ main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
tmpResult = main.FALSE
intentAddResult = bool( pingResult and intentAddResult
and tmpResult )
# TODO Check that intents were added?
+ # Print the intent states
+ intents = main.ONOScli1.intents( )
+ intentStates = []
+ for intent in json.loads( intents ): # Iter through intents of a node
+ intentStates.append( intent.get( 'state', None ) )
+ out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
+ main.log.info( dict( out ) )
+
utilities.assert_equals(
expect=True,
actual=intentAddResult,
@@ -565,6 +582,18 @@
actual=intentCheck,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ # Print the intent states
+ intents = []
+ intents.append( ONOS1Intents )
+ intentStates = []
+ for node in intents: # Iter through ONOS nodes
+ nodeStates = []
+ for intent in json.loads( node ): # Iter through intents of a node
+ nodeStates.append( intent[ 'state' ] )
+ intentStates.append( nodeStates )
+ out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
+ main.log.info( dict( out ) )
+
# NOTE: Hazelcast has no durability, so intents are lost across system
# restarts
@@ -572,7 +601,8 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- if intentState == ONOS1Intents:
+ sameIntents = main.TRUE
+ if intentState and intentState == ONOS1Intents:
sameIntents = main.TRUE
main.log.report( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
@@ -773,7 +803,7 @@
note = "note it takes about " + str( int( cliTime ) ) + \
" seconds for the test to make all the cli calls to fetch " +\
"the topology from each ONOS instance"
- main.log.report(
+ main.log.info(
"Very crass estimate for topology discovery/convergence( " +
str( note ) + " ): " + str( elapsed ) + " seconds, " +
str( count ) + " tries" )