WIP: HA updates, includes network partition
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index a87e862..b1c0f77 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -483,8 +483,11 @@
tmpId = main.ONOScli4.addHostIntent(
host1Id,
host2Id )
- main.log.info( "Added intent with id: " + tmpId )
- intentIds.append( tmpId )
+ if tmpId:
+ main.log.info( "Added intent with id: " + tmpId )
+ intentIds.append( tmpId )
+ else:
+ main.log.error( "addHostIntent reutrned None" )
else:
main.log.error( "Error, getHost() failed" )
main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
@@ -503,10 +506,13 @@
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
+ installedCheck = True
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
for intent in json.loads( intents ): # Iter through intents of a node
state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
intentId = intent.get( 'id', None )
intentStates.append( ( intentId, state ) )
# add submitted intents not in the store
@@ -521,16 +527,28 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
+ main.ONOScli1.leaders()
+ main.ONOScli1.partitions()
+ # for node in nodes:
+ # node.pendingMap()
+ pendingMap = main.ONOScli1.pendingMap()
+ main.ONOScli2.pendingMap()
+ main.ONOScli3.pendingMap()
+ main.ONOScli4.pendingMap()
+ main.ONOScli5.pendingMap()
+ main.ONOScli6.pendingMap()
+ main.ONOScli7.pendingMap()
intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents)
+ and not missingIntents and installedCheck )
utilities.assert_equals(
expect=True,
actual=intentAddResult,
onpass="Pushed host intents to ONOS",
onfail="Error in pushing host intents to ONOS" )
- if not intentAddResult:
+ if not intentAddResult or "key" in pendingMap:
import time
+ installedCheck = True
main.log.info( "Sleeping 60 seconds to see if intents are found" )
time.sleep( 60 )
onosIds = main.ONOScli1.getAllIntentsId()
@@ -544,6 +562,8 @@
for intent in json.loads( intents ):
# Iter through intents of a node
state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
intentId = intent.get( 'id', None )
intentStates.append( ( intentId, state ) )
# add submitted intents not in the store
@@ -556,6 +576,14 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
+ main.ONOScli1.leaders()
+ main.ONOScli1.pendingMap()
+ main.ONOScli2.pendingMap()
+ main.ONOScli3.pendingMap()
+ main.ONOScli4.pendingMap()
+ main.ONOScli5.pendingMap()
+ main.ONOScli6.pendingMap()
+ main.ONOScli7.pendingMap()
def CASE4( self, main ):
"""
@@ -593,6 +621,8 @@
actual=PingResult,
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+
+ installedCheck = True
if PingResult is not main.TRUE:
# Print the intent states
intents = main.ONOScli1.intents()
@@ -602,6 +632,8 @@
# Iter through intents of a node
for intent in json.loads( intents ):
state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
intentId = intent.get( 'id', None )
intentStates.append( ( intentId, state ) )
intentStates.sort()
@@ -609,6 +641,44 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
+ main.ONOScli1.leaders()
+ main.ONOScli1.partitions()
+ main.ONOScli1.pendingMap()
+ main.ONOScli2.pendingMap()
+ main.ONOScli3.pendingMap()
+ main.ONOScli4.pendingMap()
+ main.ONOScli5.pendingMap()
+ main.ONOScli6.pendingMap()
+ main.ONOScli7.pendingMap()
+ if not installedCheck:
+ main.log.info( "Waiting 60 seconds to see if intent states change" )
+ time.sleep( 60 )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ main.ONOScli1.leaders()
+ main.ONOScli1.partitions()
+ main.ONOScli1.pendingMap()
+ main.ONOScli2.pendingMap()
+ main.ONOScli3.pendingMap()
+ main.ONOScli4.pendingMap()
+ main.ONOScli5.pendingMap()
+ main.ONOScli6.pendingMap()
+ main.ONOScli7.pendingMap()
def CASE5( self, main ):
"""
@@ -1869,7 +1939,7 @@
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
description = "Restore a link to ensure that Link Discovery is " + \
- "working properly"
+ "working properly"
main.log.report( description )
main.case( description )
@@ -2155,7 +2225,11 @@
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
- withdrawResult = oldLeader.electionTestWithdraw()
+ oldLeader = None
+ else:
+ main.log.error( "Leader election --- why am I HERE?!?")
+ if oldLeader:
+ withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
@@ -2208,7 +2282,10 @@
main.step( "Run for election on old leader( just so everyone " +
"is in the hat )" )
- runResult = oldLeader.electionTestRun()
+ if oldLeader:
+ runResult = oldLeader.electionTestRun()
+ else:
+ runResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
actual=runResult,