Fix HA tests to match recent ONOS changes
- Minor refactoring to prepare for Upgrade tests
- Fix for change in raft log location
- Fix for auto balancing of leader electors
- Fix for path not found being printed as an exception for intents
Change-Id: I55af0d09c9b3bf74c0f221e7caa6840373a8779d
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params
index d7c4cab..bbf11ae 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.params
@@ -33,7 +33,7 @@
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
- <appString>drivers,openflow,proxyarp,mobility</appString>
+ <appString>drivers,openflow,proxyarp,mobility,events</appString>
</ENV>
<GIT>
<pull>False</pull>
diff --git a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params
index f1520f7..a72a475 100644
--- a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params
+++ b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params
@@ -19,7 +19,7 @@
#CASE15: Check that Leadership Election is still functional
#CASE16: Install Distributed Primitives app
#CASE17: Check for basic functionality with distributed primitives
- <testcases>1,2,8,21,3,4,5,14,16,17,[61,8,7,4,15,17,62]*1000,8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,21,3,4,5,14,16,17,[61,8,7,4,15,17,62,7,8,4,15,17]*1000,8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<apps></apps>
<ONOS_Configuration>
@@ -30,7 +30,7 @@
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
- <appString>drivers,openflow,proxyarp,mobility</appString>
+ <appString>drivers,openflow,proxyarp,mobility,events</appString>
</ENV>
<GIT>
<pull>False</pull>
diff --git a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
index 8238b94..b7385fe 100644
--- a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
+++ b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
@@ -142,7 +142,7 @@
assert main.killCount is not None, "main.killCount not defined"
except AttributeError as e:
main.log.warn( "Node to kill not selected, defaulting to node 1" )
- main.nodeIndex = 0
+ main.nodeIndex = -1
main.killCount = 1
main.case( "Stopping ONOS nodes - iteration " + str( main.killCount ) )
@@ -168,6 +168,7 @@
utilities.assert_equals( expect=main.TRUE, actual=killResults,
onpass="ONOS nodes stopped successfully",
onfail="ONOS nodes NOT successfully stopped" )
+ main.Cluster.reset()
main.step( "Checking ONOS nodes" )
nodeResults = utilities.retry( main.Cluster.nodesCheck,
@@ -193,7 +194,7 @@
"""
The bring up stopped nodes
"""
- main.HA.bringUpStoppedNode( main )
+ main.HA.bringUpStoppedNodes( main )
def CASE7( self, main ):
"""
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params
index 6dcfb5b..fe4cd80 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.params
@@ -35,7 +35,7 @@
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
- <appString>drivers,openflow,proxyarp,mobility</appString>
+ <appString>drivers,openflow,proxyarp,mobility,events</appString>
</ENV>
<GIT>
<pull>False</pull>
diff --git a/TestON/tests/HA/HAkillNodes/HAkillNodes.params b/TestON/tests/HA/HAkillNodes/HAkillNodes.params
index b9ed44a..409bd1f 100644
--- a/TestON/tests/HA/HAkillNodes/HAkillNodes.params
+++ b/TestON/tests/HA/HAkillNodes/HAkillNodes.params
@@ -35,7 +35,7 @@
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
- <appString>drivers,openflow,proxyarp,mobility</appString>
+ <appString>drivers,openflow,proxyarp,mobility,events</appString>
</ENV>
<GIT>
<pull>False</pull>
diff --git a/TestON/tests/HA/HAkillNodes/HAkillNodes.py b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
index 1c82e88..cd47131 100644
--- a/TestON/tests/HA/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
@@ -181,7 +181,7 @@
"""
The bring up stopped nodes
"""
- main.HA.bringUpStoppedNode( main )
+ main.HA.bringUpStoppedNodes( main )
def CASE7( self, main ):
"""
diff --git a/TestON/tests/HA/HAsanity/HAsanity.params b/TestON/tests/HA/HAsanity/HAsanity.params
index d9e4627..5a9f8f9 100644
--- a/TestON/tests/HA/HAsanity/HAsanity.params
+++ b/TestON/tests/HA/HAsanity/HAsanity.params
@@ -34,7 +34,7 @@
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
- <appString>drivers,openflow,proxyarp,mobility</appString>
+ <appString>events,drivers,openflow,proxyarp,mobility</appString>
</ENV>
<GIT>
<pull>False</pull>
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.params b/TestON/tests/HA/HAstopNodes/HAstopNodes.params
index b9ed44a..de7f775 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.params
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.params
@@ -35,7 +35,7 @@
</ONOS_Configuration>
<ENV>
<cellName>HA</cellName>
- <appString>drivers,openflow,proxyarp,mobility</appString>
+ <appString>events,drivers,openflow,proxyarp,mobility</appString>
</ENV>
<GIT>
<pull>False</pull>
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index 2884535..4c8fe1d 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -180,7 +180,7 @@
"""
The bring up stopped nodes
"""
- main.HA.bringUpStoppedNode( main )
+ main.HA.bringUpStoppedNodes( main )
def CASE7( self, main ):
"""
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index c781a52..fa75618 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -298,7 +298,7 @@
main.step( "Checking ONOS nodes" )
nodeResults = utilities.retry( main.Cluster.nodesCheck,
False,
- attempts=5 )
+ attempts=9 )
utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
@@ -812,7 +812,7 @@
main.step( "Check Intent state" )
installedCheck = False
loopCount = 0
- while not installedCheck and loopCount < 40:
+ while not installedCheck and loopCount < 90:
installedCheck = True
# Print the intent states
intents = onosCli.CLI.intents()
@@ -2532,17 +2532,17 @@
else:
main.log.debug( "skipping saving log files" )
+ main.step( "Checking ONOS Logs for errors" )
+ for ctrl in main.Cluster.runningNodes:
+ main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
+ main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
+
main.step( "Stopping Mininet" )
mnResult = main.Mininet1.stopNet()
utilities.assert_equals( expect=main.TRUE, actual=mnResult,
onpass="Mininet stopped",
onfail="MN cleanup NOT successful" )
- main.step( "Checking ONOS Logs for errors" )
- for ctrl in main.Cluster.runningNodes:
- main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
- main.log.warn( main.ONOSbench.checkLogs( ctrl.ipAddress ) )
-
try:
timerLog = open( main.logdir + "/Timers.csv", 'w' )
main.log.debug( ", ".join( main.HAlabels ) + "\n" + ", ".join( main.HAdata ) )
@@ -2663,9 +2663,9 @@
"controller",
onfail="Switches were not successfully reassigned" )
- def bringUpStoppedNode( self, main ):
+ def bringUpStoppedNodes( self, main ):
"""
- The bring up stopped nodes
+ The bring up stopped nodes.
"""
import time
assert main, "main not defined"
@@ -2696,7 +2696,7 @@
onpass="ONOS restarted successfully",
onfail="ONOS restart NOT successful" )
- main.step( "Restarting ONOS nodes" )
+ main.step( "Restarting ONOS CLI" )
cliResults = main.TRUE
for ctrl in main.kill:
cliResults = cliResults and\
@@ -2706,8 +2706,7 @@
onpass="ONOS node(s) restarted",
onfail="ONOS node(s) did not restart" )
- # Grab the time of restart so we chan check how long the gossip
- # protocol has had time to work
+ # Grab the time of restart so we can have some idea of average time
main.restartTime = time.time() - restartTime
main.log.debug( "Restart time: " + str( main.restartTime ) )
# TODO: MAke this configurable. Also, we are breaking the above timer
@@ -2738,7 +2737,96 @@
ctrl.electionTestRun()
utilities.assert_equals( expect=main.TRUE, actual=runResults,
onpass="ONOS nodes reran for election topic",
- onfail="Errror rerunning for election" )
+ onfail="Error rerunning for election" )
+
+ def upgradeNodes( self, main ):
+ """
+ Reinstall some nodes with an upgraded version.
+
+ This will reinstall nodes in main.kill with an upgraded version.
+ """
+ import time
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert main.kill, "main.kill not defined"
+ nodeNames = [ node.name for node in main.kill ]
+ main.step( "Upgrading" + str( nodeNames ) + " ONOS nodes" )
+
+ stopResults = main.TRUE
+ uninstallResults = main.TRUE
+ startResults = main.TRUE
+ sshResults = main.TRUE
+ isup = main.TRUE
+ restartTime = time.time()
+ for ctrl in main.kill:
+ stopResults = stopResults and\
+ ctrl.onosStop( ctrl.ipAddress )
+ uninstallResults = uninstallResults and\
+ ctrl.onosUninstall( ctrl.ipAddress )
+ # Install the new version of onos
+ startResults = startResults and\
+ ctrl.onosInstall( options="-fv", node=ctrl.ipAddress )
+ sshResults = sshResults and\
+ ctrl.onosSecureSSH( node=ctrl.ipAddress )
+ isup = isup and ctrl.isup( ctrl.ipAddress )
+ utilities.assert_equals( expect=main.TRUE, actual=stopResults,
+ onpass="ONOS nodes stopped successfully",
+ onfail="ONOS nodes NOT successfully stopped" )
+ utilities.assert_equals( expect=main.TRUE, actual=uninstallResults,
+ onpass="ONOS nodes uninstalled successfully",
+ onfail="ONOS nodes NOT successfully uninstalled" )
+ utilities.assert_equals( expect=main.TRUE, actual=startResults,
+ onpass="ONOS nodes started successfully",
+ onfail="ONOS nodes NOT successfully started" )
+ utilities.assert_equals( expect=main.TRUE, actual=sshResults,
+ onpass="Successfully secured onos ssh",
+ onfail="Failed to secure onos ssh" )
+ utilities.assert_equals( expect=main.TRUE, actual=isup,
+ onpass="ONOS nodes fully started",
+ onfail="ONOS nodes NOT fully started" )
+
+ main.step( "Restarting ONOS CLI" )
+ cliResults = main.TRUE
+ for ctrl in main.kill:
+ cliResults = cliResults and\
+ ctrl.startOnosCli( ctrl.ipAddress )
+ ctrl.active = True
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS node(s) restarted",
+ onfail="ONOS node(s) did not restart" )
+
+ # Grab the time of restart so we can have some idea of average time
+ main.restartTime = time.time() - restartTime
+ main.log.debug( "Restart time: " + str( main.restartTime ) )
+ # TODO: Make this configurable.
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.Cluster.nodesCheck,
+ False,
+ sleep=15,
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for ctrl in main.Cluster.active():
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ ctrl.name,
+ ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanAndExit()
+
+ self.commonChecks()
+
+ main.step( "Rerun for election on the node(s) that were killed" )
+ runResults = main.TRUE
+ for ctrl in main.kill:
+ runResults = runResults and\
+ ctrl.electionTestRun()
+ utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ onpass="ONOS nodes reran for election topic",
+ onfail="Error rerunning for election" )
def tempCell( self, cellName, ipList ):
main.step( "Create cell file" )
@@ -3048,8 +3136,8 @@
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
- print "Elapsed time: " + str( elapsed )
- print "CLI time: " + str( cliTime )
+ main.log.debug( "Elapsed time: " + str( elapsed ) )
+ main.log.debug( "CLI time: " + str( cliTime ) )
if all( e is None for e in devices ) and\
all( e is None for e in hosts ) and\
@@ -3493,15 +3581,18 @@
onfail="Inconsistent leaderboards" )
if sameResult:
+ # Check that the leader is one of the active nodes
+ ips = sorted( main.Cluster.getIps( activeOnly=True ) )
leader = leaders[ 0 ][ 0 ]
- if onosCli.ipAddress in leader:
- correctLeader = True
+ if leader in ips:
+ legitimate = True
else:
- correctLeader = False
- main.step( "First node was elected leader" )
+ legitimate = False
+ main.log.debug( leaders )
+ main.step( "Active node was elected leader?" )
utilities.assert_equals(
expect=True,
- actual=correctLeader,
+ actual=legitimate,
onpass="Correct leader was elected",
onfail="Incorrect leader" )
main.Cluster.testLeader = leader
@@ -3620,18 +3711,6 @@
else:
main.log.info( "Expected no leader, got: " + str( newLeader ) )
correctCandidateResult = main.FALSE
- elif len( oldLeaders[ 0 ] ) >= 3:
- if newLeader == oldLeaders[ 0 ][ 2 ]:
- # correct leader was elected
- correctCandidateResult = main.TRUE
- else:
- correctCandidateResult = main.FALSE
- main.log.error( "Candidate {} was elected. {} should have had priority.".format(
- newLeader, oldLeaders[ 0 ][ 2 ] ) )
- else:
- main.log.warn( "Could not determine who should be the correct leader" )
- main.log.debug( oldLeaders[ 0 ] )
- correctCandidateResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
actual=correctCandidateResult,
@@ -3659,24 +3738,10 @@
time.sleep( 5 ) # Paremterize
positionResult, reRunLeaders = self.consistentLeaderboards( activeCLIs )
- # Check that the re-elected node is last on the candidate List
- if not reRunLeaders[ 0 ]:
- positionResult = main.FALSE
- elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
- main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader ),
- str( reRunLeaders[ 0 ] ) ) )
- positionResult = main.FALSE
- utilities.assert_equals(
- expect=True,
- actual=positionResult,
- onpass="Old leader successfully re-ran for election",
- onfail="Something went wrong with Leadership election after " +
- "the old leader re-ran for election" )
-
def installDistributedPrimitiveApp( self, main ):
- """
+ '''
Install Distributed Primitives app
- """
+ '''
import time
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index b0af55e..1532039 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -494,6 +494,7 @@
maxSize = float( segmentSize ) * float( multiplier )
ret = True
for n in self.runningNodes:
- ret = ret and n.server.folderSize( "/opt/onos/apache-karaf-*/data/partitions/*/*.log",
+ # Partition logs
+ ret = ret and n.server.folderSize( "/opt/onos/apache-karaf-*/data/db/partitions/*/*.log",
size=maxSize, unit=units, ignoreRoot=False )
return ret