Choose ONOS nodes so that only one partition looses a majority
- Fix counter check to only look at the counters used by the test
- Add the gen-partition file to the repo and make the test copy it to
the right place in ONOS
- clear buffer in clidriver after secureCopy
- Change individual ping timeout for ping functions in mininet driver
- increase links timeout in mininet driver due to occasional failures
- Kill iperf if it timesout to allow continued use of mininet
- Clear buffer at the end of many onos driver functions
Change-Id: I3c9235d1c6082f0ef86266405ed99c07e1b27fb5
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index d726f2e..fd05a41 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -405,7 +405,7 @@
main.log.info( self.name + ": \n---\n" + response )
return main.FALSE
- def pingallHosts( self, hostList ):
+ def pingallHosts( self, hostList, wait=1 ):
"""
Ping all specified IPv4 hosts
@@ -417,8 +417,8 @@
Returns main.FALSE if one or more of hosts specified
cannot reach each other"""
-
- cmd = " ping -c 1 -i 1 -W 8 "
+ wait = int( wait )
+ cmd = " ping -c 1 -i 1 -W " + str( wait ) + " "
try:
main.log.info( "Testing reachability between specified hosts" )
@@ -437,7 +437,7 @@
for temp in pingList:
# Current host pings all other hosts specified
pingCmd = str( host ) + cmd + str( temp )
- self.handle.sendline( pingCmd )
+ self.handle.sendline( pingCmd, timeout=wait + 1 )
self.handle.expect( "mininet>" )
response = self.handle.before
if re.search( ',\s0\%\spacket\sloss', response ):
@@ -463,19 +463,20 @@
main.cleanup()
main.exit()
- def pingIpv6Hosts(self, hostList, prefix='1000::'):
+ def pingIpv6Hosts( self, hostList, prefix='1000::', wait=1 ):
"""
- IPv6 ping all hosts in hostList. If no prefix passed this will use
- default prefix of 1000::
+ IPv6 ping all hosts in hostList. If no prefix passed this will use
+ default prefix of 1000::
- Returns main.TRUE if all hosts specified can reach each other
+ Returns main.TRUE if all hosts specified can reach each other
- Returns main.FALSE if one or more of hosts specified cannot reach each other
+ Returns main.FALSE if one or more of hosts specified cannot reach each other
"""
try:
main.log.info( "Testing reachability between specified IPv6 hosts" )
isReachable = main.TRUE
- cmd = " ping6 -c 1 -i 1 -W 8 "
+ wait = int( wait )
+ cmd = " ping6 -c 1 -i 1 -W " + str( wait ) + " "
pingResponse = "IPv6 Pingall output:\n"
failedPings = 0
for host in hostList:
@@ -489,7 +490,7 @@
for temp in pingList:
# Current host pings all other hosts specified
pingCmd = str( host ) + cmd + prefix + str( temp[1:] )
- self.handle.sendline( pingCmd )
+ self.handle.sendline( pingCmd, timeout=wait + 1 )
self.handle.expect( "mininet>" )
response = self.handle.before
if re.search( ',\s0\%\spacket\sloss', response ):
@@ -518,15 +519,19 @@
def pingHost( self, **pingParams ):
"""
- Ping from one mininet host to another
- Currently the only supported Params: SRC and TARGET"""
- args = utilities.parse_args( [ "SRC", "TARGET" ], **pingParams )
+ Ping from one mininet host to another
+ Currently the only supported Params: SRC, TARGET, and WAIT
+ """
+ args = utilities.parse_args( [ "SRC", "TARGET", 'WAIT' ], **pingParams )
+ wait = args['WAIT']
+ wait = int( wait if wait else 1 )
command = args[ "SRC" ] + " ping " + \
- args[ "TARGET" ] + " -c 1 -i 1 -W 8"
+ args[ "TARGET" ] + " -c 1 -i 1 -W " + str( wait ) + " "
try:
main.log.info( "Sending: " + command )
self.handle.sendline( command )
- i = self.handle.expect( [ command, pexpect.TIMEOUT ] )
+ i = self.handle.expect( [ command, pexpect.TIMEOUT ],
+ timeout=wait + 1 )
if i == 1:
main.log.error(
self.name +
@@ -561,17 +566,20 @@
def ping6pair( self, **pingParams ):
"""
IPv6 Ping between a pair of mininet hosts
- Currently the only supported Params are: SRC , TARGET
+ Currently the only supported Params are: SRC, TARGET, and WAIT
FLOWLABEL and -I (src interface) will be added later after running some tests.
Example: main.Mininet1.ping6pair( src="h1", target="1000::2" )
"""
- args = utilities.parse_args( [ "SRC", "TARGET" ], **pingParams )
- command = args[ "SRC" ] + " ping6 " + \
- args[ "TARGET" ] + " -c 1 -i 1 -W 8"
+ args = utilities.parse_args( [ "SRC", "TARGET", 'WAIT' ], **pingParams )
+ wait = args['WAIT']
+ wait = int( wait if wait else 1 )
+ command = args[ "SRC" ] + " ping " + \
+ args[ "TARGET" ] + " -c 1 -i 1 -W " + str( wait ) + " "
try:
main.log.info( "Sending: " + command )
self.handle.sendline( command )
- i = self.handle.expect( [ command, pexpect.TIMEOUT ] )
+ i = self.handle.expect( [ command, pexpect.TIMEOUT ],
+ timeout=wait + 1 )
if i == 1:
main.log.error(
self.name +
@@ -1039,7 +1047,7 @@
main.log.info( self.name + ": List network links" )
try:
response = self.execute( cmd='links', prompt='mininet>',
- timeout=10 )
+ timeout=20 )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1104,12 +1112,15 @@
else:
main.log.error( self.name + ": iperf test failed" )
return main.FALSE
-
except pexpect.TIMEOUT:
- main.log.error( self.name + ": TIMEOUT exception found")
- main.log.error( self.name + ": Exception: Cannot connect to iperf on port 5001" )
+ main.log.error( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + " response: " +
+ repr ( self.handle.before ) )
+ # NOTE: Send ctrl-c to make sure iperf is done
+ self.handle.sendline( "\x03" )
+ self.handle.expect( "Interrupt" )
+ self.handle.expect( "mininet>" )
return main.FALSE
-
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index c261872..2fae817 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -192,6 +192,7 @@
self.handle.sendline( "onos-package" )
self.handle.expect( "onos-package" )
self.handle.expect( "tar.gz", opTimeout )
+ self.handle.expect( "\$" )
handle = str( self.handle.before )
main.log.info( "onos-package command returned: " +
handle )
@@ -221,6 +222,7 @@
"BUILD FAILED" ],
timeout=120 )
handle = str( self.handle.before )
+ self.handle.expect( "\$" )
main.log.info( "onos-build command returned: " +
handle )
@@ -752,12 +754,11 @@
# Expect the cellname in the ONOSCELL variable.
# Note that this variable name is subject to change
# and that this driver will have to change accordingly
- self.handle.expect(str(cellname))
+ self.handle.expect( str( cellname ) )
handleBefore = self.handle.before
handleAfter = self.handle.after
# Get the rest of the handle
- self.handle.sendline("")
- self.handle.expect("\$")
+ self.handle.expect( "\$" )
handleMore = self.handle.before
cell_result = handleBefore + handleAfter + handleMore
@@ -768,7 +769,6 @@
main.cleanup()
main.exit()
return main.TRUE
-
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -793,19 +793,11 @@
self.handle.expect( "\$" )
handleBefore = self.handle.before
handleAfter = self.handle.after
- # Get the rest of the handle
- self.handle.sendline( "" )
- self.handle.expect( "\$" )
- handleMore = self.handle.before
-
main.log.info( "Verify cell returned: " + handleBefore +
- handleAfter + handleMore )
-
+ handleAfter )
return main.TRUE
except pexpect.ExceptionPexpect as e:
- main.log.error( self.name + ": Pexpect exception found of type " +
- str( type( e ) ) )
- main.log.error ( e.get_trace() )
+ main.log.exception( self.name + ": Pexpect exception found: " )
main.log.error( self.name + ": " + self.handle.before )
main.cleanup()
main.exit()
@@ -851,9 +843,7 @@
return main.TRUE
except pexpect.ExceptionPexpect as e:
- main.log.error( self.name + ": Pexpect exception found of type " +
- str( type( e ) ) )
- main.log.error ( e.get_trace() )
+ main.log.exception( self.name + ": Pexpect exception found: " )
main.log.error( self.name + ": " + self.handle.before )
main.cleanup()
main.exit()
@@ -904,23 +894,12 @@
self.handle.expect( "\$" )
handleBefore = self.handle.before
- print "handle_before = ", self.handle.before
- # handleAfter = str( self.handle.after )
-
- # self.handle.sendline( "" )
- # self.handle.expect( "\$" )
- # handleMore = str( self.handle.before )
-
main.log.info( "Command sent successfully" )
-
# Obtain return handle that consists of result from
# the onos command. The string may need to be
# configured further.
- # returnString = handleBefore + handleAfter
returnString = handleBefore
- print "return_string = ", returnString
return returnString
-
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -954,26 +933,28 @@
"onos\sstart/running,\sprocess",
"ONOS\sis\salready\sinstalled",
pexpect.TIMEOUT ], timeout=60 )
-
if i == 0:
main.log.warn( "Network is unreachable" )
+ self.handle.expect( "\$" )
return main.FALSE
elif i == 1:
main.log.info(
"ONOS was installed on " +
node +
" and started" )
+ self.handle.expect( "\$" )
return main.TRUE
elif i == 2:
main.log.info( "ONOS is already installed on " + node )
+ self.handle.expect( "\$" )
return main.TRUE
elif i == 3:
main.log.info(
"Installation of ONOS on " +
node +
" timed out" )
+ self.handle.expect( "\$" )
return main.FALSE
-
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -999,7 +980,7 @@
"start/running",
"Unknown\sinstance",
pexpect.TIMEOUT ], timeout=120 )
-
+ self.handle.expect( "\$" )
if i == 0:
main.log.info( "Service is already running" )
return main.TRUE
@@ -1035,7 +1016,7 @@
"Could not resolve hostname",
"Unknown\sinstance",
pexpect.TIMEOUT ], timeout=60 )
-
+ self.handle.expect( "\$" )
if i == 0:
main.log.info( "ONOS service stopped" )
return main.TRUE
@@ -1049,7 +1030,6 @@
else:
main.log.error( "ONOS service failed to stop" )
return main.FALSE
-
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1070,13 +1050,10 @@
self.handle.sendline( "" )
self.handle.expect( "\$", timeout=60 )
self.handle.sendline( "onos-uninstall " + str( nodeIp ) )
- self.handle.expect( "\$" )
-
+ self.handle.expect( "\$", timeout=60 )
main.log.info( "ONOS " + nodeIp + " was uninstalled" )
-
# onos-uninstall command does not return any text
return main.TRUE
-
except pexpect.TIMEOUT:
main.log.exception( self.name + ": Timeout in onosUninstall" )
return main.FALSE
@@ -1186,10 +1163,7 @@
timeout=120 )
if i == 1:
return main.FALSE
- #self.handle.sendline( "" )
- #self.handle.expect( "\$" )
return main.TRUE
-
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1237,7 +1211,7 @@
main.cleanup()
main.exit()
- def isup(self, node = "", timeout = 120):
+ def isup( self, node="", timeout=120 ):
"""
Run's onos-wait-for-start which only returns once ONOS is at run
level 100(ready for use)
@@ -1245,8 +1219,8 @@
Returns: main.TRUE if ONOS is running and main.FALSE on timeout
"""
try:
- self.handle.sendline("onos-wait-for-start " + node )
- self.handle.expect("onos-wait-for-start")
+ self.handle.sendline( "onos-wait-for-start " + node )
+ self.handle.expect( "onos-wait-for-start" )
# NOTE: this timeout is arbitrary"
i = self.handle.expect(["\$", pexpect.TIMEOUT], timeout)
if i == 0:
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 0202a15..7d6ef0b 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -145,12 +145,11 @@
"""
result = super( CLI, self ).execute( self )
defaultPrompt = '.*[$>\#]'
- args = utilities.parse_args( [
- "CMD",
- "TIMEOUT",
- "PROMPT",
- "MORE" ],
- **execparams )
+ args = utilities.parse_args( [ "CMD",
+ "TIMEOUT",
+ "PROMPT",
+ "MORE" ],
+ **execparams )
expectPrompt = args[ "PROMPT" ] if args[ "PROMPT" ] else defaultPrompt
self.LASTRSP = ""
@@ -164,20 +163,18 @@
args[ "MORE" ] = " "
self.handle.sendline( cmd )
self.lastCommand = cmd
- index = self.handle.expect( [
- expectPrompt,
- "--More--",
- 'Command not found.',
- pexpect.TIMEOUT,
- "^:$" ],
- timeout=timeoutVar )
+ index = self.handle.expect( [ expectPrompt,
+ "--More--",
+ 'Command not found.',
+ pexpect.TIMEOUT,
+ "^:$" ],
+ timeout=timeoutVar )
if index == 0:
self.LASTRSP = self.LASTRSP + \
self.handle.before + self.handle.after
- main.log.info(
- "Executed :" + str(
- cmd ) + " \t\t Expected Prompt '" + str(
- expectPrompt) + "' Found" )
+ main.log.info( "Executed :" + str(cmd ) +
+ " \t\t Expected Prompt '" + str( expectPrompt) +
+ "' Found" )
elif index == 1:
self.LASTRSP = self.LASTRSP + self.handle.before
self.handle.send( args[ "MORE" ] )
@@ -196,26 +193,25 @@
main.log.error( "Command not found" )
self.LASTRSP = self.LASTRSP + self.handle.before
elif index == 3:
- main.log.error( "Expected Prompt not found , Time Out!!" )
+ main.log.error( "Expected Prompt not found, Time Out!!" )
main.log.error( expectPrompt )
- return "Expected Prompt not found , Time Out!!"
-
+ self.LASTRSP = self.LASTRSP + self.handle.before
+ return self.LASTRSP
elif index == 4:
self.LASTRSP = self.LASTRSP + self.handle.before
# self.handle.send( args[ "MORE" ] )
self.handle.sendcontrol( "D" )
main.log.info(
- "Found More screen to go , Sending a key to proceed" )
+ "Found More screen to go, Sending a key to proceed" )
indexMore = self.handle.expect(
[ "^:$", expectPrompt ], timeout=timeoutVar )
while indexMore == 0:
main.log.info(
- "Found another More screen to go , Sending a key to proceed" )
+ "Found another More screen to go, Sending a key to proceed" )
self.handle.sendcontrol( "D" )
indexMore = self.handle.expect(
[ "^:$", expectPrompt ], timeout=timeoutVar )
self.LASTRSP = self.LASTRSP + self.handle.before
-
main.last_response = self.remove_contol_chars( self.LASTRSP )
return self.LASTRSP
@@ -297,7 +293,6 @@
pexpect.EOF,
pexpect.TIMEOUT ],
120 )
-
if i == 0: # ask for ssh key confirmation
main.log.info( "ssh key confirmation received, sending yes" )
self.handle.sendline( 'yes' )
@@ -327,10 +322,7 @@
"@" +
ipAddress )
returnVal = main.FALSE
-
- self.handle.sendline( "" )
- self.handle.expect( "$" )
-
+ self.handle.expect( "\$" )
return returnVal
def scp( self, remoteHost, filePath, dstPath, direction="from" ):
diff --git a/TestON/tests/HAminorityRestart/HAminorityRestart.py b/TestON/tests/HAminorityRestart/HAminorityRestart.py
index d175fd7..ca8a194 100644
--- a/TestON/tests/HAminorityRestart/HAminorityRestart.py
+++ b/TestON/tests/HAminorityRestart/HAminorityRestart.py
@@ -48,6 +48,7 @@
start tcpdump
"""
import imp
+ import pexpect
main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
@@ -189,6 +190,16 @@
main.log.wiki(graphs)
main.step( "Creating ONOS package" )
+ # copy gen-partions file to ONOS
+ # NOTE: this assumes TestON and ONOS are on the same machine
+ srcFile = main.testDir + "/" + main.TEST + "/dependencies/onos-gen-partitions"
+ dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
+ cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
+ main.ONOSbench.ip_address,
+ srcFile,
+ dstDir,
+ pwd=main.ONOSbench.pwd,
+ direction="from" )
packageResult = main.ONOSbench.onosPackage()
utilities.assert_equals( expect=main.TRUE, actual=packageResult,
onpass="ONOS package successful",
@@ -203,6 +214,19 @@
utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
onpass="ONOS install successful",
onfail="ONOS install failed" )
+ # clean up gen-partitions file
+ try:
+ main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
+ main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
+ main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
+ main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
+ main.log.info( " Cleaning custom gen partitions file, response was: \n" +
+ str( main.ONOSbench.handle.before ) )
+ except ( pexpect.TIMEOUT, pexpect.EOF ):
+ main.log.exception( "ONOSbench: pexpect exception found:" +
+ main.ONOSbench.handle.before )
+ main.cleanup()
+ main.exit()
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
@@ -1680,17 +1704,19 @@
main.log.debug( "Checking logs for errors on " + node.name + ":" )
main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
- main.step( "Killing 3 ONOS nodes" )
+ n = len( main.nodes ) # Number of nodes
+ p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
+ main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
+ if n > 3:
+ main.kill.append( p - 1 )
+ # NOTE: This only works for cluster sizes of 3,5, or 7.
+
+ main.step( "Killing " + str( len( main.kill ) ) + " ONOS nodes" )
killTime = time.time()
- # TODO: Randomize these nodes or base this on partitions
- # TODO: use threads in this case
- killResults = main.ONOSbench.onosKill( main.nodes[0].ip_address )
- time.sleep( 10 )
- killResults = killResults and\
- main.ONOSbench.onosKill( main.nodes[1].ip_address )
- time.sleep( 10 )
- killResults = killResults and\
- main.ONOSbench.onosKill( main.nodes[2].ip_address )
+ killResults = main.TRUE
+ for i in main.kill:
+ killResults = killResults and\
+ main.ONOSbench.onosKill( main.nodes[i].ip_address )
utilities.assert_equals( expect=main.TRUE, actual=killResults,
onpass="ONOS Killed successfully",
onfail="ONOS kill NOT successful" )
@@ -1699,21 +1725,20 @@
count = 0
onosIsupResult = main.FALSE
while onosIsupResult == main.FALSE and count < 10:
- onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
- onos2Isup = main.ONOSbench.isup( main.nodes[1].ip_address )
- onos3Isup = main.ONOSbench.isup( main.nodes[2].ip_address )
- onosIsupResult = onos1Isup and onos2Isup and onos3Isup
+ onosIsupResult = main.TRUE
+ for i in main.kill:
+ onosIsupResult = onosIsupResult and\
+ main.ONOSbench.isup( main.nodes[i].ip_address )
count = count + 1
- # TODO: if it becomes an issue, we can retry this step a few times
utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
onpass="ONOS restarted successfully",
onfail="ONOS restart NOT successful" )
main.step( "Restarting ONOS main.CLIs" )
- cliResult1 = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
- cliResult2 = main.ONOScli2.startOnosCli( main.nodes[1].ip_address )
- cliResult3 = main.ONOScli3.startOnosCli( main.nodes[2].ip_address )
- cliResults = cliResult1 and cliResult2 and cliResult3
+ cliResults = main.TRUE
+ for i in main.kill:
+ cliResults = cliResults and\
+ main.CLIs[i].startOnosCli( main.nodes[i].ip_address )
utilities.assert_equals( expect=main.TRUE, actual=cliResults,
onpass="ONOS cli restarted",
onfail="ONOS cli did not restart" )
@@ -1722,17 +1747,6 @@
# protocol has had time to work
main.restartTime = time.time() - killTime
main.log.debug( "Restart time: " + str( main.restartTime ) )
- '''
- # FIXME: revisit test plan for election with madan
- # Rerun for election on restarted nodes
- run1 = main.CLIs[0].electionTestRun()
- run2 = main.CLIs[1].electionTestRun()
- run3 = main.CLIs[2].electionTestRun()
- runResults = run1 and run2 and run3
- utilities.assert_equals( expect=main.TRUE, actual=runResults,
- onpass="Reran for election",
- onfail="Failed to rerun for election" )
- '''
# TODO: MAke this configurable. Also, we are breaking the above timer
time.sleep( 60 )
main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
@@ -2052,11 +2066,12 @@
main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leaderList = []
- # FIXME: make sure this matches nodes that were restarted
- restarted = [ main.nodes[0].ip_address, main.nodes[1].ip_address,
- main.nodes[2].ip_address ]
+ restarted = []
+ for i in main.kill:
+ restarted.append( main.nodes[i].ip_address )
leaderResult = main.TRUE
+
for cli in main.CLIs:
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
@@ -3409,23 +3424,7 @@
onfail="Added counters are incorrect" )
main.step( "Check counters are consistant across nodes" )
- onosCounters = []
- threads = []
- for i in range( main.numCtrls ):
- t = main.Thread( target=main.CLIs[i].counters,
- name="counters-" + str( i ) )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCounters.append( t.result )
- tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
- if all( tmp ):
- main.log.info( "Counters are consistent across all nodes" )
- consistentCounterResults = main.TRUE
- else:
- main.log.error( "Counters are not consistent across all nodes" )
- consistentCounterResults = main.FALSE
+ onosCounters, consistentCounterResults = main.Counters.consistentCheck()
utilities.assert_equals( expect=main.TRUE,
actual=consistentCounterResults,
onpass="ONOS counters are consistent " +
@@ -3441,7 +3440,6 @@
actual=incrementCheck,
onpass="Added counters are correct",
onfail="Added counters are incorrect" )
-
# DISTRIBUTED SETS
main.step( "Distributed Set get" )
size = len( onosSet )
diff --git a/TestON/tests/HAminorityRestart/README b/TestON/tests/HAminorityRestart/README
new file mode 100644
index 0000000..a913f85
--- /dev/null
+++ b/TestON/tests/HAminorityRestart/README
@@ -0,0 +1,24 @@
+This test is designed to verify that an ONOS cluster behaves correctly when
+ONOS nodes die. Currently, we will kill nodes so that each raft partition will
+lose a member, but we make sure that there is always a majority of nodes
+available in each partition.
+
+As written, the test only supports an ONOS cluster of 3,5, or 7 nodes.
+This is because the test doesn't apply to a single node cluster, ONOS clusters
+should be deployed in odd numbers, and the partition generation and node
+killing scheme used doesn't give the same properties for clusters of more
+than 7 nodes. Namely, each partition won't have exactly one node killed.
+
+The gerneral structure for the test:
+- Startup
+- Assign switches
+- Verify ONOS state and functionality
+ - Device mastership
+ - Intents
+ - Leadership election
+ - Distributed Primitives
+- Kill some ONOS nodes
+- Verify ONOS state and functionality
+- Dataplane failures
+ - link down and up
+ - switch down and up
diff --git a/TestON/tests/HAminorityRestart/dependencies/Counters.py b/TestON/tests/HAminorityRestart/dependencies/Counters.py
index 21308c2..6614887 100644
--- a/TestON/tests/HAminorityRestart/dependencies/Counters.py
+++ b/TestON/tests/HAminorityRestart/dependencies/Counters.py
@@ -1,14 +1,19 @@
def __init__( self ):
self.default = ''
-def counterCheck( counterName, counterValue ):
+def consistentCheck():
"""
- Add Text here
+ Checks that TestON counters are consistent across all nodes.
+
+ Returns the tuple (onosCounters, consistent)
+ - onosCounters is the parsed json output of the counters command on all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across all
+ nodes or main.FALSE
"""
import json
correctResults = main.TRUE
# Get onos counters results
- onosCounters = []
+ onosCountersRaw = []
threads = []
for i in range( main.numCtrls ):
t = main.Thread( target=main.CLIs[i].counters,
@@ -17,25 +22,58 @@
t.start()
for t in threads:
t.join()
- onosCounters.append( t.result )
- tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+ onosCountersRaw.append( t.result )
+ onosCounters = []
+ for i in range( main.numCtrls ):
+ try:
+ onosCounters.append( json.loads( onosCountersRaw[i] ) )
+ except ( ValueError, TypeError ):
+ main.log.error( "Could not parse counters response from ONOS" +
+ str( i + 1 ) )
+ main.log.warn( repr( onosCountersRaw[ i ] ) )
+ return main.FALSE
+
+ testCounters = {}
+ # make a list of all the "TestON-*" counters in ONOS
+ # lookes like a dict whose keys are the name of the ONOS node and values
+ # are a list of the counters. I.E.
+ # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
+ # {"name":"TestON-Partitions","value":56} ]
+ # }
+ # NOTE: There is an assumtion that all nodes are active
+ # based on the above for loops
+ for controller in enumerate( onosCounters ):
+ for dbType in controller[1]:
+ for dbName, items in dbType.iteritems():
+ for item in items:
+ if 'TestON' in item['name']:
+ node = 'ONOS' + str( controller[0] + 1 )
+ try:
+ testCounters[node].append( item )
+ except KeyError:
+ testCounters[node] = [ item ]
+ # compare the counters on each node
+ tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
if all( tmp ):
consistent = main.TRUE
else:
consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters" )
- for node in onosCounters:
- main.log.debug( node )
+ main.log.error( "ONOS nodes have different values for counters:\n" +
+ testCounters )
+ return ( onosCounters, consistent )
+def counterCheck( counterName, counterValue ):
+ """
+ Checks that TestON counters are consistent across all nodes and that
+ specified counter is in ONOS with the given value
+ """
+ import json
+ correctResults = main.TRUE
+ # Get onos counters results and consistentCheck
+ onosCounters, consistent = main.Counters.consistentCheck()
# Check for correct values
for i in range( main.numCtrls ):
- try:
- current = json.loads( onosCounters[i] )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( i + 1 ) )
- main.log.warn( repr( onosCounters[ i ] ) )
- return main.FALSE
+ current = onosCounters[i]
onosValue = None
try:
for database in current:
diff --git a/TestON/tests/HAminorityRestart/dependencies/onos-gen-partitions b/TestON/tests/HAminorityRestart/dependencies/onos-gen-partitions
new file mode 100755
index 0000000..bf9a77b
--- /dev/null
+++ b/TestON/tests/HAminorityRestart/dependencies/onos-gen-partitions
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+'''
+ Generate the partitions json file from the $OC* environment variables
+
+ Usage: onos-gen-partitions [output file]
+ If output file is not provided, the json is written to stdout.
+'''
+
+from os import environ
+from collections import deque, OrderedDict
+import re
+import json
+import sys
+
+convert = lambda text: int(text) if text.isdigit() else text.lower()
+alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
+
+def get_OC_vars():
+ vars = []
+ for var in environ:
+ if re.match(r"OC[0-9]+", var):
+ vars.append(var)
+ return sorted(vars, key=alphanum_key)
+
+def get_nodes(vars, port=9876):
+ node = lambda k: { 'id': k, 'ip': k, 'tcpPort': port }
+ return [ node(environ[v]) for v in vars ]
+
+def generate_permutations(nodes, k):
+ l = deque(nodes)
+ perms = {}
+ for i in range(1, len(nodes)+1):
+ perms['p%d' % i] = list(l)[:k]
+ l.rotate(-1)
+ return OrderedDict(sorted(perms.iteritems(), key=lambda (k, v): alphanum_key(k)))
+
+def generate_permutations2(nodes, k):
+ l = deque(nodes)
+ perms = {}
+ for i in range(1, (len(nodes) + 1) / 2 + 1):
+ perms['p%d' % i] = list(l)[:k]
+ l.rotate(-2)
+ return OrderedDict(sorted(perms.iteritems(), key=lambda (k, v): alphanum_key(k)))
+
+
+if __name__ == '__main__':
+ vars = get_OC_vars()
+ nodes = get_nodes(vars)
+ partitions = generate_permutations2(nodes, 3)
+ data = {
+ 'nodes': nodes,
+ 'partitions': partitions
+ }
+ output = json.dumps(data, indent=4)
+
+ if len(sys.argv) == 2:
+ filename = sys.argv[1]
+ with open(filename, 'w') as f:
+ f.write(output)
+ else:
+ print output