Updates to HATests including app subsystem
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index d6e3daf..b3894b7 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -59,36 +59,32 @@
# set global variables
global ONOS1Ip
global ONOS1Port
- global ONOS2Ip
global ONOS2Port
- global ONOS3Ip
global ONOS3Port
- global ONOS4Ip
global ONOS4Port
- global ONOS5Ip
global ONOS5Port
- global ONOS6Ip
global ONOS6Port
- global ONOS7Ip
global ONOS7Port
global numControllers
numControllers = int( main.params[ 'num_controllers' ] )
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
- ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
- ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
- ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
- ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
+ global CLIs
+ CLIs = []
+ global nodes
+ nodes = []
+ for i in range( 1, numControllers + 1 ):
+ CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ nodes.append( getattr( main, 'ONOS' + str( i ) ) )
+
main.step( "Applying cell variable to environment" )
cellResult = main.ONOSbench.setCell( cellName )
verifyResult = main.ONOSbench.verifyCell()
@@ -96,14 +92,10 @@
# FIXME:this is short term fix
main.log.report( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
+
main.log.report( "Uninstalling ONOS" )
- main.ONOSbench.onosUninstall( ONOS1Ip )
- main.ONOSbench.onosUninstall( ONOS2Ip )
- main.ONOSbench.onosUninstall( ONOS3Ip )
- main.ONOSbench.onosUninstall( ONOS4Ip )
- main.ONOSbench.onosUninstall( ONOS5Ip )
- main.ONOSbench.onosUninstall( ONOS6Ip )
- main.ONOSbench.onosUninstall( ONOS7Ip )
+ for node in nodes:
+ main.ONOSbench.onosUninstall( node.ip_address )
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
@@ -116,6 +108,8 @@
main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
+ if gitPullResult == main.ERROR:
+ main.log.error( "Error pulling git branch" )
main.step( "Using mvn clean & install" )
cleanInstallResult = main.ONOSbench.cleanInstall()
@@ -206,6 +200,71 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
+ roleCall = main.TRUE
+ roleCheck = main.TRUE
+ try:
+ for i in range( 1, 29 ): # switches 1 through 28
+ ip = nodes[ 0 ].ip_address # ONOS1
+ # set up correct variables:
+ if i == 1:
+ deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
+ elif i == 2:
+ deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
+ elif i == 3:
+ deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
+ elif i == 4:
+ deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
+ elif i == 5:
+ deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
+ elif i == 6:
+ deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
+ elif i == 7:
+ deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
+ elif i >= 8 and i <= 17:
+ dpid = '3' + str( i ).zfill( 3 )
+ deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
+ elif i >= 18 and i <= 27:
+ dpid = '6' + str( i ).zfill( 3 )
+ deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
+ elif i == 28:
+ deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
+ else:
+ main.log.error( "You didn't write an else statement for " +
+ "switch s" + str( i ) )
+ # Assign switch
+ assert deviceId, "No device id for s" + str( i ) + " in ONOS"
+ # TODO: make this controller dynamic
+ roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
+ ip )
+ # Check assignment
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ roleCheck = roleCheck and main.TRUE
+ else:
+ roleCheck = roleCheck and main.FALSE
+ main.log.error( "Error, controller " + ip + " is not" +
+ " master " + "of device " +
+ str( deviceId ) )
+ except ( AttributeError, AssertionError ):
+ main.log.exception( "Something is wrong with ONOS device view" )
+ main.log.info( main.ONOScli1.devices() )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=roleCall,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with deviceRole calls" )
+
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=roleCheck,
+ onpass="Switches were successfully reassigned to designated " +
+ "controller",
+ onfail="Switches were not successfully reassigned" )
+ mastershipCheck = mastershipCheck and roleCall and roleCheck
+ utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch" +
+ " mastership" )
+
def CASE3( self, main ):
"""
Assign intents
@@ -225,7 +284,13 @@
# install onos-app-fwd
main.log.info( "Install reactive forwarding app" )
- main.ONOScli1.featureInstall( "onos-app-fwd" )
+ appResults = main.ONOScli1.activateApp( "org.onosproject.fwd" )
+
+ # FIXME: add this to asserts
+ appCheck = main.ONOScli1.appToIDCheck()
+ if appCheck != main.TRUE:
+ main.log.warn( CLIs[0].apps() )
+ main.log.warn( CLIs[0].appIDs() )
# REACTIVE FWD test
pingResult = main.FALSE
@@ -242,11 +307,16 @@
# uninstall onos-app-fwd
main.log.info( "Uninstall reactive forwarding app" )
- main.ONOScli1.featureUninstall( "onos-app-fwd" )
- # timeout for fwd flows
- time.sleep( 10 )
+ appResults = appResults and main.ONOScli1.deactivateApp( "org.onosproject.fwd" )
+ appCheck2 = main.ONOScli1.appToIDCheck()
+ if appCheck2 != main.TRUE:
+ main.log.warn( CLIs[0].apps() )
+ main.log.warn( CLIs[0].appIDs() )
- main.step( "Add host intents" )
+ # timeout for fwd flows
+ time.sleep( 11 )
+
+ main.step( "Add host intents" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -268,33 +338,45 @@
host1Id = host1Dict.get( 'id', None )
host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
- tmpId = main.ONOScli1.addHostIntent(
- host1Id,
- host2Id )
+ tmpId = main.ONOScli1.addHostIntent( host1Id, host2Id )
if tmpId:
main.log.info( "Added intent with id: " + tmpId )
intentIds.append( tmpId )
else:
- main.log.error( "addHostIntent reutrned None" )
+ main.log.error( "addHostIntent returned: " +
+ repr( tmpId ) )
else:
- main.log.error( "Error, getHost() failed" )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.error( "Error, getHost() failed for h" + str( i ) +
+ " and/or h" + str( i + 10 ) )
+ hosts = main.ONOScli1.hosts()
+ main.log.warn( "Hosts output: " )
+ try:
+ main.log.warn( json.dumps( json.loads( hosts ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( hosts ) )
hostResult = main.FALSE
+ # FIXME: DEBUG
+ intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
main.log.info( "Intents in ONOS: " + str( onosIds ) )
for intent in intentIds:
if intent in onosIds:
- pass # intent submitted is still in onos
+ pass # intent submitted is in onos
else:
intentAddResult = False
+ # FIXME: DEBUG
+ if intentAddResult:
+ intentStop = time.time()
+ else:
+ intentStop = None
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
- installedCheck = True
+ installedCheck = True
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
try:
@@ -327,12 +409,9 @@
indent=4,
separators=( ',', ': ' ) ) )
# check for all intent partitions
- # check for election
topics = []
for i in range( 14 ):
topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
main.log.debug( topics )
ONOStopics = [ j['topic'] for j in parsedLeaders ]
for topic in topics:
@@ -476,6 +555,7 @@
Ping across added host intents
"""
import json
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -678,7 +758,6 @@
Reading state of ONOS
"""
import json
- import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -691,9 +770,9 @@
# ( intents,flows, topology,... ) from each ONOS node
# We can then compare them with each other and also with past states
- main.step( "Get the Mastership of each switch from each controller" )
+ main.step( "Check that each switch has a master" )
global mastershipState
- mastershipState = []
+ mastershipState = '[]'
# Assert that each device has a master
rolesNotNull = main.ONOScli1.rolesNotNull()
@@ -1113,17 +1192,10 @@
main.log.report( description )
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- temp = ()
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
+ node = main.ONOS1
+ temp = ( node, node.name, node.ip_address, 6633 )
ctrls.append( temp )
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1137,9 +1209,9 @@
startTime = time.time()
# Give time for Gossip to work
while topoResult == main.FALSE and elapsed < 60:
- count = count + 1
+ count += 1
if count > 1:
- # TODO: Depricate STS usage
+ # TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
cliStart = time.time()
devices = []
@@ -1277,7 +1349,7 @@
" seconds for link down to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
- onpass="Link down succesful",
+ onpass="Link down successful",
onfail="Failed to bring link down" )
# TODO do some sort of check here
@@ -1304,7 +1376,7 @@
" seconds for link up to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
- onpass="Link up succesful",
+ onpass="Link up successful",
onfail="Failed to bring link up" )
# TODO do some sort of check here
@@ -1340,7 +1412,7 @@
if device and device[ 'available' ] is False:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Kill switch succesful",
+ onpass="Kill switch successful",
onfail="Failed to kill switch?" )
def CASE12( self, main ):
@@ -1371,7 +1443,6 @@
main.step( "Add back " + switch )
main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
- # TODO: New dpid or same? Ask Thomas?
for peer in links:
main.Mininet1.addLink( switch, peer )
main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
@@ -1387,7 +1458,7 @@
if device and device[ 'available' ]:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="add switch succesful",
+ onpass="add switch successful",
onfail="Failed to add switch?" )
def CASE13( self, main ):
@@ -1400,25 +1471,15 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
# printing colors to terminal
- colors = {}
- colors[ 'cyan' ] = '\033[96m'
- colors[ 'purple' ] = '\033[95m'
- colors[ 'blue' ] = '\033[94m'
- colors[ 'green' ] = '\033[92m'
- colors[ 'yellow' ] = '\033[93m'
- colors[ 'red' ] = '\033[91m'
- colors[ 'end' ] = '\033[0m'
+ colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
+ 'blue': '\033[94m', 'green': '\033[92m',
+ 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
description = "Test Cleanup"
main.log.report( description )
main.case( description )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
- main.step( "Checking ONOS Logs for errors" )
- print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
- colors[ 'end' ]
- print main.ONOSbench.checkLogs( ONOS1Ip )
-
main.step( "Copying MN pcap and ONOS log files to test station" )
testname = main.TEST
teststationUser = main.params[ 'TESTONUSER' ]
@@ -1456,7 +1517,15 @@
main.ONOSbench.handle.expect( "\$" )
# sleep so scp can finish
time.sleep( 10 )
+
+ main.step( "Stopping Mininet" )
main.Mininet1.stopNet()
+
+ main.step( "Checking ONOS Logs for errors" )
+ print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
+ colors[ 'end' ]
+ print main.ONOSbench.checkLogs( ONOS1Ip )
+
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -1472,11 +1541,10 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
+
leaderResult = main.TRUE
- # install app on onos 1
main.log.info( "Install leadership election app" )
- main.ONOScli1.featureInstall( "onos-app-election" )
- # wait for election
+ main.ONOScli1.activateApp( "org.onosproject.election" )
# check for leader
leader = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
@@ -1502,32 +1570,6 @@
"'" )
leaderResult = main.FALSE
- # install on other nodes and check for leader.
- # Should be onos1 and each app should show the same leader
- for controller in range( 2, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- node.featureInstall( "onos-app-election" )
- leaderN = node.electionTestLeader()
- # verify leader is ONOS1
- if leaderN == ONOS1Ip:
- # all is well
- pass
- elif leaderN == main.FALSE:
- # error in response
- # TODO: add check for "Command not found:" in the driver, this
- # means the app isn't loaded
- main.log.report( "Something is wrong with " +
- "electionTestLeader function, check the" +
- " error logs" )
- leaderResult = main.FALSE
- elif leader != leaderN:
- leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
- str( leaderN ) +
- " as the leader of the election app. Leader" +
- " should be " +
- str( leader ) )
if leaderResult:
main.log.report( "Leadership election tests passed( consistent " +
"view of leader across listeners and a leader " +
@@ -1551,7 +1593,7 @@
main.case( description )
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
- # TODO: do some sanity checking on leader before using it
+ # do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader == ONOS1Ip:
oldLeader = getattr( main, "ONOScli1" )
@@ -1572,7 +1614,6 @@
onfail="App was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
- leaderList = []
leaderN = main.ONOScli1.electionTestLeader()
if leaderN == leader:
main.log.report( "ONOS still sees " + str( leaderN ) +