Merge branch 'devl/newFunctionality' of https://github.com/OPENNETWORKINGLAB/ONLabTest into devl/newFunctionality
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 1d41acf..8187e4e 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -79,8 +79,6 @@
self.testResult = "Summary"
self.stepName = ""
self.stepCache = ""
- # make this into two lists? one for step names, one for results?
- # this way, the case result could be a true AND of these results
self.EXPERIMENTAL_MODE = False
self.test_target = None
self.lastcommand = None
diff --git a/TestON/core/testparser.py b/TestON/core/testparser.py
index 37f50f0..24b1ca2 100644
--- a/TestON/core/testparser.py
+++ b/TestON/core/testparser.py
@@ -16,7 +16,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with TestON. If not, see <http://www.gnu.org/licenses/>.
+ along with TestON. If not, see <http://www.gnu.org/licenses/>.
'''
@@ -26,21 +26,21 @@
def __init__(self,testFile):
try :
testFileHandler = open(testFile, 'r')
- except IOError:
+ except IOError:
print "No such file "+testFile
sys.exit(0)
-
+
testFileList = testFileHandler.readlines()
- self.testscript = testFileList
+ self.testscript = testFileList
self.caseCode = {}
self.caseBlock = ''
self.statementsList = []
- index = 0
+ index = 0
self.statementsList = []
#initialSpaces = len(line) -len(line.lstrip())
while index < len(testFileList):
testFileList[index] = re.sub("^\s{8}|^\s{4}", "", testFileList[index])
- # Skip multiline comments
+ # Skip multiline comments
if re.match('^(\'\'\')|^(\"\"\")',testFileList[index],0) :
index = index + 1
try :
@@ -48,16 +48,15 @@
index = index + 1
except IndexError,e:
print ''
-
- # skip empty lines and single line comments
+ # skip empty lines and single line comments
elif not re.match('#|^\s*$',testFileList[index],0):
self.statementsList.append(testFileList[index])
index = index + 1
-
+
def case_code(self):
- index = 0
- statementsList = self.statementsList
+ index = 0
+ statementsList = self.statementsList
while index < len(statementsList):
#print statementsList[index]
m= re.match('def\s+CASE(\d+)',statementsList[index],0)
@@ -76,18 +75,16 @@
except IndexError,e:
#print 'IndexError'
print ''
-
self.caseCode [str(m.group(1))] = self.caseBlock
#print "Case CODE "+self.caseCode [str(m.group(1))]
index = index + 1
-
- return self.caseCode
-
+ return self.caseCode
+
def step_code(self,caseStatements):
index = 0
- step = 0
- stepCode = {}
- step_flag = False
+ step = 0
+ stepCode = {}
+ step_flag = False
while index < len(caseStatements):
m= re.match('main\.step',caseStatements[index],0)
stepBlock = ''
@@ -99,13 +96,13 @@
while i < index :
block += caseStatements[i]
i = i + 1
- stepCode[step] = block
+ stepCode[step] = block
step = step + 1
- stepBlock= stepBlock + caseStatements[index]
+ stepBlock = stepBlock + caseStatements[index]
index = index + 1
try :
while not re.match('main\.step',caseStatements[index],0) :
- stepBlock= stepBlock + caseStatements[index]
+ stepBlock = stepBlock + caseStatements[index]
if index < len(caseStatements)-1:
index = index + 1
else :
@@ -121,11 +118,10 @@
if not step_flag :
stepCode[step] = "".join(caseStatements)
return stepCode
-
+
def getStepCode(self):
case_step_code = {}
case_block = self.case_code()
-
for case in case_block :
case_step_code[case] = {}
step_block = self.step_code(case_block[case])
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 3b5a5c7..6a31a60 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -368,6 +368,64 @@
main.log.info( self.name + ": \n---\n" + response )
return main.FALSE
+ def pingallHosts( self, hostList, pingType='ipv4' ):
+ """
+ Ping all specified hosts with a specific ping type
+
+ Acceptable pingTypes:
+ - 'ipv4'
+ - 'ipv6'
+
+ Acceptable hostList:
+ - ['h1','h2','h3','h4']
+
+ Returns main.TRUE if all hosts specified can reach
+ each other
+
+ Returns main.FALSE if one or more of hosts specified
+ cannot reach each other"""
+
+ if pingType == "ipv4":
+ cmd = " ping -c 1 -i 1 -W 8 "
+ elif pingType == "ipv6":
+ cmd = " ping6 -c 1 -i 1 -W 8 "
+ else:
+ main.log.warn( "Invalid pingType specified" )
+ return
+
+ try:
+ main.log.info( "Testing reachability between specified hosts" )
+
+ isReachable = main.TRUE
+
+ for host in hostList:
+ listIndex = hostList.index(host)
+ # List of hosts to ping other than itself
+ pingList = hostList[:listIndex] + hostList[(listIndex+1):]
+
+ for temp in pingList:
+ # Current host pings all other hosts specified
+ pingCmd = str(host) + cmd + str(temp)
+ self.handle.sendline( pingCmd )
+ i = self.handle.expect( [ pingCmd, pexpect.TIMEOUT ] )
+ j = self.handle.expect( [ "mininet>", pexpect.TIMEOUT ] )
+ response = self.handle.before
+ if re.search( ',\s0\%\spacket\sloss', response ):
+ main.log.info( str(host) + " -> " + str(temp) )
+ else:
+ main.log.info( str(host) + " -> X ("+str(temp)+") "
+ " Destination Unreachable" )
+ # One of the host to host pair is unreachable
+ isReachable = main.FALSE
+
+ return isReachable
+
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+
def pingHost( self, **pingParams ):
"""
Ping from one mininet host to another
@@ -1378,7 +1436,7 @@
response = main.FALSE
return response
- def arping( self, host="", ip="10.128.20.211" ):
+ def arping( self, host="", ip="10.128.20.211", ethDevice="" ):
"""
Description:
Sends arp message from mininet host for hosts discovery
@@ -1388,7 +1446,9 @@
ip - ip address that does not exist in the network so there would
be no reply.
"""
- cmd = " py " + host + ".cmd(\"arping -c 1 " + ip + "\")"
+ if ethDevice:
+ ethDevice = '-I ' + ethDevice + ' '
+ cmd = " py " + host + ".cmd(\"arping -c 1 " + ethDevice + ip + "\")"
try:
main.log.warn( "Sending: " + cmd )
self.handle.sendline( cmd )
@@ -1568,7 +1628,7 @@
if switchesJson == "": # if rest call fails
main.log.error(
self.name +
- ".compare_switches(): Empty JSON object given from ONOS" )
+ ".compareSwitches(): Empty JSON object given from ONOS" )
return main.FALSE
onos = switchesJson
onosDPIDs = []
@@ -1877,7 +1937,7 @@
if onosMAC == mnIntf[ 'hw_addr' ].lower() :
match = True
for ip in mnIntf[ 'ips' ]:
- if ip in onosHost[ 'ips' ]:
+ if ip in onosHost[ 'ipAddresses' ]:
pass # all is well
else:
# misssing ip
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 71bf427..0004cfc 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -92,7 +92,7 @@
return main.ERROR
else:
main.log.error( "Error, unexpected output in the ping file" )
- main.log.warn( outputs )
+ #main.log.warn( outputs )
return main.TRUE
def arping( self, host="", ip="10.128.20.211" ):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index a67205c..0052d40 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1814,15 +1814,18 @@
"""
try:
tempFlows = json.loads( self.flows() )
+ #print tempFlows[0]
returnValue = main.TRUE
+
for device in tempFlows:
for flow in device.get( 'flows' ):
if flow.get( 'state' ) != 'ADDED' and flow.get( 'state' ) != \
'PENDING_ADD':
main.log.info( self.name + ": flow Id: " +
- flow.get( 'flowId' ) +
+ flow.get( 'groupId' ) +
" | state:" + flow.get( 'state' ) )
returnValue = main.FALSE
+
return returnValue
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
@@ -3129,6 +3132,20 @@
try:
cmdStr = "set-test-add " + str( setName ) + " " + str( values )
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
positiveMatch = "\[(.*)\] was added to the set " + str( setName )
negativeMatch = "\[(.*)\] was already in set " + str( setName )
@@ -3184,6 +3201,20 @@
else:
cmdStr += str( setName ) + " " + str( values )
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
if clear:
@@ -3269,6 +3300,20 @@
cmdStr = "set-test-get "
cmdStr += setName + " " + values
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
@@ -3334,7 +3379,7 @@
Required arguments:
setName - The name of the set to remove from.
returns:
- The integer value of the size returned or
+ The integer value of the size returned or
None on error
"""
try:
@@ -3348,6 +3393,20 @@
cmdStr = "set-test-get -s "
cmdStr += setName
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
match = re.search( pattern, output )
@@ -3450,6 +3509,20 @@
cmdStr += "-i "
cmdStr += counter
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
pattern = counter + " was incremented to (\d+)"
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 81351ed..09bc3d1 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -20,6 +20,7 @@
import time
import pexpect
import os.path
+from requests.models import Response
sys.path.append( "../" )
from drivers.common.clidriver import CLI
@@ -483,6 +484,22 @@
main.cleanup()
main.exit()
+ def getBranchName( self ):
+ self.handle.sendline( "cd " + self.home )
+ self.handle.expect( "ONOS\$" )
+ self.handle.sendline( "git name-rev --name-only HEAD" )
+ self.handle.expect( "git name-rev --name-only HEAD" )
+ self.handle.expect( "\$" )
+
+ lines = self.handle.before.splitlines()
+ if lines[1] == "master":
+ return "master"
+ elif lines[1] == "onos-1.0":
+ return "onos-1.0"
+ else:
+ main.log.info( lines[1] )
+ return "unexpected ONOS branch for SDN-IP test"
+
def getVersion( self, report=False ):
"""
Writes the COMMIT number to the report to be parsed
@@ -928,7 +945,7 @@
"""
try:
self.handle.sendline( "" )
- self.handle.expect( "\$" )
+ self.handle.expect( "\$", timeout=60 )
self.handle.sendline( "onos-uninstall " + str( nodeIp ) )
self.handle.expect( "\$" )
@@ -937,6 +954,9 @@
# onos-uninstall command does not return any text
return main.TRUE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": Timeout in onosUninstall" )
+ return main.FALSE
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1942,10 +1962,10 @@
order = [ "OC1", "OC2", "OC3","OC4","OC5","OC6","OC7","OCN","OCI" ]
ONOSIps = []
- try:
+ try:
if os.path.exists("myIps"):
ipFile = open("myIps","r+")
- else:
+ else:
ipFile = open("myIps","w+")
fileONOSIps = ipFile.readlines()
@@ -2008,10 +2028,12 @@
- output modes:
"s" - Simple. Quiet output mode that just prints
- the occurances of each search term
+ the occurences of each search term
- "d" - Detailed. Prints occurances as well as the entire
- line for each of the last 5 occurances
+ "d" - Detailed. Prints number of occurences as well as the entire
+ line for each of the last 5 occurences
+
+ - returns total of the number of instances of all search terms
'''
main.log.info("========================== Log Report ===========================\n")
@@ -2023,7 +2045,7 @@
for term in range(len(searchTerms)):
logLines[term][0] = searchTerms[term]
-
+ totalHits = 0
for term in range(len(searchTerms)):
cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep " + searchTerms[term]
self.handle.sendline(cmd)
@@ -2041,7 +2063,8 @@
main.log.info( str(count[0]) + ": " + str(count[1]) )
if term == len(searchTerms)-1:
print("\n")
-
+ totalHits += int(count[1])
+
if outputMode != "s" and outputMode != "S":
outputString = ""
for i in logLines:
@@ -2053,6 +2076,7 @@
main.log.info(outputString)
main.log.info("================================================================\n")
+ return totalHits
def getOnosIpFromEnv(self):
diff --git a/TestON/drivers/common/cli/quaggaclidriver.py b/TestON/drivers/common/cli/quaggaclidriver.py
index 1c63206..f391e3a 100644
--- a/TestON/drivers/common/cli/quaggaclidriver.py
+++ b/TestON/drivers/common/cli/quaggaclidriver.py
@@ -40,9 +40,9 @@
ip_address="1.1.1.1",
port=self.port,
pwd=self.pwd )
- main.log.info( "connect parameters:" + str( self.user_name ) + ";"
- + str( self.ip_address ) + ";" + str( self.port )
- + ";" + str(self.pwd ) )
+ #main.log.info( "connect parameters:" + str( self.user_name ) + ";"
+ # + str( self.ip_address ) + ";" + str( self.port )
+ # + ";" + str(self.pwd ) )
if self.handle:
# self.handle.expect( "",timeout=10 )
@@ -186,7 +186,7 @@
return intents
# This method extracts all actual routes from ONOS CLI
- def extractActualRoutes( self, getRoutesResult ):
+ def extractActualRoutesOneDotZero( self, getRoutesResult ):
routesJsonObj = json.loads( getRoutesResult )
allRoutesActual = []
@@ -199,6 +199,18 @@
return sorted( allRoutesActual )
+ def extractActualRoutesMaster( self, getRoutesResult ):
+ routesJsonObj = json.loads( getRoutesResult )
+
+ allRoutesActual = []
+ for route in routesJsonObj['routes4']:
+ if route[ 'prefix' ] == '172.16.10.0/24':
+ continue
+ allRoutesActual.append(
+ route[ 'prefix' ] + "/" + route[ 'nextHop' ] )
+
+ return sorted( allRoutesActual )
+
# This method extracts all actual route intents from ONOS CLI
def extractActualRouteIntents( self, getIntentsResult ):
intents = []
@@ -489,7 +501,7 @@
if routesAdded == numRoutes:
return main.TRUE
return main.FALSE
-
+
# Please use deleteRoutes method instead of this one!
def delRoute( self, net, numRoutes, routeRate ):
try:
@@ -562,7 +574,7 @@
child.expect( "Flow table show" )
count = 0
while True:
- i = child.expect( [ '17\d\.\d{1,3}\.\d{1,3}\.\d{1,3}',
+ i = child.expect( [ '17\d\.\d{1,3}\.\d{1,3}\.\d{1,3}',
'CLI#', pexpect.TIMEOUT ] )
if i == 0:
count = count + 1
diff --git a/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py b/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py
index caaa864..3f41180 100644
--- a/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py
+++ b/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py
@@ -1,14 +1,15 @@
"""
Wrapper functions for FuncIntent
This functions include Onosclidriver and Mininetclidriver driver functions
+ Author: kelvin@onlab.us
"""
def __init__( self ):
self.default = ''
def hostIntent( main,
- name="",
- host1="",
- host2="",
+ name,
+ host1,
+ host2,
host1Id="",
host2Id="",
mac1="",
@@ -19,7 +20,40 @@
sw2="",
expectedLink=0 ):
"""
- Add host intents
+ Description:
+ Verify add-host-intent
+ Steps:
+ - Discover hosts
+ - Add host intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of host intent to add eg. IPV4 | VLAN | Dualstack
+ host1 - Name of first host
+ host2 - Name of second host
+ Optional:
+ host1Id - ONOS id of the first host eg. 00:00:00:00:00:01/-1
+ host2Id - ONOS id of the second host
+ mac1 - Mac address of first host
+ mac2 - Mac address of the second host
+ vlan1 - Vlan tag of first host, defaults to -1
+ vlan2 - Vlan tag of second host, defaults to -1
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
"""
import time
@@ -30,18 +64,18 @@
global itemName
itemName = name
- h1Name = host1
- h2Name = host2
h1Id = host1Id
h2Id = host2Id
h1Mac = mac1
h2Mac = mac2
vlan1 = vlan1
vlan2 = vlan2
+ hostNames = [ host1 , host2 ]
intentsId = []
stepResult = main.TRUE
pingResult = main.TRUE
intentResult = main.TRUE
+ removeIntentResult = main.TRUE
flowResult = main.TRUE
topoResult = main.TRUE
linkDownResult = main.TRUE
@@ -49,21 +83,17 @@
if main.hostsData:
if not h1Mac:
- h1Mac = main.hostsData[ h1Name ][ 'mac' ]
+ h1Mac = main.hostsData[ host1 ][ 'mac' ]
if not h2Mac:
- h2Mac = main.hostsData[ h2Name ][ 'mac' ]
- print '1', main.hostsData[ h1Name ]
- print '2', main.hostsData[ h2Name ]
- if main.hostsData[ h1Name ][ 'vlan' ] != '-1':
- print 'vlan1', main.hostsData[ h1Name ][ 'vlan' ]
- vlan1 = main.hostsData[ h1Name ][ 'vlan' ]
- if main.hostsData[ h2Name ][ 'vlan' ] != '-1':
- print 'vlan2', main.hostsData[ h2Name ][ 'vlan' ]
- vlan2 = main.hostsData[ h2Name ][ 'vlan' ]
+ h2Mac = main.hostsData[ host2 ][ 'mac' ]
+ if main.hostsData[ host1 ][ 'vlan' ] != '-1':
+ vlan1 = main.hostsData[ host1 ][ 'vlan' ]
+ if main.hostsData[ host2 ][ 'vlan' ] != '-1':
+ vlan2 = main.hostsData[ host2 ][ 'vlan' ]
if not h1Id:
- h1Id = main.hostsData[ h1Name ][ 'id' ]
+ h1Id = main.hostsData[ host1 ][ 'id' ]
if not h2Id:
- h2Id = main.hostsData[ h2Name ][ 'id' ]
+ h2Id = main.hostsData[ host2 ][ 'id' ]
assert h1Id and h2Id, "You must specify host IDs"
if not ( h1Id and h2Id ):
@@ -72,8 +102,8 @@
# Discover hosts using arping
main.log.info( itemName + ": Discover host using arping" )
- main.Mininet1.arping( host=h1Name )
- main.Mininet1.arping( host=h2Name )
+ main.Mininet1.arping( host=host1 )
+ main.Mininet1.arping( host=host2 )
host1 = main.CLIs[ 0 ].getHost( mac=h1Mac )
host2 = main.CLIs[ 0 ].getHost( mac=h2Mac )
@@ -83,27 +113,32 @@
hostIdTwo=h2Id )
intentsId.append( intent1 )
time.sleep( 5 )
- intent2 = main.CLIs[ 0 ].addHostIntent( hostIdOne=h2Id,
- hostIdTwo=h1Id )
- intentsId.append( intent2 )
# Check intents state
- time.sleep( 50 )
+ time.sleep( 30 )
intentResult = checkIntentState( main, intentsId )
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
# Verify flows
checkFlowsState( main )
# Ping hosts
- pingHost( main, h1Name, h2Name )
+ firstPingResult = pingallHosts( main, hostNames )
+ if not firstPingResult:
+ main.log.debug( "First ping failed, there must be" +
+ " something wrong with ONOS performance" )
+
# Ping hosts again...
- pingResult = pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
time.sleep( 5 )
# Test rerouting if these variables exist
if sw1 and sw2 and expectedLink:
# link down
- link( main, sw1, sw2, "down" )
+ linkDownResult = link( main, sw1, sw2, "down" )
intentResult = intentResult and checkIntentState( main, intentsId )
# Verify flows
@@ -113,37 +148,49 @@
topoResult = checkTopology( main, expectedLink )
# Ping hosts
- pingResult = pingResult and pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
intentResult = checkIntentState( main, intentsId )
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
# link up
- link( main, sw1, sw2, "up" )
+ linkUpResult = link( main, sw1, sw2, "up" )
time.sleep( 5 )
# Verify flows
checkFlowsState( main )
# Check OnosTopology
- topoResult = checkTopology( main, expectedLink )
+ topoResult = checkTopology( main, main.numLinks )
# Ping hosts
- pingResult = pingResult and pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
- # Remove intents
- for intent in intentsId:
- main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
+ intentResult = checkIntentState( main, intentsId )
- print main.CLIs[ 0 ].intents()
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
stepResult = pingResult and linkDownResult and linkUpResult \
- and intentResult
+ and intentResult and removeIntentResult
return stepResult
def pointIntent( main,
- name="",
- host1="",
- host2="",
+ name,
+ host1,
+ host2,
deviceId1="",
deviceId2="",
port1="",
@@ -161,9 +208,54 @@
sw1="",
sw2="",
expectedLink=0 ):
+
"""
- Add Point intents
+ Description:
+ Verify add-point-intent
+ Steps:
+ - Get device ids | ports
+ - Add point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+ host1 - Name of first host
+ host2 - Name of second host
+ Optional:
+ deviceId1 - ONOS device id of the first switch, the same as the
+ location of the first host eg. of:0000000000000001/1,
+ located at device 1 port 1
+ deviceId2 - ONOS device id of the second switch
+ port1 - The port number where the first host is attached
+ port2 - The port number where the second host is attached
+ ethType - Ethernet type eg. IPV4, IPV6
+ mac1 - Mac address of first host
+ mac2 - Mac address of the second host
+ bandwidth - Bandwidth capacity
+ lambdaAlloc - Allocate lambda, defaults to False
+ ipProto - IP protocol
+ ip1 - IP address of first host
+ ip2 - IP address of second host
+ tcp1 - TCP port of first host
+ tcp2 - TCP port of second host
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
"""
+
import time
assert main, "There is no main variable"
assert name, "variable name is empty"
@@ -171,19 +263,21 @@
global itemName
itemName = name
- h1Name = host1
- h2Name = host2
+ host1 = host1
+ host2 = host2
+ hostNames = [ host1, host2 ]
intentsId = []
pingResult = main.TRUE
intentResult = main.TRUE
+ removeIntentResult = main.TRUE
flowResult = main.TRUE
topoResult = main.TRUE
linkDownResult = main.TRUE
linkUpResult = main.TRUE
# Adding bidirectional point intents
- main.log.info( itemName + ": Adding host intents" )
+ main.log.info( itemName + ": Adding point intents" )
intent1 = main.CLIs[ 0 ].addPointIntent( ingressDevice=deviceId1,
egressDevice=deviceId2,
portIngress=port1,
@@ -218,21 +312,30 @@
intentsId.append( intent2 )
# Check intents state
- time.sleep( 50 )
+ time.sleep( 30 )
intentResult = checkIntentState( main, intentsId )
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
# Verify flows
checkFlowsState( main )
# Ping hosts
- pingHost( main, h1Name, h2Name )
+ firstPingResult = pingallHosts( main, hostNames )
+ if not firstPingResult:
+ main.log.debug( "First ping failed, there must be" +
+ " something wrong with ONOS performance" )
+
# Ping hosts again...
- pingResult = pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
time.sleep( 5 )
+ # Test rerouting if these variables exist
if sw1 and sw2 and expectedLink:
# link down
- link( main, sw1, sw2, "down" )
+ linkDownResult = link( main, sw1, sw2, "down" )
intentResult = intentResult and checkIntentState( main, intentsId )
# Verify flows
@@ -242,36 +345,48 @@
topoResult = checkTopology( main, expectedLink )
# Ping hosts
- pingResult = pingResult and pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
intentResult = checkIntentState( main, intentsId )
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
# link up
- link( main, sw1, sw2, "up" )
+ linkUpResult = link( main, sw1, sw2, "up" )
time.sleep( 5 )
# Verify flows
checkFlowsState( main )
# Check OnosTopology
- topoResult = checkTopology( main, expectedLink )
+ topoResult = checkTopology( main, main.numLinks )
# Ping hosts
- pingResult = pingResult and pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
- # Remove intents
- for intent in intentsId:
- main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
+ intentResult = checkIntentState( main, intentsId )
- print main.CLIs[ 0 ].intents()
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
stepResult = pingResult and linkDownResult and linkUpResult \
- and intentResult
+ and intentResult and removeIntentResult
return stepResult
def singleToMultiIntent( main,
- name="",
- hostNames="",
+ name,
+ hostNames,
devices="",
ports=None,
ethType="",
@@ -285,17 +400,60 @@
sw2="",
expectedLink=0 ):
"""
- Add Single to Multi Point intents
- If main.hostsData is not defined, variables data should be passed in the
- same order index wise.
+ Verify Single to Multi Point intents
+ NOTE:If main.hostsData is not defined, variables data should be passed in the
+ same order index wise. All devices in the list should have the same
+ format, either all the devices have its port or it doesn't.
eg. hostName = [ 'h1', 'h2' ,.. ]
devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
ports = [ '1', '1', ..]
...
+ Description:
+ Verify add-single-to-multi-intent iterates through the list of given
+ host | devices and add intents
+ Steps:
+ - Get device ids | ports
+ - Add single to multi point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+ hostNames - List of host names
+ Optional:
+ devices - List of device ids in the same order as the hosts
+ in hostNames
+ ports - List of port numbers in the same order as the device in
+ devices
+ ethType - Ethernet type eg. IPV4, IPV6
+ macs - List of hosts mac address in the same order as the hosts in
+ hostNames
+ bandwidth - Bandwidth capacity
+ lambdaAlloc - Allocate lambda, defaults to False
+ ipProto - IP protocol
+ ipAddresses - IP addresses of host in the same order as the hosts in
+ hostNames
+ tcp - TCP ports in the same order as the hosts in hostNames
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
"""
+
import time
+ import copy
assert main, "There is no main variable"
- assert name, "variable name is empty"
assert hostNames, "You must specify hosts"
assert devices or main.hostsData, "You must specify devices"
@@ -304,84 +462,110 @@
tempHostsData = {}
intentsId = []
+ macsDict = {}
+ ipDict = {}
if hostNames and devices:
if len( hostNames ) != len( devices ):
- main.log.error( "hosts and devices does not have the same length" )
- print "hostNames = ", len( hostNames )
- print "devices = ", len( devices )
+ main.log.debug( "hosts and devices does not have the same length" )
+ #print "len hostNames = ", len( hostNames )
+ #print "len devices = ", len( devices )
return main.FALSE
if ports:
if len( ports ) != len( devices ):
main.log.error( "Ports and devices does " +
"not have the same length" )
- print "devices = ", len( devices )
- print "ports = ", len( ports )
+ #print "len devices = ", len( devices )
+ #print "len ports = ", len( ports )
return main.FALSE
+ for i in range( len( devices ) ):
+ macsDict[ devices[ i ] ] = macs[ i ]
else:
main.log.info( "Device Ports are not specified" )
elif hostNames and not devices and main.hostsData:
+ devices = []
main.log.info( "singleToMultiIntent function is using main.hostsData" )
- print main.hostsData
+ for host in hostNames:
+ devices.append( main.hostsData.get( host ).get( 'location' ) )
+ macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'mac' )
+ ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'ipAddresses' )
+ #print main.hostsData
+
+ #print 'host names = ', hostNames
+ #print 'devices = ', devices
+ #print "macsDict = ", macsDict
pingResult = main.TRUE
intentResult = main.TRUE
+ removeIntentResult = main.TRUE
flowResult = main.TRUE
topoResult = main.TRUE
linkDownResult = main.TRUE
linkUpResult = main.TRUE
+ devicesCopy = copy.copy( devices )
+ if ports:
+ portsCopy = copy.copy( ports )
+ main.log.info( itemName + ": Adding single point to multi point intents" )
# Adding bidirectional point intents
- main.log.info( itemName + ": Adding host intents" )
- intent1 = main.CLIs[ 0 ].addSinglepointToMultipointIntent(
- ingressDevice=deviceId1,
- egressDevice=deviceId2,
- portIngress=port1,
- portEgress=port2,
+ for i in range( len( devices ) ):
+ ingressDevice = devicesCopy[ i ]
+ egressDeviceList = copy.copy( devicesCopy )
+ egressDeviceList.remove( ingressDevice )
+ if ports:
+ portIngress = portsCopy[ i ]
+ portEgressList = copy.copy( portsCopy )
+ del portEgressList[ i ]
+ else:
+ portIngress = ""
+ portEgressList = None
+ if not macsDict:
+ srcMac = ""
+ else:
+ srcMac = macsDict[ ingressDevice ]
+ if srcMac == None:
+ main.log.debug( "There is no MAC in device - " + ingressDevice )
+ srcMac = ""
+
+ intentsId.append( main.CLIs[ 0 ].addSinglepointToMultipointIntent(
+ ingressDevice=ingressDevice,
+ egressDeviceList=egressDeviceList,
+ portIngress=portIngress,
+ portEgressList=portEgressList,
ethType=ethType,
- ethSrc=mac1,
- ethDst=mac2,
+ ethSrc=srcMac,
bandwidth=bandwidth,
lambdaAlloc=lambdaAlloc,
ipProto=ipProto,
- ipSrc=ip1,
- ipDst=ip2,
- tcpSrc=tcp1,
- tcpDst=tcp2 )
+ ipSrc="",
+ ipDst="",
+ tcpSrc="",
+ tcpDst="" ) )
- intentsId.append( intent1 )
- time.sleep( 5 )
- intent2 = main.CLIs[ 0 ].addPointIntent( ingressDevice=deviceId2,
- egressDevice=deviceId1,
- portIngress=port2,
- portEgress=port1,
- ethType=ethType,
- ethSrc=mac2,
- ethDst=mac1,
- bandwidth=bandwidth,
- lambdaAlloc=lambdaAlloc,
- ipProto=ipProto,
- ipSrc=ip2,
- ipDst=ip1,
- tcpSrc=tcp2,
- tcpDst=tcp1 )
- intentsId.append( intent2 )
+ pingResult = pingallHosts( main, hostNames )
# Check intents state
- time.sleep( 50 )
+ time.sleep( 30 )
intentResult = checkIntentState( main, intentsId )
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
# Verify flows
checkFlowsState( main )
# Ping hosts
- pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
# Ping hosts again...
- pingResult = pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
time.sleep( 5 )
+ # Test rerouting if these variables exist
if sw1 and sw2 and expectedLink:
# link down
- link( main, sw1, sw2, "down" )
+ linkDownResult = link( main, sw1, sw2, "down" )
intentResult = intentResult and checkIntentState( main, intentsId )
# Verify flows
@@ -391,94 +575,297 @@
topoResult = checkTopology( main, expectedLink )
# Ping hosts
- pingResult = pingResult and pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
intentResult = checkIntentState( main, intentsId )
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
# link up
- link( main, sw1, sw2, "up" )
+ linkUpResult = link( main, sw1, sw2, "up" )
time.sleep( 5 )
# Verify flows
checkFlowsState( main )
# Check OnosTopology
+ topoResult = checkTopology( main, main.numLinks )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
+ stepResult = pingResult and linkDownResult and linkUpResult \
+ and intentResult and removeIntentResult
+
+ return stepResult
+
+def multiToSingleIntent( main,
+ name,
+ hostNames,
+ devices="",
+ ports=None,
+ ethType="",
+ macs=None,
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ipAddresses="",
+ tcp="",
+ sw1="",
+ sw2="",
+ expectedLink=0 ):
+ """
+ Verify Single to Multi Point intents
+ NOTE:If main.hostsData is not defined, variables data should be passed in the
+ same order index wise. All devices in the list should have the same
+ format, either all the devices have its port or it doesn't.
+ eg. hostName = [ 'h1', 'h2' ,.. ]
+ devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
+ ports = [ '1', '1', ..]
+ ...
+ Description:
+ Verify add-multi-to-single-intent
+ Steps:
+ - Get device ids | ports
+ - Add single to multi point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+ hostNames - List of host names
+ Optional:
+ devices - List of device ids in the same order as the hosts
+ in hostNames
+ ports - List of port numbers in the same order as the device in
+ devices
+ ethType - Ethernet type eg. IPV4, IPV6
+ macs - List of hosts mac address in the same order as the hosts in
+ hostNames
+ bandwidth - Bandwidth capacity
+ lambdaAlloc - Allocate lambda, defaults to False
+ ipProto - IP protocol
+ ipAddresses - IP addresses of host in the same order as the hosts in
+ hostNames
+ tcp - TCP ports in the same order as the hosts in hostNames
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
+ """
+
+ import time
+ import copy
+ assert main, "There is no main variable"
+ assert hostNames, "You must specify hosts"
+ assert devices or main.hostsData, "You must specify devices"
+
+ global itemName
+ itemName = name
+ tempHostsData = {}
+ intentsId = []
+
+ macsDict = {}
+ ipDict = {}
+ if hostNames and devices:
+ if len( hostNames ) != len( devices ):
+ main.log.debug( "hosts and devices does not have the same length" )
+ #print "len hostNames = ", len( hostNames )
+ #print "len devices = ", len( devices )
+ return main.FALSE
+ if ports:
+ if len( ports ) != len( devices ):
+ main.log.error( "Ports and devices does " +
+ "not have the same length" )
+ #print "len devices = ", len( devices )
+ #print "len ports = ", len( ports )
+ return main.FALSE
+ for i in range( len( devices ) ):
+ macsDict[ devices[ i ] ] = macs[ i ]
+ else:
+ main.log.info( "Device Ports are not specified" )
+ elif hostNames and not devices and main.hostsData:
+ devices = []
+ main.log.info( "singleToMultiIntent function is using main.hostsData" )
+ for host in hostNames:
+ devices.append( main.hostsData.get( host ).get( 'location' ) )
+ macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'mac' )
+ ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'ipAddresses' )
+ #print main.hostsData
+
+ #print 'host names = ', hostNames
+ #print 'devices = ', devices
+ #print "macsDict = ", macsDict
+
+ pingResult = main.TRUE
+ intentResult = main.TRUE
+ removeIntentResult = main.TRUE
+ flowResult = main.TRUE
+ topoResult = main.TRUE
+ linkDownResult = main.TRUE
+ linkUpResult = main.TRUE
+
+ devicesCopy = copy.copy( devices )
+ if ports:
+ portsCopy = copy.copy( ports )
+ main.log.info( itemName + ": Adding single point to multi point intents" )
+ # Adding bidirectional point intents
+ for i in range( len( devices ) ):
+ egressDevice = devicesCopy[ i ]
+ ingressDeviceList = copy.copy( devicesCopy )
+ ingressDeviceList.remove( egressDevice )
+ if ports:
+ portEgress = portsCopy[ i ]
+ portIngressList = copy.copy( portsCopy )
+ del portIngressList[ i ]
+ else:
+ portEgress = ""
+ portIngressList = None
+ if not macsDict:
+ dstMac = ""
+ else:
+ dstMac = macsDict[ egressDevice ]
+ if dstMac == None:
+ main.log.debug( "There is no MAC in device - " + egressDevice )
+ dstMac = ""
+
+ intentsId.append( main.CLIs[ 0 ].addMultipointToSinglepointIntent(
+ ingressDeviceList=ingressDeviceList,
+ egressDevice=egressDevice,
+ portIngressList=portIngressList,
+ portEgress=portEgress,
+ ethType=ethType,
+ ethDst=dstMac,
+ bandwidth=bandwidth,
+ lambdaAlloc=lambdaAlloc,
+ ipProto=ipProto,
+ ipSrc="",
+ ipDst="",
+ tcpSrc="",
+ tcpDst="" ) )
+
+ pingResult = pingallHosts( main, hostNames )
+
+ # Check intents state
+ time.sleep( 30 )
+ intentResult = checkIntentState( main, intentsId )
+
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ # Ping hosts again...
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ time.sleep( 5 )
+
+ # Test rerouting if these variables exist
+ if sw1 and sw2 and expectedLink:
+ # link down
+ linkDownResult = link( main, sw1, sw2, "down" )
+ intentResult = intentResult and checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
topoResult = checkTopology( main, expectedLink )
# Ping hosts
- pingResult = pingResult and pingHost( main, h1Name, h2Name )
+ pingResult = pingResult and pingallHosts( main, hostNames )
- # Remove intents
- for intent in intentsId:
- main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
+ intentResult = checkIntentState( main, intentsId )
- print main.CLIs[ 0 ].intents()
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
+ # link up
+ linkUpResult = link( main, sw1, sw2, "up" )
+ time.sleep( 5 )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, main.numLinks )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
stepResult = pingResult and linkDownResult and linkUpResult \
- and intentResult
+ and intentResult and removeIntentResult
return stepResult
-def link( main, sw1, sw2, option):
-
- # link down
- main.log.info( itemName + ": Bring link " + option + "between " +
- sw1 + " and " + sw2 )
- main.Mininet1.link( end1=sw1, end2=sw2, option=option )
-
-def pingAllHost( main, hosts ):
+def pingallHosts( main, hostList, pingType="ipv4" ):
# Ping all host in the hosts list variable
- import itertools
-
- main.log.info( itemName + ": Ping host list - " + hosts )
- hostCombination = itertools.permutation( hosts, 2 )
+ print "Pinging : ", hostList
pingResult = main.TRUE
- for hostPair in hostCombination:
- pingResult = pingResult and main.Mininet.pingHost(
- src=hostPair[ 0 ],
- target=hostPair[ 1 ] )
- return pingResult
-
-def pingHost( main, h1Name, h2Name ):
-
- # Ping hosts
- main.log.info( itemName + ": Ping " + h1Name + " and " +
- h2Name )
- pingResult1 = main.Mininet1.pingHost( src=h1Name , target=h2Name )
- if not pingResult1:
- main.log.info( itemName + ": " + h1Name + " cannot ping "
- + h2Name )
- pingResult2 = main.Mininet1.pingHost( src=h2Name , target=h1Name )
- if not pingResult2:
- main.log.info( itemName + ": " + h2Name + " cannot ping "
- + h1Name )
- pingResult = pingResult1 and pingResult2
- if pingResult:
- main.log.info( itemName + ": Successfully pinged " +
- "both hosts" )
- else:
- main.log.info( itemName + ": Failed to ping " +
- "both hosts" )
+ pingResult = main.Mininet1.pingallHosts( hostList, pingType )
return pingResult
def getHostsData( main ):
"""
Use fwd app and pingall to discover all the hosts
"""
- """
- hosts json format:
- """
import json
activateResult = main.TRUE
appCheck = main.TRUE
+ getDataResult = main.TRUE
main.log.info( "Activating reactive forwarding app " )
activateResult = main.CLIs[ 0 ].activateApp( "org.onosproject.fwd" )
for i in range( main.numCtrls ):
appCheck = appCheck and main.CLIs[ i ].appToIDCheck()
-
- if appCheck != main.TRUE:
- main.log.warn( main.CLIs[ 0 ].apps() )
- main.log.warn( main.CLIs[ 0 ].appIDs() )
+ if appCheck != main.TRUE:
+ main.log.warn( main.CLIs[ i ].apps() )
+ main.log.warn( main.CLIs[ i ].appIDs() )
pingResult = main.Mininet1.pingall()
hostsJson = json.loads( main.CLIs[ 0 ].hosts() )
@@ -497,9 +884,17 @@
main.hostsData[ host ][ 'ipAddresses' ] = hostj[ 'ipAddresses' ]
main.log.info( "Deactivating reactive forwarding app " )
- activateResult = main.CLIs[ 0 ].deactivateApp( "org.onosproject.fwd" )
+ deactivateResult = main.CLIs[ 0 ].deactivateApp( "org.onosproject.fwd" )
+ if activateResult and deactivateResult and main.hostsData:
+ main.log.info( "Successfully used fwd app to discover hosts " )
+ getDataResult = main.TRUE
+ else:
+ main.log.info( "Failed to use fwd app to discover hosts " )
+ getDataResult = main.FALSE
+
print main.hostsData
- return pingResult
+
+ return getDataResult
def checkTopology( main, expectedLink ):
statusResult = main.TRUE
@@ -513,21 +908,20 @@
expectedLink )\
and statusResult
if not statusResult:
- main.log.info( itemName + ": Topology mismatch" )
+ main.log.error( itemName + ": Topology mismatch" )
else:
main.log.info( itemName + ": Topology match" )
return statusResult
def checkIntentState( main, intentsId ):
- main.log.info( itemName + ": Check host intents state" )
+ intentResult = main.TRUE
+
+ main.log.info( itemName + ": Checking intents state" )
for i in range( main.numCtrls ):
- intentResult = main.CLIs[ i ].checkIntentState( intentsId=intentsId )
- if not intentResult:
- main.log.info( itemName + ": Check host intents state again")
- for i in range( main.numCtrls ):
- intentResult = main.CLIs[ i ].checkIntentState(
- intentsId=intentsId )
+ intentResult = intentResult and \
+ main.CLIs[ i ].checkIntentState( intentsId=intentsId )
+
return intentResult
def checkFlowsState( main ):
@@ -536,8 +930,38 @@
checkFlowsResult = main.CLIs[ 0 ].checkFlowsState()
return checkFlowsResult
-def printMsg( main, h1Name, h2Name ):
- main.log.info("PINGING HOST INSIDE printMSG")
- pingHost( main, itemName, h1Name, h2Name )
- print 'lala'
+def link( main, sw1, sw2, option):
+ # link down
+ main.log.info( itemName + ": Bring link " + option + "between " +
+ sw1 + " and " + sw2 )
+ linkResult = main.Mininet1.link( end1=sw1, end2=sw2, option=option )
+ return linkResult
+
+def removeAllIntents( main, intentsId ):
+ """
+ Remove all intents in the intentsId
+ """
+ import time
+ intentsRemaining = []
+ removeIntentResult = main.TRUE
+ # Remove intents
+ for intent in intentsId:
+ main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
+
+ time.sleep( 5 )
+ # Checks if there is remaining intents using intents()
+ intentsRemaining = main.CLIs[ 0 ].intents( jsonFormat=False )
+ # If there is remianing intents then remove intents should fail
+
+ if intentsRemaining:
+ main.log.info( itemName + ": There are " +
+ str( len( intentsRemaining ) ) + " intents remaining, "
+ + "failed to remove all the intents " )
+ removeIntentResult = main.FALSE
+ main.log.info( intentsRemaining )
+ else:
+ main.log.info( itemName + ": There are no intents remaining, " +
+ "successfully removed all the intents." )
+ removeIntentResult = main.TRUE
+ return removeIntentResult
diff --git a/TestON/tests/FuncIntent/FuncIntent.params b/TestON/tests/FuncIntent/FuncIntent.params
deleted file mode 100755
index 6a346bc..0000000
--- a/TestON/tests/FuncIntent/FuncIntent.params
+++ /dev/null
@@ -1,34 +0,0 @@
-<PARAMS>
-
- <testcases>10,11,12,13,1001,1002</testcases>
-
- <SCALE>1,3</SCALE>
- <availableNodes>3</availableNodes>
- <ENV>
- <cellName>functionality</cellName>
- <cellApps>drivers,openflow,proxyarp,mobility</cellApps>
- </ENV>
- <GIT>
- <pull>False</pull>
- <branch>master</branch>
- </GIT>
- <CTRL>
- <num>3</num>
- <ip1>OC1</ip1>
- <port1>6633</port1>
- <ip2>OC2</ip2>
- <port2>6633</port2>
- <ip3>OC3</ip3>
- <port3>6633</port3>
- </CTRL>
- <BENCH>
- <user>admin</user>
- <ip1>OCN</ip1>
- </BENCH>
- <MININET>
- <switch>7</switch>
- <links>20</links>
- <topo>~/mininet/custom/newFuncTopo.py</topo>
- </MININET>
-
-</PARAMS>
diff --git a/TestON/tests/FuncIntent/FuncIntent.py b/TestON/tests/FuncIntent/FuncIntent.py
index d3583e6..d5dc2a7 100644
--- a/TestON/tests/FuncIntent/FuncIntent.py
+++ b/TestON/tests/FuncIntent/FuncIntent.py
@@ -5,8 +5,6 @@
import time
import json
-time.sleep( 1 )
-
class FuncIntent:
def __init__( self ):
@@ -253,7 +251,7 @@
stepResult = main.TRUE
main.step( "Discover all hosts using pingall " )
stepResult = main.wrapper.getHostsData( main )
- utilities.assert_equals( expect=main.FALSE,
+ utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="Successfully discovered hosts",
onfail="Failed to discover hosts" )
@@ -280,15 +278,6 @@
import time
import json
import re
- """
- Create your item(s) here
- item = { 'name': '', 'host1':
- { 'name': '', 'MAC': '00:00:00:00:00:0X',
- 'id':'00:00:00:00:00:0X/-X' } , 'host2':
- { 'name': '', 'MAC': '00:00:00:00:00:0X',
- 'id':'00:00:00:00:00:0X/-X'}, 'link': { 'switch1': '',
- 'switch2': '', 'expect':'' } }
- """
# Assert variables - These variable's name|format must be followed
# if you want to use the wrapper function
@@ -318,7 +307,7 @@
onfail="IPV4: Add host intent failed" )
stepResult = main.TRUE
- main.step( "DUALSTACK: Add host intents between h3 and h11" )
+ main.step( "DUALSTACK1: Add host intents between h3 and h11" )
stepResult = main.wrapper.hostIntent( main,
name='DUALSTACK',
host1='h3',
@@ -331,8 +320,25 @@
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
- onpass="DUALSTACK: Add host intent successful",
- onfail="DUALSTACK: Add host intent failed" )
+ onpass="DUALSTACK1: Add host intent" +
+ " successful",
+ onfail="DUALSTACK1: Add host intent failed" )
+
+ stepResult = main.TRUE
+ main.step( "DUALSTACK2: Add host intents between h1 and h9" )
+ stepResult = main.wrapper.hostIntent( main,
+ name='DUALSTACK2',
+ host1='h1',
+ host2='h11',
+ sw1='s5',
+ sw2='s2',
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="DUALSTACK2: Add host intent" +
+ " successful",
+ onfail="DUALSTACK2: Add host intent failed" )
def CASE1002( self, main ):
"""
@@ -356,16 +362,6 @@
import time
import json
import re
- """
- Create your item(s) here
- item = { 'name':'', 'host1': { 'name': '' },
- 'host2': { 'name': '' },
- 'ingressDevice':'' , 'egressDevice':'',
- 'ingressPort':'', 'egressPort':'',
- 'option':{ 'ethType':'', 'ethSrc':'', 'ethDst':'' } ,
- 'link': { 'switch1': '', 'switch2':'', 'expect':'' } }
-
- """
# Assert variables - These variable's name|format must be followed
# if you want to use the wrapper function
@@ -375,27 +371,6 @@
assert main.numSwitch, "Placed the total number of switch topology in \
main.numSwitch"
- ipv4 = { 'name':'IPV4', 'ingressDevice':'of:0000000000000005/1' ,
- 'host1': { 'name': 'h1' }, 'host2': { 'name': 'h9' },
- 'egressDevice':'of:0000000000000006/1', 'option':
- { 'ethType':'IPV4', 'ethSrc':'00:00:00:00:00:01',
- 'ethDst':'00:00:00:00:00:09' }, 'link': { 'switch1':'s5',
- 'switch2':'s2', 'expect':'18' } }
-
- """
- ipv4 = { 'name':'IPV4', 'ingressDevice':'of:0000000000000005/1' ,
- 'host1': { 'name': 'h1' }, 'host2': { 'name': 'h9' },
- 'egressDevice':'of:0000000000000006/1', 'option':
- { 'ethType':'IPV4', 'ethSrc':'00:00:00:00:00:01' },
- 'link': { 'switch1':'s5', 'switch2':'s2', 'expect':'18' } }
- """
- dualStack1 = { 'name':'IPV4', 'ingressDevice':'0000000000000005/3' ,
- 'host1': { 'name': 'h3' }, 'host2': { 'name': 'h11' },
- 'egressDevice':'0000000000000006/3', 'option':
- { 'ethType':'IPV4', 'ethSrc':'00:00:00:00:00:03',
- 'ethDst':'00:00:00:00:00:0B' }, 'link': { 'switch1':'s5',
- 'switch2':'s2', 'expect':'18' } }
-
main.case( "Add point intents between 2 devices" )
stepResult = main.TRUE
@@ -428,6 +403,37 @@
onpass="IPV4: Add point intent successful",
onfail="IPV4: Add point intent failed" )
+ stepResult = main.TRUE
+ main.step( "DUALSTACK1: Add point intents between h1 and h9" )
+ stepResult = main.wrapper.pointIntent(
+ main,
+ name="DUALSTACK1",
+ host1="h3",
+ host2="h11",
+ deviceId1="of:0000000000000005",
+ deviceId2="of:0000000000000006",
+ port1="3",
+ port2="3",
+ ethType="IPV4",
+ mac1="00:00:00:00:00:03",
+ mac2="00:00:00:00:00:0B",
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ip1="",
+ ip2="",
+ tcp1="",
+ tcp2="",
+ sw1="s5",
+ sw2="s2",
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="DUALSTACK1: Add point intent" +
+ " successful",
+ onfail="DUALSTACK1: Add point intent failed" )
+
def CASE1003( self, main ):
"""
Add single point to multi point intents
@@ -447,7 +453,59 @@
- Ping hosts
- Remove intents
"""
+ assert main, "There is no main"
+ assert main.CLIs, "There is no main.CLIs"
+ assert main.Mininet1, "Mininet handle should be named Mininet1"
+ assert main.numSwitch, "Placed the total number of switch topology in \
+ main.numSwitch"
+ main.case( "Add single point to multi point intents between devices" )
+
+ stepResult = main.TRUE
+ main.step( "IPV4: Add single point to multi point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ devices = [ 'of:0000000000000005/8', 'of:0000000000000006/8', \
+ 'of:0000000000000007/8' ]
+ macs = [ '00:00:00:00:00:08', '00:00:00:00:00:10', '00:00:00:00:00:18' ]
+ stepResult = main.wrapper.singleToMultiIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ devices=devices,
+ ports=None,
+ ethType="IPV4",
+ macs=macs,
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ipAddresses="",
+ tcp="",
+ sw1="s5",
+ sw2="s2",
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4: Successfully added single point"
+ + " to multi point intents",
+ onfail="IPV4: Failed to add single point" +
+ " to multi point intents" )
+
+ main.step( "IPV4_2: Add single point to multi point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ stepResult = main.wrapper.singleToMultiIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ ethType="IPV4",
+ lambdaAlloc=False )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4_2: Successfully added single point"
+ + " to multi point intents",
+ onfail="IPV4_2: Failed to add single point" +
+ " to multi point intents" )
def CASE1004( self, main ):
"""
Add multi point to single point intents
@@ -467,3 +525,56 @@
- Ping hosts
- Remove intents
"""
+ assert main, "There is no main"
+ assert main.CLIs, "There is no main.CLIs"
+ assert main.Mininet1, "Mininet handle should be named Mininet1"
+ assert main.numSwitch, "Placed the total number of switch topology in \
+ main.numSwitch"
+
+ main.case( "Add multi point to single point intents between devices" )
+
+ stepResult = main.TRUE
+ main.step( "IPV4: Add multi point to single point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ devices = [ 'of:0000000000000005/8', 'of:0000000000000006/8', \
+ 'of:0000000000000007/8' ]
+ macs = [ '00:00:00:00:00:08', '00:00:00:00:00:10', '00:00:00:00:00:18' ]
+ stepResult = main.wrapper.multiToSingleIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ devices=devices,
+ ports=None,
+ ethType="IPV4",
+ macs=macs,
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ipAddresses="",
+ tcp="",
+ sw1="s5",
+ sw2="s2",
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4: Successfully added multi point"
+ + " to single point intents",
+ onfail="IPV4: Failed to add multi point" +
+ " to single point intents" )
+
+ main.step( "IPV4_2: Add multi point to single point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ stepResult = main.wrapper.multiToSingleIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ ethType="IPV4",
+ lambdaAlloc=False )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4_2: Successfully added multi point"
+ + " to single point intents",
+ onfail="IPV4_2: Failed to add multi point" +
+ " to single point intents" )
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 2499625..bb69fcc 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -49,6 +49,9 @@
main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -227,10 +231,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults )
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -253,8 +255,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -388,8 +394,6 @@
"""
Assign intents
"""
- # FIXME: we must reinstall intents until we have a persistant
- # datastore!
import time
import json
assert numControllers, "numControllers not defined"
@@ -397,11 +401,14 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
+ # NOTE: we must reinstall intents until we have a persistant intent
+ # datastore!
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -410,6 +417,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -429,6 +437,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -451,8 +461,10 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
- main.step( "Check app ids check" )
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -462,15 +474,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -513,7 +525,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -608,13 +623,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -747,9 +760,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -783,83 +798,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -942,6 +976,37 @@
main.log.error( repr( pendingMap ) )
main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -956,7 +1021,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -1378,7 +1442,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1489,6 +1553,7 @@
"""
The Failure case.
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -1505,6 +1570,7 @@
main.case( "Restart entire ONOS cluster" )
main.step( "Killing ONOS nodes" )
killResults = main.TRUE
+ killTime = time.time()
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
killResults = killResults and killed
@@ -1542,6 +1608,29 @@
utilities.assert_equals( expect=main.TRUE, actual=cliResults,
onpass="ONOS cli started",
onfail="ONOS clis did not restart" )
+
+ # Grab the time of restart so we chan check how long the gossip
+ # protocol has had time to work
+ main.restartTime = time.time() - killTime
+ main.log.debug( "Restart time: " + str( main.restartTime ) )
+ '''
+ # FIXME: revisit test plan for election with madan
+ # Rerun for election on restarted nodes
+ run1 = CLIs[0].electionTestRun()
+ run2 = CLIs[1].electionTestRun()
+ run3 = CLIs[2].electionTestRun()
+ ...
+ ...
+ runResults = run1 and run2 and run3
+ utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ onpass="Reran for election",
+ onfail="Failed to rerun for election" )
+ '''
+ # TODO: Make this configurable
+ time.sleep( 60 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1630,6 +1719,7 @@
elif rolesResults and not consistentMastership:
mastershipCheck = main.TRUE
+ '''
description2 = "Compare switch roles from before failure"
main.step( description2 )
try:
@@ -1655,15 +1745,13 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed" )
+ '''
# NOTE: we expect mastership to change on controller failure
- mastershipCheck = mastershipCheck and consistentMastership
main.step( "Get the intents and compare across all nodes" )
ONOSIntents = []
@@ -1769,7 +1857,7 @@
sameIntents = main.TRUE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
else:
@@ -1804,8 +1892,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1849,7 +1935,7 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
@@ -1864,10 +1950,6 @@
main.log.error(
"Inconsistent view of leader for the election test app" )
# TODO: print the list
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1891,16 +1973,26 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
+ main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
portsResults = main.TRUE
linksResults = main.TRUE
@@ -1949,9 +2041,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2142,8 +2234,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
@@ -2420,6 +2510,15 @@
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ timerLog.write( "Restart\n" )
+ timerLog.write( str( main.restartTime ) )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
+
def CASE14( self, main ):
"""
start election app on all onos nodes
@@ -2430,10 +2529,17 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
leaders = []
for cli in CLIs:
cli.electionTestRun()
@@ -2445,19 +2551,23 @@
str( leader ) + "'" )
leaderResult = main.FALSE
leaders.append( leader )
- if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
- main.log.error( "Results of electionTestLeader is order of CLIs:" +
- str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
def CASE15( self, main ):
"""
@@ -2474,12 +2584,44 @@
description = "Check that Leadership Election is still functional"
main.log.report( description )
main.case( description )
+ # NOTE: Need to re-run since being a canidate is not persistant
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ cli.electionTestRun()
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ if leader is None or leader == main.FALSE:
+ main.log.report( cli.name + ": Leader for the election app " +
+ "should be an ONOS node, instead got '" +
+ str( leader ) + "'" )
+ leaderResult = main.FALSE
+ leaders.append( leader )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=leaderResult,
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2495,8 +2637,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2535,11 +2677,6 @@
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2558,6 +2695,7 @@
onpass="App re-ran for election",
onfail="App failed to run for election" )
+ main.step( "Leader did not change when old leader re-ran" )
afterRun = main.ONOScli1.electionTestLeader()
# verify leader didn't just change
if afterRun == leaderList[ 0 ]:
@@ -2572,14 +2710,6 @@
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
- case15Result = withdrawResult and leaderResult and runResult and\
- afterResult
- utilities.assert_equals(
- expect=main.TRUE,
- actual=case15Result,
- onpass="Leadership election is still functional",
- onfail="Leadership Election is no longer functional" )
-
def CASE16( self, main ):
"""
Install Distributed Primitives app
@@ -2613,6 +2743,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
+ time.sleep (5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2656,11 +2787,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -2669,8 +2802,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -2679,6 +2816,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -2686,6 +2824,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -2694,8 +2833,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 0dd40ed..256561a 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -49,6 +49,9 @@
main.log.report( "ONOS HA test: Restart minority of ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -227,10 +231,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults )
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -253,8 +255,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -286,8 +292,6 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -395,11 +399,12 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -408,6 +413,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -427,6 +433,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -449,8 +457,10 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
- main.step( "Check app ids check" )
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -460,15 +470,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -511,7 +521,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -606,13 +619,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -745,9 +756,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -781,83 +794,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -940,6 +972,37 @@
main.log.error( repr( pendingMap ) )
main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -954,7 +1017,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -1376,7 +1438,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1495,6 +1557,7 @@
assert nodes, "nodes not defined"
main.case( "Restart minority of ONOS nodes" )
main.step( "Killing 3 ONOS nodes" )
+ killTime = time.time()
# TODO: Randomize these nodes or base this on partitions
# TODO: use threads in this case
killResults = main.ONOSbench.onosKill( nodes[0].ip_address )
@@ -1533,7 +1596,28 @@
# Grab the time of restart so we chan check how long the gossip
# protocol has had time to work
- main.restartTime = time.time()
+ main.restartTime = time.time() - killTime
+ main.log.debug( "Restart time: " + str( main.restartTime ) )
+ '''
+ # FIXME: revisit test plan for election with madan
+ # Rerun for election on restarted nodes
+ run1 = CLIs[0].electionTestRun()
+ run2 = CLIs[1].electionTestRun()
+ run3 = CLIs[2].electionTestRun()
+ runResults = run1 and run2 and run3
+ utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ onpass="Reran for election",
+ onfail="Failed to rerun for election" )
+ '''
+ # TODO: MAke this configurable. Also, we are breaking the above timer
+ time.sleep( 60 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
+ time.sleep( 200 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1569,7 +1653,6 @@
main.step( "Read device roles from ONOS" )
ONOSMastership = []
- mastershipCheck = main.FALSE
consistentMastership = True
rolesResults = True
threads = []
@@ -1619,9 +1702,9 @@
sort_keys=True,
indent=4,
separators=( ',', ': ' ) ) )
- elif rolesResults and not consistentMastership:
- mastershipCheck = main.TRUE
+ # NOTE: we expect mastership to change on controller failure
+ '''
description2 = "Compare switch roles from before failure"
main.step( description2 )
try:
@@ -1647,15 +1730,12 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed" )
- # NOTE: we expect mastership to change on controller failure
- mastershipCheck = consistentMastership
+ '''
main.step( "Get the intents and compare across all nodes" )
ONOSIntents = []
@@ -1760,7 +1840,7 @@
sameIntents = main.TRUE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
else:
@@ -1795,8 +1875,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1841,7 +1919,7 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
@@ -1861,10 +1939,6 @@
main.log.error(
"Inconsistent view of leader for the election test app" )
# TODO: print the list
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1888,16 +1962,26 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
+ main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
portsResults = main.TRUE
linksResults = main.TRUE
@@ -1946,7 +2030,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -2139,8 +2223,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
@@ -2417,6 +2499,15 @@
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ timerLog.write( "Restart\n" )
+ timerLog.write( str( main.restartTime ) )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
+
def CASE14( self, main ):
"""
start election app on all onos nodes
@@ -2427,10 +2518,17 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
leaders = []
for cli in CLIs:
cli.electionTestRun()
@@ -2442,19 +2540,23 @@
str( leader ) + "'" )
leaderResult = main.FALSE
leaders.append( leader )
- if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
- main.log.error( "Results of electionTestLeader is order of CLIs:" +
- str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
def CASE15( self, main ):
"""
@@ -2471,12 +2573,29 @@
description = "Check that Leadership Election is still functional"
main.log.report( description )
main.case( description )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ leaders.append( leader )
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2492,8 +2611,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2532,11 +2651,6 @@
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2555,6 +2669,7 @@
onpass="App re-ran for election",
onfail="App failed to run for election" )
+ main.step( "Leader did not change when old leader re-ran" )
afterRun = main.ONOScli1.electionTestLeader()
# verify leader didn't just change
if afterRun == leaderList[ 0 ]:
@@ -2569,14 +2684,6 @@
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
- case15Result = withdrawResult and leaderResult and runResult and\
- afterResult
- utilities.assert_equals(
- expect=main.TRUE,
- actual=case15Result,
- onpass="Leadership election is still functional",
- onfail="Leadership Election is no longer functional" )
-
def CASE16( self, main ):
"""
Install Distributed Primitives app
@@ -2610,6 +2717,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
+ time.sleep (5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2653,11 +2761,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -2666,8 +2776,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -2676,6 +2790,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -2683,6 +2798,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -2691,8 +2807,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
index 3c4541c..e4658e7 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
@@ -120,12 +120,27 @@
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
- main.step( "Using mvn clean & install" )
+ main.step( "Using mvn clean and install" )
cleanInstallResult = main.ONOSbench.cleanInstall()
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+ # GRAPHS
+ # NOTE: important params here:
+ # job = name of Jenkins job
+ # Plot Name = Plot-HA, only can be used if multiple plots
+ # index = The number of the graph under plot name
+ job = "HANetworkPartition"
+ graphs = '<ac:structured-macro ac:name="html">\n'
+ graphs += '<ac:plain-text-body><![CDATA[\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+ '/plot/getPlot?index=0&width=500&height=300"' +\
+ 'noborder="0" width="500" height="300" scrolling="yes" '+\
+ 'seamless="seamless"></iframe>\n'
+ graphs += ']]></ac:plain-text-body>\n'
+ graphs += '</ac:structured-macro>\n'
+ main.log.wiki(graphs)
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
@@ -218,6 +233,7 @@
onfail="Test startup NOT successful" )
if case1Result == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -1414,7 +1430,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host[ 'ips' ] == []:
+ if host[ 'ipAddresses' ] == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index dace37a..2440ec6 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -17,7 +17,6 @@
#CASE15: Check that Leadership Election is still functional
#CASE16: Install Distributed Primitives app
#CASE17: Check for basic functionality with distributed primitives
- #1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13
#1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13
<testcases>1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 8bc21b0..fe513ac 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -49,6 +49,9 @@
"""
main.log.report( "ONOS HA Sanity test - initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -229,10 +233,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults )
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -255,8 +257,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -397,11 +403,12 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -410,6 +417,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -429,6 +437,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -451,8 +461,15 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
- main.step( "Check app ids check" )
+ '''
+ main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
+ import time
+ time.sleep(60)
+ '''
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -462,15 +479,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -513,7 +530,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -608,13 +628,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -747,9 +765,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -783,83 +803,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -942,6 +981,46 @@
main.log.error( repr( pendingMap ) )
main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
+
+
+ '''
+ #DEBUG
+ if PingResult == main.FALSE:
+ import time
+ time.sleep( 100000 )
+ '''
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -956,7 +1035,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -1378,7 +1456,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1615,8 +1693,6 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -1727,7 +1803,7 @@
sameIntents = main.TRUE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
else:
@@ -1762,8 +1838,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1821,10 +1895,6 @@
main.log.report( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1848,16 +1918,26 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
+ main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
portsResults = main.TRUE
linksResults = main.TRUE
@@ -1906,7 +1986,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -2099,8 +2179,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
@@ -2378,11 +2456,11 @@
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
try:
- gossipIntentLog = open( main.logdir + "/Timers.csv", 'w')
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- gossipIntentLog.write( "Gossip Intents\n" )
- gossipIntentLog.write( str( gossipTime ) )
- gossipIntentLog.close()
+ timerLog.write( "Gossip Intents\n" )
+ timerLog.write( str( gossipTime ) )
+ timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2396,10 +2474,17 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
leaders = []
for cli in CLIs:
cli.electionTestRun()
@@ -2411,19 +2496,23 @@
str( leader ) + "'" )
leaderResult = main.FALSE
leaders.append( leader )
- if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
- main.log.error( "Results of electionTestLeader is order of CLIs:" +
- str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
def CASE15( self, main ):
"""
@@ -2440,12 +2529,29 @@
description = "Check that Leadership Election is still functional"
main.log.report( description )
main.case( description )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ leaders.append( leader )
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2461,8 +2567,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2501,11 +2607,6 @@
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2524,6 +2625,7 @@
onpass="App re-ran for election",
onfail="App failed to run for election" )
+ main.step( "Leader did not change when old leader re-ran" )
afterRun = main.ONOScli1.electionTestLeader()
# verify leader didn't just change
if afterRun == leaderList[ 0 ]:
@@ -2538,14 +2640,6 @@
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
- case15Result = withdrawResult and leaderResult and runResult and\
- afterResult
- utilities.assert_equals(
- expect=main.TRUE,
- actual=case15Result,
- onpass="Leadership election is still functional",
- onfail="Leadership Election is no longer functional" )
-
def CASE16( self, main ):
"""
Install Distributed Primitives app
@@ -2579,6 +2673,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
+ time.sleep (5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2622,11 +2717,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -2635,8 +2732,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -2645,6 +2746,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -2652,6 +2754,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -2660,8 +2763,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
index d027c36..f057592 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
@@ -16,7 +16,7 @@
#CASE15: Check that Leadership Election is still functional
#1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13
#extra hosts test 1,2,8,11,8,12,8
- <testcases>1,2,8,3,4,5,14,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,15,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index 0f68a02..7194a3b 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -49,6 +49,9 @@
main.log.report( "ONOS Single node cluster restart " +
"HA test - initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -92,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -171,8 +175,6 @@
onos1Isup = main.ONOSbench.isup( ONOS1Ip )
if onos1Isup:
break
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
onpass="ONOS startup successful",
onfail="ONOS startup failed" )
@@ -191,18 +193,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
- appCheck = main.TRUE
- threads = []
- for i in range( numControllers ):
- t = main.Thread( target=CLIs[i].appToIDCheck,
- name="appToIDCheck-" + str( i ),
- args=[] )
- threads.append( t )
- t.start()
-
- for t in threads:
- t.join()
- appCheck = appCheck and t.result
+ appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
@@ -210,11 +201,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onos1Isup and cliResults )
-
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -234,8 +222,10 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device."
main.step( "Assign switches to controllers" )
for i in range( 1, 29 ):
@@ -255,7 +245,7 @@
else:
mastershipCheck = main.FALSE
if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
+ main.log.info( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -340,13 +330,14 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- # FIXME: we must reinstall intents until we have a persistant
- # datastore!
- main.log.report( "Adding host intents" )
+ # NOTE: we must reinstall intents until we have a persistant intent
+ # datastore!
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -355,6 +346,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
@@ -363,6 +355,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -385,6 +379,8 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
+
+ main.step( "Check app ids" )
appCheck2 = main.ONOScli1.appToIDCheck()
if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
@@ -393,7 +389,7 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -435,7 +431,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -530,13 +529,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -667,9 +664,11 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -695,7 +694,7 @@
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
if PingResult == main.TRUE:
- main.log.report(
+ main.log.info(
"Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
@@ -703,83 +702,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -862,6 +880,37 @@
main.log.error( repr( pendingMap ) )
main.log.debug( main.ONOScli1.flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -873,7 +922,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -961,7 +1009,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -1074,7 +1122,9 @@
iCounterValue = 0
main.log.report( "Restart ONOS node" )
- main.log.case( "Restart ONOS node" )
+ main.case( "Restart ONOS node" )
+ main.caseExplaination = "Killing ONOS process and restart cli " +\
+ "sessions once onos is up."
main.step( "Killing ONOS processes" )
killResult = main.ONOSbench.onosKill( ONOS1Ip )
start = time.time()
@@ -1105,6 +1155,8 @@
main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
str( elapsed ) )
time.sleep( 5 )
+ # rerun on election apps
+ main.ONOScli1.electionTestRun()
def CASE7( self, main ):
"""
@@ -1115,7 +1167,6 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
main.case( "Running ONOS Constant State Tests" )
-
main.step( "Check that each switch has a master" )
# Assert that each device has a master
rolesNotNull = main.ONOScli1.rolesNotNull()
@@ -1162,8 +1213,6 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -1207,9 +1256,9 @@
sameIntents = main.TRUE
if intentState and intentState == ONOS1Intents:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
- # what the aceptable states are
+ # what the acceptable states are
else:
try:
main.log.warn( "ONOS1 intents: " )
@@ -1241,14 +1290,13 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
+ main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leader = ONOS1Ip
@@ -1264,7 +1312,7 @@
# all is well
pass
elif leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
@@ -1275,25 +1323,12 @@
str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
onpass="Leadership election passed",
onfail="Something went wrong with Leadership election" )
- result = ( mastershipCheck and intentCheck and FlowTables and
- rolesNotNull and leaderResult )
- result = int( result )
- if result == main.TRUE:
- main.log.report( "Constant State Tests Passed" )
- utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Constant State Tests Passed",
- onfail="Constant state tests failed" )
-
def CASE8( self, main ):
"""
Compare topo
@@ -1309,15 +1344,24 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- node = main.ONOS1
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ node = main.ONOS1
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1344,7 +1388,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -1445,8 +1489,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
def CASE9( self, main ):
"""
@@ -1662,10 +1704,17 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.ONOScli1.electionTestRun()
# check for leader
leader = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
@@ -1674,32 +1723,27 @@
pass
elif leader is None:
# No leader elected
- main.log.report( "No leader was elected" )
+ main.log.error( "No leader was elected" )
leaderResult = main.FALSE
elif leader == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader" +
+ main.log.error( "Something is wrong with electionTestLeader" +
" function, check the error logs" )
leaderResult = main.FALSE
else:
# error in response
- main.log.report(
+ main.log.error(
"Unexpected response from electionTestLeader function:'" +
str( leader ) +
"'" )
leaderResult = main.FALSE
-
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
def CASE15( self, main ):
"""
@@ -1726,13 +1770,15 @@
oldLeader = None
else:
main.log.error( "Leader election --- why am I HERE?!?")
+ leaderResult = main.FALSE
+ oldLeader = None
if oldLeader:
withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
leaderN = main.ONOScli1.electionTestLeader()
@@ -1750,9 +1796,7 @@
elif leaderN is None:
main.log.info(
"There is no leader after the app withdrew from election" )
- if leaderResult:
- main.log.report( "Leadership election tests passed( There is no " +
- "leader after the old leader resigned )" )
+ leaderResult = main.TRUE
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1770,20 +1814,21 @@
actual=runResult,
onpass="App re-ran for election",
onfail="App failed to run for election" )
- leader = main.ONOScli1.electionTestLeader()
+
+ main.step( "Node became leader when it ran for election" )
+ afterRun = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
- if leader == ONOS1Ip:
- leaderResult = main.TRUE
+ if afterRun == ONOS1Ip:
+ afterResult = main.TRUE
else:
- leaderResult = main.FALSE
- # TODO: assert on run and withdraw results?
+ afterResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
- onfail="ONOS1's election app was not leader after it re-ran " +
- "for election" )
+ actual=afterResult,
+ onpass="Old leader successfully re-ran for election",
+ onfail="Something went wrong with Leadership election after " +
+ "the old leader re-ran for election" )
def CASE16( self, main ):
"""
@@ -1861,11 +1906,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -1874,8 +1921,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -1884,6 +1935,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -1891,6 +1943,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -1899,8 +1952,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.params b/TestON/tests/SdnIpTest/SdnIpTest.params
index 7f7ef2d..e47adf6 100755
--- a/TestON/tests/SdnIpTest/SdnIpTest.params
+++ b/TestON/tests/SdnIpTest/SdnIpTest.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>4</testcases>
+ <testcases>100, [4]*5</testcases>
#Environment variables
<ENV>
@@ -8,13 +8,15 @@
</ENV>
<CTRL>
+ <numCtrl>1</numCtrl>
<ip1>10.128.4.52</ip1>
<port1>6633</port1>
</CTRL>
<GIT>
- <autoPull>off</autoPull>
- <checkout>master</checkout>
+ <autoPull>on</autoPull>
+ <branch1>master</branch1>
+ <branch2>onos-1.0</branch2>
</GIT>
<JSON>
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.py b/TestON/tests/SdnIpTest/SdnIpTest.py
index 85ac21e..ebbd57e 100644
--- a/TestON/tests/SdnIpTest/SdnIpTest.py
+++ b/TestON/tests/SdnIpTest/SdnIpTest.py
@@ -1,22 +1,86 @@
-# from cupshelpers.config import prefix
-# Testing the basic functionality of SDN-IP
-
-
+# Testing the functionality of SDN-IP with single ONOS instance
class SdnIpTest:
def __init__( self ):
self.default = ''
+ global branchName
-# from cupshelpers.config import prefix
+ # This case is to setup ONOS
+ def CASE100( self, main ):
+ """
+ CASE100 is to compile ONOS and push it to the test machines
+ Startup sequence:
+ git pull
+ mvn clean install
+ onos-package
+ cell <name>
+ onos-verify-cell
+ onos-install -f
+ onos-wait-for-start
+ """
+ main.case( "Setting up test environment" )
-# Testing the basic functionality of SDN-IP
+ cellName = main.params[ 'ENV' ][ 'cellName' ]
+ ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
+ main.step( "Applying cell variable to environment" )
+ cellResult = main.ONOSbench.setCell( cellName )
+ verifyResult = main.ONOSbench.verifyCell()
-class SdnIpTest:
+ branchName = main.ONOSbench.getBranchName()
+ main.log.info( "ONOS is on branch: " + branchName )
- def __init__( self ):
- self.default = ''
+ main.log.report( "Uninstalling ONOS" )
+ main.ONOSbench.onosUninstall( ONOS1Ip )
+
+ cleanInstallResult = main.TRUE
+ gitPullResult = main.TRUE
+
+ main.step( "Git pull" )
+ gitPullResult = main.ONOSbench.gitPull()
+
+ main.step( "Using mvn clean & install" )
+ cleanInstallResult = main.TRUE
+# if gitPullResult == main.TRUE:
+# cleanInstallResult = main.ONOSbench.cleanInstall()
+# else:
+# main.log.warn( "Did not pull new code so skipping mvn " +
+# "clean install" )
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ main.ONOSbench.getVersion( report=True )
+
+ #cellResult = main.ONOSbench.setCell( cellName )
+ #verifyResult = main.ONOSbench.verifyCell()
+ main.step( "Creating ONOS package" )
+ packageResult = main.ONOSbench.onosPackage()
+
+ main.step( "Installing ONOS package" )
+ onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
+ node=ONOS1Ip )
+
+ main.step( "Checking if ONOS is up yet" )
+ for i in range( 2 ):
+ onos1Isup = main.ONOSbench.isup( ONOS1Ip )
+ if onos1Isup:
+ break
+ if not onos1Isup:
+ main.log.report( "ONOS1 didn't start!" )
+
+ cliResult = main.ONOScli.startOnosCli( ONOS1Ip )
+
+ case1Result = ( cleanInstallResult and packageResult and
+ cellResult and verifyResult and
+ onos1InstallResult and
+ onos1Isup and cliResult )
+
+ utilities.assert_equals( expect=main.TRUE, actual=case1Result,
+ onpass="ONOS startup successful",
+ onfail="ONOS startup NOT successful" )
+
+ if case1Result == main.FALSE:
+ main.cleanup()
+ main.exit()
def CASE4( self, main ):
"""
@@ -33,14 +97,12 @@
import time
import json
from operator import eq
- # from datetime import datetime
from time import localtime, strftime
- main.case("The test case is to help to setup the TestON environment \
- and test new drivers" )
- # SDNIPJSONFILEPATH = "../tests/SdnIpTest/sdnip.json"
+ main.case("This case is to testing the functionality of SDN-IP with \
+ single ONOS instance" )
SDNIPJSONFILEPATH = \
- "/home/admin/workspace/onos/tools/package/config/sdnip.json"
+ "/home/admin/ONOS/tools/package/config/sdnip.json"
# all expected routes for all BGP peers
allRoutesExpected = []
main.step( "Start to generate routes for all BGP peers" )
@@ -79,31 +141,6 @@
routeIntentsExpected = routeIntentsExpectedHost3 + \
routeIntentsExpectedHost4 + routeIntentsExpectedHost5
- cellName = main.params[ 'ENV' ][ 'cellName' ]
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- main.step( "Set cell for ONOS-cli environment" )
- main.ONOScli.setCell( cellName )
- verifyResult = main.ONOSbench.verifyCell()
-
- main.log.report( "Removing raft logs" )
- main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
- main.ONOSbench.onosUninstall( ONOS1Ip )
-
- main.step( "Installing ONOS package" )
- onos1InstallResult = main.ONOSbench.onosInstall(
- options="-f", node=ONOS1Ip )
-
- main.step( "Checking if ONOS is up yet" )
- time.sleep( 150 )
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
-
- main.step( "Start ONOS-cli" )
-
- main.ONOScli.startOnosCli( ONOS1Ip )
-
main.step( "Get devices in the network" )
listResult = main.ONOScli.devices( jsonFormat=False )
main.log.info( listResult )
@@ -138,8 +175,8 @@
prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
main.log.info( prefixesHostX )
for prefix in prefixesHostX:
- allRoutesExpected.append(
- prefix + "/" + "192.168.40." + str( i - 100 ) )
+ allRoutesExpected.append( prefix + "/" + "192.168.40."
+ + str( i - 100 ) )
routeIntentsExpectedHostX = \
main.QuaggaCliHost.generateExpectedOnePeerRouteIntents(
@@ -158,13 +195,19 @@
QuaggaCliHostX.addRoutes( prefixesHostX, 1 )
time.sleep( 60 )
-
# get routes inside SDN-IP
getRoutesResult = main.ONOScli.routes( jsonFormat=True )
# parse routes from ONOS CLI
- allRoutesActual = \
- main.QuaggaCliHost3.extractActualRoutes( getRoutesResult )
+ if branchName == "master":
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+ elif branchName == "onos-1.0":
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesOneDotZero( getRoutesResult )
+ else:
+ main.log("ONOS is on wrong branch")
+ exit
allRoutesStrExpected = str( sorted( allRoutesExpected ) )
allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
@@ -189,7 +232,7 @@
main.step( "Check MultiPointToSinglePointIntent intents installed" )
# routeIntentsExpected are generated when generating routes
- # get rpoute intents from ONOS CLI
+ # get route intents from ONOS CLI
routeIntentsActual = \
main.QuaggaCliHost3.extractActualRouteIntents(
getIntentsResult )
@@ -243,7 +286,7 @@
"***PointToPointIntent Intents in SDN-IP are wrong!***" )
#============================= Ping Test ========================
- # wait until all MultiPointToSinglePoint
+ # Wait until all MultiPointToSinglePoint intents are in system
time.sleep( 20 )
pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
pingTestResultsFile = \
@@ -306,22 +349,19 @@
time.sleep( 20 )
pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
pingTestResultsFile = \
- "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-" \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-"\
+ strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
pingTestResults = main.QuaggaCliHost.pingTest(
"1.168.30.100", pingTestScript, pingTestResultsFile )
main.log.info( pingTestResults )
time.sleep( 100 )
- # main.step( "Test whether Mininet is started" )
- # main.Mininet2.handle.sendline( "xterm host1" )
- # main.Mininet2.handle.expect( "mininet>" )
-
def CASE3( self, main ):
"""
Test the SDN-IP functionality
allRoutesExpected: all expected routes for all BGP peers
- routeIntentsExpected: all expected MultiPointToSinglePointIntent intents
+ routeIntentsExpected: all expected MultiPointToSinglePointIntent \
+ intents
bgpIntentsExpected: expected PointToPointIntent intents
allRoutesActual: all routes from ONOS LCI
routeIntentsActual: actual MultiPointToSinglePointIntent intents from \
@@ -338,7 +378,7 @@
environment and test new drivers" )
# SDNIPJSONFILEPATH = "../tests/SdnIpTest/sdnip.json"
SDNIPJSONFILEPATH = \
- "/home/admin/workspace/onos/tools/package/config/sdnip.json"
+ "/home/admin/ONOS/tools/package/config/sdnip.json"
# all expected routes for all BGP peers
allRoutesExpected = []
main.step( "Start to generate routes for all BGP peers" )