Pulled commits from master before snapshot
diff --git a/TestON/core/Thread.py b/TestON/core/Thread.py
index da30339..fcc06b5 100644
--- a/TestON/core/Thread.py
+++ b/TestON/core/Thread.py
@@ -18,6 +18,8 @@
if self.target is not None:
self.result = self.target( *self.args, **self.kwargs )
except Exception as e:
- print "Thread-" + str( self.threadID ) + " '" + self.name + "'"\
- ":something went wrong with " + self.target + " method"
+ print "ThreadID:" + str( self.threadID ) + ", Name:" +\
+ self.name + "- something went wrong with " +\
+ str( self.target.im_class ) + "." +\
+ str( self.target.im_func ) + " method: "
print e
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 667f99c..271c0dd 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -32,6 +32,7 @@
import new
import xmldict
import importlib
+import os
module = new.module("test")
import openspeak
global path, drivers_path, core_path, tests_path,logs_path
@@ -78,8 +79,6 @@
self.testResult = "Summary"
self.stepName = ""
self.stepCache = ""
- # make this into two lists? one for step names, one for results?
- # this way, the case result could be a true AND of these results
self.EXPERIMENTAL_MODE = False
self.test_target = None
self.lastcommand = None
@@ -164,11 +163,19 @@
driverClass = getattr(driverModule, driverName)
driverObject = driverClass()
+ try:
+ self.componentDictionary[component]['host'] = os.environ[str( self.componentDictionary[component]['host'])]
+ except KeyError:
+ self.log.info("Missing OC environment variable! Using stored IPs")
+ except Exception as inst:
+ self.log.error("Uncaught exception: " + str(inst))
+
connect_result = driverObject.connect(user_name = self.componentDictionary[component]['user'] if ('user' in self.componentDictionary[component].keys()) else getpass.getuser(),
ip_address= self.componentDictionary[component]['host'] if ('host' in self.componentDictionary[component].keys()) else 'localhost',
pwd = self.componentDictionary[component]['password'] if ('password' in self.componentDictionary[component].keys()) else 'changeme',
port = self.componentDictionary[component]['port'] if ('port' in self.componentDictionary[component].keys()) else None,
options = driver_options)
+
if not connect_result:
self.log.error("Exiting form the test execution because the connecting to the "+component+" component failed.")
self.exit()
@@ -251,13 +258,14 @@
self.log.summary( self.caseExplaination )
self.log.wiki( "<ul>" )
for line in self.stepCache.splitlines():
- self.log.error( repr(line) )
if re.search( " - PASS$", line ):
self.log.wiki( "<li>" + line + " <ac:emoticon ac:name=\"tick\" /></li>\n" )
elif re.search( " - FAIL$", line ):
self.log.wiki( "<li>" + line + " <ac:emoticon ac:name=\"cross\" /></li>\n" )
elif re.search( " - No Result$", line ):
self.log.wiki( "<li>" + line + " <ac:emoticon ac:name=\"warning\" /></li>\n" )
+ else: # Should only be on fail message
+ self.log.wiki( "<ul><li>" + line + "</li></ul>\n" )
self.log.wiki( "</ul>" )
self.log.summary( self.stepCache )
self.stepCache = ""
@@ -268,38 +276,38 @@
try :
step = stepList[self.stepCount]
self.STEPRESULT = self.NORESULT
+ self.onFailMsg = "\t\tNo on fail message given"
exec code[testCaseNumber][step] in module.__dict__
self.stepCount = self.stepCount + 1
if step > 0:
self.stepCache += "\t"+str(testCaseNumber)+"."+str(step)+" "+self.stepName+" - "
if self.STEPRESULT == self.TRUE:
self.stepCache += "PASS\n"
- #self.stepCache += "PASS <ac:emoticon ac:name=\"tick\" /></li>\n"
elif self.STEPRESULT == self.FALSE:
self.stepCache += "FAIL\n"
- #self.stepCache += "FAIL <ac:emoticon ac:name=\"cross\" /></li>\n"
+ # TODO: Print the on-fail statement here
+ self.stepCache += "\t\t" + self.onFailMsg + "\n"
else:
self.stepCache += "No Result\n"
- #self.stepCache += "No Result <ac:emoticon ac:name=\"warning\" /></li>\n"
self.stepResults.append(self.STEPRESULT)
- except TypeError,e:
- print "\nException in the following section of code: " +\
- str(testCaseNumber) + "." + str(step) + ": " +\
- self.stepName
+ except StandardError as e:
+ self.log.exception( "\nException in the following section of" +
+ " code: " + str(testCaseNumber) + "." +
+ str(step) + ": " + self.stepName )
#print code[testCaseNumber][step]
self.stepCount = self.stepCount + 1
- self.log.exception(e)
self.logger.updateCaseResults(self)
#WIKI results
self.log.wiki( "<ul>" )
- for line in self.stepCachesplitlines():
- self.log.error( repr(line) )
+ for line in self.stepCache.splitlines():
if re.search( " - PASS$", line ):
self.log.wiki( "<li>" + line + " <ac:emoticon ac:name=\"tick\" /></li>\n" )
elif re.search( " - FAIL$", line ):
self.log.wiki( "<li>" + line + " <ac:emoticon ac:name=\"cross\" /></li>\n" )
elif re.search( " - No Result$", line ):
self.log.wiki( "<li>" + line + " <ac:emoticon ac:name=\"warning\" /></li>\n" )
+ else: # Should only be on fail message
+ self.log.wiki( "<ul><li>" + line + "</li></ul>\n" )
self.log.wiki( "</ul>" )
#summary results
self.log.summary( self.stepCache )
diff --git a/TestON/core/testparser.py b/TestON/core/testparser.py
index 37f50f0..24b1ca2 100644
--- a/TestON/core/testparser.py
+++ b/TestON/core/testparser.py
@@ -16,7 +16,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with TestON. If not, see <http://www.gnu.org/licenses/>.
+ along with TestON. If not, see <http://www.gnu.org/licenses/>.
'''
@@ -26,21 +26,21 @@
def __init__(self,testFile):
try :
testFileHandler = open(testFile, 'r')
- except IOError:
+ except IOError:
print "No such file "+testFile
sys.exit(0)
-
+
testFileList = testFileHandler.readlines()
- self.testscript = testFileList
+ self.testscript = testFileList
self.caseCode = {}
self.caseBlock = ''
self.statementsList = []
- index = 0
+ index = 0
self.statementsList = []
#initialSpaces = len(line) -len(line.lstrip())
while index < len(testFileList):
testFileList[index] = re.sub("^\s{8}|^\s{4}", "", testFileList[index])
- # Skip multiline comments
+ # Skip multiline comments
if re.match('^(\'\'\')|^(\"\"\")',testFileList[index],0) :
index = index + 1
try :
@@ -48,16 +48,15 @@
index = index + 1
except IndexError,e:
print ''
-
- # skip empty lines and single line comments
+ # skip empty lines and single line comments
elif not re.match('#|^\s*$',testFileList[index],0):
self.statementsList.append(testFileList[index])
index = index + 1
-
+
def case_code(self):
- index = 0
- statementsList = self.statementsList
+ index = 0
+ statementsList = self.statementsList
while index < len(statementsList):
#print statementsList[index]
m= re.match('def\s+CASE(\d+)',statementsList[index],0)
@@ -76,18 +75,16 @@
except IndexError,e:
#print 'IndexError'
print ''
-
self.caseCode [str(m.group(1))] = self.caseBlock
#print "Case CODE "+self.caseCode [str(m.group(1))]
index = index + 1
-
- return self.caseCode
-
+ return self.caseCode
+
def step_code(self,caseStatements):
index = 0
- step = 0
- stepCode = {}
- step_flag = False
+ step = 0
+ stepCode = {}
+ step_flag = False
while index < len(caseStatements):
m= re.match('main\.step',caseStatements[index],0)
stepBlock = ''
@@ -99,13 +96,13 @@
while i < index :
block += caseStatements[i]
i = i + 1
- stepCode[step] = block
+ stepCode[step] = block
step = step + 1
- stepBlock= stepBlock + caseStatements[index]
+ stepBlock = stepBlock + caseStatements[index]
index = index + 1
try :
while not re.match('main\.step',caseStatements[index],0) :
- stepBlock= stepBlock + caseStatements[index]
+ stepBlock = stepBlock + caseStatements[index]
if index < len(caseStatements)-1:
index = index + 1
else :
@@ -121,11 +118,10 @@
if not step_flag :
stepCode[step] = "".join(caseStatements)
return stepCode
-
+
def getStepCode(self):
case_step_code = {}
case_block = self.case_code()
-
for case in case_block :
case_step_code[case] = {}
step_block = self.step_code(case_block[case])
diff --git a/TestON/core/utilities.py b/TestON/core/utilities.py
index dda43bf..1ba403d 100644
--- a/TestON/core/utilities.py
+++ b/TestON/core/utilities.py
@@ -84,27 +84,27 @@
return result
return assertHandling
- def _assert (self,**assertParam):
+ def _assert (self,**assertParam):
'''
It will take the arguments :
- expect:'Expected output'
- actual:'Actual output'
+ expect:'Expected output'
+ actual:'Actual output'
onpass:'Action or string to be triggered or displayed respectively when the assert passed'
onfail:'Action or string to be triggered or displayed respectively when the assert failed'
not:'optional argument to specify the negation of the each assertion type'
operator:'assertion type will be defined by using operator. Like equal , greater, lesser, matches.'
-
+
It will return the assertion result.
-
+
'''
-
+
arguments = self.parse_args(["EXPECT","ACTUAL","ONPASS","ONFAIL","NOT","OPERATOR"],**assertParam)
-
+
result = 0
valuetype = ''
operation = "not "+ str(arguments["OPERATOR"]) if arguments['NOT'] and arguments['NOT'] == 1 else arguments["OPERATOR"]
operators = {'equals':{'STR':'==','NUM':'=='}, 'matches' : '=~', 'greater':'>' ,'lesser':'<'}
-
+
expectMatch = re.match('^\s*[+-]?0(e0)?\s*$', str(arguments["EXPECT"]), re.I+re.M)
if not ((not expectMatch) and (arguments["EXPECT"]==0)):
valuetype = 'NUM'
@@ -112,31 +112,30 @@
if arguments["OPERATOR"] == 'greater' or arguments["OPERATOR"] == 'lesser':
main.log.error("Numeric comparison on strings is not possibele")
return main.ERROR
-
+
valuetype = 'STR'
arguments["ACTUAL"] = str(arguments["ACTUAL"])
if arguments["OPERATOR"] != 'matches':
arguments["EXPECT"] = str(arguments["EXPECT"])
-
+
try :
opcode = operators[str(arguments["OPERATOR"])][valuetype] if arguments["OPERATOR"] == 'equals' else operators[str(arguments["OPERATOR"])]
-
+
except KeyError:
print "Key Error in assertion"
return main.FALSE
-
+
if opcode == '=~':
try:
assert re.search(str(arguments["EXPECT"]),str(arguments["ACTUAL"]))
result = main.TRUE
except AssertionError:
try :
- assert re.match(str(arguments["EXPECT"]),str(arguments["ACTUAL"]))
+ assert re.match(str(arguments["EXPECT"]),str(arguments["ACTUAL"]))
result = main.TRUE
except AssertionError:
main.log.error("Assertion Failed")
result = main.FALSE
-
else :
try:
if str(opcode)=="==":
@@ -145,27 +144,21 @@
result = main.TRUE
else :
result = main.FALSE
-
elif str(opcode) == ">":
main.log.info("Verifying the Expected is Greater than the actual or not using assert_greater")
if (ast.literal_eval(arguments["EXPECT"]) > ast.literal_eval(arguments["ACTUAL"])) :
result = main.TRUE
else :
result = main.FALSE
-
elif str(opcode) == "<":
main.log.info("Verifying the Expected is Lesser than the actual or not using assert_lesser")
if (ast.literal_eval(arguments["EXPECT"]) < ast.literal_eval(arguments["ACTUAL"])):
result = main.TRUE
else :
result = main.FALSE
-
-
except AssertionError:
main.log.error("Assertion Failed")
result = main.FALSE
-
-
result = result if result else 0
result = not result if arguments["NOT"] and arguments["NOT"] == 1 else result
resultString = ""
@@ -179,7 +172,8 @@
else :
main.log.error(arguments["ONFAIL"])
main.log.report(arguments["ONFAIL"])
-
+ main.onFailMsg = arguments[ 'ONFAIL' ]
+
msg = arguments["ON" + str(resultString)]
if not isinstance(msg,str):
@@ -190,8 +184,7 @@
main.last_result = result
return result
-
-
+
def parse_args(self,args, **kwargs):
'''
It will accept the (key,value) pair and will return the (key,value) pairs with keys in uppercase.
diff --git a/TestON/dependencies/Jenkins_getresult_HA.py b/TestON/dependencies/Jenkins_getresult_HA.py
index 7b4fc09..181e1a9 100755
--- a/TestON/dependencies/Jenkins_getresult_HA.py
+++ b/TestON/dependencies/Jenkins_getresult_HA.py
@@ -6,6 +6,7 @@
import datetime
import time
import argparse
+import glob
import shutil
parser = argparse.ArgumentParser()
@@ -43,10 +44,10 @@
##Graphs
graphs += '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
- graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=2&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
- graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=1&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
- graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=0&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
- graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/getPlot?index=3&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=2&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=1&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=0&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/'+job+'/plot/Plot-HA/getPlot?index=3&width=500&height=300" noborder="0" width="500" height="300" scrolling="yes" seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
graphs += '</ac:structured-macro>\n'
header +="<p> <a href='https://wiki.onosproject.org/display/OST/Test+Plan+-+HA'>Test Plan for HA Test Cases</a></p>"
@@ -63,7 +64,8 @@
path = "/home/admin/ONLabTest/TestON/logs/" + name + "/"
try:
#IF exists, move the csv file to the workspace
- shutil.copy(path + test + ".csv", workspace)
+ for csvFile in glob.glob( path + '*.csv' ):
+ shutil.copy( csvFile, workspace )
except IOError:
#File probably doesn't exist
pass
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
index 42fec2d..7136ac6 100755
--- a/TestON/dependencies/rotate.sh
+++ b/TestON/dependencies/rotate.sh
@@ -49,6 +49,6 @@
root_dir="/home/admin/packet_captures"
timestamp=`date +%Y_%B_%d_%H_%M_%S`
#Maybe this should be an argument? pack-and-rotate supports that
-nr_max=20
+nr_max=10
pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log*" ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index ce6e02d..57d8c9b 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -252,17 +252,16 @@
Optional parameter timeout allows you to specify how long to
wait for pingall to complete
Optional:
- timeout(seconds) - This is the pexpect timeout; The function will
- timeout if the amount of time between failed
- pings exceedes this time and pingall is still
- running
+ timeout(seconds) - How long to wait before breaking the pingall
shortCircuit - Break the pingall based on the number of failed hosts
ping
acceptableFailed - Set the number of acceptable failed pings for the
function to still return main.TRUE
Returns:
main.TRUE if pingall completes with no pings dropped
- otherwise main.FALSE"""
+ otherwise main.FALSE
+ """
+ import time
try:
timeout = int( timeout )
if self.handle:
@@ -273,6 +272,7 @@
failedPings = 0
returnValue = main.TRUE
self.handle.sendline( "pingall" )
+ startTime = time.time()
while True:
i = self.handle.expect( [ "mininet>","X",
pexpect.EOF,
@@ -293,6 +293,12 @@
+ str( failedPings ) +
" pings failed" )
break
+ if ( time.time() - startTime ) > timeout:
+ returnValue = main.FALSE
+ main.log.error( self.name +
+ ": Aborting pingall - " +
+ "Function took too long " )
+ break
elif i == 2:
main.log.error( self.name +
": EOF exception found" )
@@ -361,6 +367,64 @@
main.log.info( self.name + ": \n---\n" + response )
return main.FALSE
+ def pingallHosts( self, hostList, pingType='ipv4' ):
+ """
+ Ping all specified hosts with a specific ping type
+
+ Acceptable pingTypes:
+ - 'ipv4'
+ - 'ipv6'
+
+ Acceptable hostList:
+ - ['h1','h2','h3','h4']
+
+ Returns main.TRUE if all hosts specified can reach
+ each other
+
+ Returns main.FALSE if one or more of hosts specified
+ cannot reach each other"""
+
+ if pingType == "ipv4":
+ cmd = " ping -c 1 -i 1 -W 8 "
+ elif pingType == "ipv6":
+ cmd = " ping6 -c 1 -i 1 -W 8 "
+ else:
+ main.log.warn( "Invalid pingType specified" )
+ return
+
+ try:
+ main.log.info( "Testing reachability between specified hosts" )
+
+ isReachable = main.TRUE
+
+ for host in hostList:
+ listIndex = hostList.index(host)
+ # List of hosts to ping other than itself
+ pingList = hostList[:listIndex] + hostList[(listIndex+1):]
+
+ for temp in pingList:
+ # Current host pings all other hosts specified
+ pingCmd = str(host) + cmd + str(temp)
+ self.handle.sendline( pingCmd )
+ i = self.handle.expect( [ pingCmd, pexpect.TIMEOUT ] )
+ j = self.handle.expect( [ "mininet>", pexpect.TIMEOUT ] )
+ response = self.handle.before
+ if re.search( ',\s0\%\spacket\sloss', response ):
+ main.log.info( str(host) + " -> " + str(temp) )
+ else:
+ main.log.info( str(host) + " -> X ("+str(temp)+") "
+ " Destination Unreachable" )
+ # One of the host to host pair is unreachable
+ isReachable = main.FALSE
+
+ return isReachable
+
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+
def pingHost( self, **pingParams ):
"""
Ping from one mininet host to another
@@ -1304,11 +1368,11 @@
self.handle.sendline('')
i = self.handle.expect( [ 'mininet>', pexpect.EOF, pexpect.TIMEOUT ],
timeout=2)
+ response = main.TRUE
if i == 0:
- self.stopNet()
+ response = self.stopNet()
elif i == 1:
return main.TRUE
- response = main.TRUE
# print "Disconnecting Mininet"
if self.handle:
self.handle.sendline( "exit" )
@@ -1371,20 +1435,32 @@
response = main.FALSE
return response
- def arping( self, src, dest, destmac ):
- self.handle.sendline( '' )
- self.handle.expect( [ "mininet", pexpect.EOF, pexpect.TIMEOUT ] )
-
- self.handle.sendline( src + ' arping ' + dest )
+ def arping( self, host="", ip="10.128.20.211", ethDevice="" ):
+ """
+ Description:
+ Sends arp message from mininet host for hosts discovery
+ Required:
+ host - hosts name
+ Optional:
+ ip - ip address that does not exist in the network so there would
+ be no reply.
+ """
+ if ethDevice:
+ ethDevice = '-I ' + ethDevice + ' '
+ cmd = " py " + host + ".cmd(\"arping -c 1 " + ethDevice + ip + "\")"
try:
- self.handle.expect( [ destmac, pexpect.EOF, pexpect.TIMEOUT ] )
- main.log.info( self.name + ": ARP successful" )
- self.handle.expect( [ "mininet", pexpect.EOF, pexpect.TIMEOUT ] )
+ main.log.warn( "Sending: " + cmd )
+ self.handle.sendline( cmd )
+ response = self.handle.before
+ self.handle.sendline( "" )
+ self.handle.expect( "mininet>" )
return main.TRUE
- except Exception:
- main.log.warn( self.name + ": ARP FAILURE" )
- self.handle.expect( [ "mininet", pexpect.EOF, pexpect.TIMEOUT ] )
- return main.FALSE
+
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
def decToHex( self, num ):
return hex( num ).split( 'x' )[ 1 ]
@@ -1551,7 +1627,7 @@
if switchesJson == "": # if rest call fails
main.log.error(
self.name +
- ".compare_switches(): Empty JSON object given from ONOS" )
+ ".compareSwitches(): Empty JSON object given from ONOS" )
return main.FALSE
onos = switchesJson
onosDPIDs = []
@@ -1860,7 +1936,7 @@
if onosMAC == mnIntf[ 'hw_addr' ].lower() :
match = True
for ip in mnIntf[ 'ips' ]:
- if ip in onosHost[ 'ips' ]:
+ if ip in onosHost[ 'ipAddresses' ]:
pass # all is well
else:
# misssing ip
@@ -1905,6 +1981,7 @@
hostStr = handlePy.replace( "]", "" )
hostStr = hostStr.replace( "'", "" )
hostStr = hostStr.replace( "[", "" )
+ hostStr = hostStr.replace( " ", "" )
hostList = hostStr.split( "," )
return hostList
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index e0a205d..c2c011a 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -91,6 +91,7 @@
elif re.search( "found multiple mininet", outputs ):
return main.ERROR
else:
+ # TODO: Parse for failed pings, give some truncated output
main.log.error( "Error, unexpected output in the ping file" )
main.log.warn( outputs )
return main.TRUE
@@ -409,7 +410,6 @@
cmd = "sudo -E python opticalTest.py " + controller
main.log.info( self.name + ": cmd = " + cmd )
self.handle.sendline( cmd )
- #self.handle.expect( "Press ENTER to push Topology.json" )
time.sleep(30)
self.handle.sendline( "" )
self.handle.sendline( "" )
@@ -448,8 +448,8 @@
# Close the ssh connection
self.handle.sendline( "" )
# self.handle.expect( "\$" )
- i = self.handle.expect( [ '\$', 'mininet>', pexpect.TIMEOUT ],
- timeout=2)
+ i = self.handle.expect( [ '\$', 'mininet>', pexpect.TIMEOUT,
+ pexpect.EOF ], timeout=2 )
if i == 0:
self.handle.sendline( "exit" )
self.handle.expect( "closed" )
@@ -460,7 +460,6 @@
self.handle.sendline( "exit" )
self.handle.expect( "exit" )
self.handle.expect( "closed" )
-
else:
main.log.error( "Connection failed to the host" )
return main.TRUE
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 703e15f..5207f1e 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -188,7 +188,8 @@
main.cleanup()
main.exit()
- def startOnosCli( self, ONOSIp, karafTimeout="" ):
+ def startOnosCli( self, ONOSIp, karafTimeout="",
+ commandlineTimeout=10, onosStartTimeout=60 ):
"""
karafTimeout is an optional argument. karafTimeout value passed
by user would be used to set the current karaf shell idle timeout.
@@ -206,7 +207,7 @@
try:
self.handle.sendline( "" )
x = self.handle.expect( [
- "\$", "onos>" ], timeout=10 )
+ "\$", "onos>" ], commandlineTimeout)
if x == 1:
main.log.info( "ONOS cli is already running" )
@@ -216,7 +217,7 @@
self.handle.sendline( "onos -w " + str( ONOSIp ) )
i = self.handle.expect( [
"onos>",
- pexpect.TIMEOUT ], timeout=60 )
+ pexpect.TIMEOUT ], onosStartTimeout )
if i == 0:
main.log.info( str( ONOSIp ) + " CLI Started successfully" )
@@ -280,15 +281,23 @@
lvlStr = "--level=" + level
self.handle.sendline( "" )
- self.handle.expect( "onos>" )
+ i = self.handle.expect( [ "onos>", pexpect.TIMEOUT ] )
+ # TODO: look for bash prompt as well
+ if i == 1:
+ self.handle.sendline( "" )
+ self.handle.expect( "onos>" )
self.handle.sendline( "log:log " + lvlStr + " " + cmdStr )
+ self.handle.expect( "log:log" )
self.handle.expect( "onos>" )
response = self.handle.before
if re.search( "Error", response ):
return main.FALSE
return main.TRUE
-
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.cleanup()
+ main.exit()
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -327,34 +336,35 @@
main.log.info( "Command '" + str( cmdStr ) + "' sent to "
+ self.name + "." )
if debug:
- main.log.debug( "Raw output" )
- main.log.debug( repr( response ) )
+ main.log.debug( self.name + ": Raw output" )
+ main.log.debug( self.name + ": " + repr( response ) )
# Remove ANSI color control strings from output
ansiEscape = re.compile( r'\x1b[^m]*m' )
response = ansiEscape.sub( '', response )
if debug:
- main.log.debug( "ansiEscape output" )
- main.log.debug( repr( response ) )
+ main.log.debug( self.name + ": ansiEscape output" )
+ main.log.debug( self.name + ": " + repr( response ) )
# Remove extra return chars that get added
response = re.sub( r"\s\r", "", response )
if debug:
- main.log.debug( "Removed extra returns from output" )
- main.log.debug( repr( response ) )
+ main.log.debug( self.name + ": Removed extra returns " +
+ "from output" )
+ main.log.debug( self.name + ": " + repr( response ) )
# Strip excess whitespace
response = response.strip()
if debug:
- main.log.debug( "parsed and stripped output" )
- main.log.debug( repr( response ) )
+ main.log.debug( self.name + ": parsed and stripped output" )
+ main.log.debug( self.name + ": " + repr( response ) )
# parse for just the output, remove the cmd from response
output = response.split( cmdStr.strip(), 1 )
if debug:
- main.log.debug( "split output" )
+ main.log.debug( self.name + ": split output" )
for r in output:
- main.log.debug( repr( r ) )
+ main.log.debug( self.name + ": " + repr( r ) )
return output[1].strip()
except IndexError:
main.log.exception( self.name + ": Object not as expected" )
@@ -471,7 +481,7 @@
def topology( self ):
"""
Definition:
- Returns the ouput of topology command.
+ Returns the output of topology command.
Return:
topology = current ONOS topology
"""
@@ -1723,7 +1733,7 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanup()
main.exit()
-
+
def checkIntentState( self, intentsId, expectedState = 'INSTALLED' ):
"""
Description:
@@ -1740,7 +1750,7 @@
try:
# Generating a dictionary: intent id as a key and state as value
intentsDict = self.getIntentState( intentsId )
- print "len of intentsDict ", str( len( intentsDict ) )
+ #print "len of intentsDict ", str( len( intentsDict ) )
if len( intentsId ) != len( intentsDict ):
main.log.info( self.name + "There is something wrong " +
"getting intents state" )
@@ -1812,15 +1822,18 @@
"""
try:
tempFlows = json.loads( self.flows() )
+ #print tempFlows[0]
returnValue = main.TRUE
+
for device in tempFlows:
for flow in device.get( 'flows' ):
if flow.get( 'state' ) != 'ADDED' and flow.get( 'state' ) != \
'PENDING_ADD':
main.log.info( self.name + ": flow Id: " +
- flow.get( 'flowId' ) +
+ flow.get( 'groupId' ) +
" | state:" + flow.get( 'state' ) )
returnValue = main.FALSE
+
return returnValue
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
@@ -2167,7 +2180,7 @@
elif logLevel == "warn":
main.log.warn( output )
else:
- main.log.info( output )
+ main.log.info( self.name + ": " + output )
return result
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
@@ -2285,8 +2298,8 @@
# TODO: Should this be main.ERROR?
return main.FALSE
else:
- main.log.error( "Error in election_test_leader: " +
- "unexpected response" )
+ main.log.error( "Error in electionTestLeader on " + self.name +
+ ": " + "unexpected response" )
main.log.error( repr( response ) )
return main.FALSE
except TypeError:
@@ -2326,8 +2339,8 @@
main.log.error( "Election app is not loaded on " + self.name )
return main.FALSE
else:
- main.log.error( "Error in election_test_run: " +
- "unexpected response" )
+ main.log.error( "Error in electionTestRun on " + self.name +
+ ": " + "unexpected response" )
main.log.error( repr( response ) )
return main.FALSE
except TypeError:
@@ -2367,8 +2380,8 @@
main.log.error( "Election app is not loaded on " + self.name )
return main.FALSE
else:
- main.log.error( "Error in election_test_withdraw: " +
- "unexpected response" )
+ main.log.error( "Error in electionTestWithdraw on " +
+ self.name + ": " + "unexpected response" )
main.log.error( repr( response ) )
return main.FALSE
except TypeError:
@@ -2931,8 +2944,21 @@
main.ERROR if there is some error in processing the test
"""
try:
- ids = json.loads( self.appIDs( jsonFormat=True ) )
- apps = json.loads( self.apps( jsonFormat=True ) )
+ bail = False
+ ids = self.appIDs( jsonFormat=True )
+ if ids:
+ ids = json.loads( ids )
+ else:
+ main.log.error( "app-ids returned nothing:" + repr( ids ) )
+ bail = True
+ apps = self.apps( jsonFormat=True )
+ if apps:
+ apps = json.loads( apps )
+ else:
+ main.log.error( "apps returned nothing:" + repr( apps ) )
+ bail = True
+ if bail:
+ return main.FALSE
result = main.TRUE
for app in apps:
appID = app.get( 'id' )
@@ -2944,7 +2970,7 @@
main.log.error( "Error parsing app: " + str( app ) )
result = main.FALSE
# get the entry in ids that has the same appID
- current = filter(lambda item: item[ 'id' ] == appID, ids)
+ current = filter( lambda item: item[ 'id' ] == appID, ids )
# main.log.debug( "Comparing " + str( app ) + " to " +
# str( current ) )
if not current: # if ids doesn't have this id
@@ -3044,10 +3070,11 @@
def setCfg( self, component, propName, value=None, check=True ):
"""
Set/Unset configuration settings from ONOS cli
- Optional arguments:
+ Required arguments:
component - The case sensitive name of the component whose
property is to be set
propName - The case sensitive name of the property to be set/unset
+ Optional arguments:
value - The value to set the property to. If None, will unset the
property and revert it to it's default value(if applicable)
check - Boolean, Check whether the option was successfully set this
@@ -3096,3 +3123,428 @@
main.cleanup()
main.exit()
+ def setTestAdd( self, setName, values ):
+ """
+ CLI command to add elements to a distributed set.
+ Arguments:
+ setName - The name of the set to add to.
+ values - The value(s) to add to the set, space seperated.
+ Example usages:
+ setTestAdd( "set1", "a b c" )
+ setTestAdd( "set2", "1" )
+ returns:
+ main.TRUE on success OR
+ main.FALSE if elements were already in the set OR
+ main.ERROR on error
+ """
+ try:
+ cmdStr = "set-test-add " + str( setName ) + " " + str( values )
+ output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
+ assert "Error executing command" not in output
+ positiveMatch = "\[(.*)\] was added to the set " + str( setName )
+ negativeMatch = "\[(.*)\] was already in set " + str( setName )
+ main.log.info( self.name + ": " + output )
+ if re.search( positiveMatch, output):
+ return main.TRUE
+ elif re.search( negativeMatch, output):
+ return main.FALSE
+ else:
+ main.log.error( self.name + ": setTestAdd did not" +
+ " match expected output" )
+ main.log.debug( self.name + " actual: " + repr( output ) )
+ return main.ERROR
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' command: " +
+ str( output ) )
+ return main.ERROR
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return main.ERROR
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanup()
+ main.exit()
+
+ def setTestRemove( self, setName, values, clear=False, retain=False ):
+ """
+ CLI command to remove elements from a distributed set.
+ Required arguments:
+ setName - The name of the set to remove from.
+ values - The value(s) to remove from the set, space seperated.
+ Optional arguments:
+ clear - Clear all elements from the set
+ retain - Retain only the given values. (intersection of the
+ original set and the given set)
+ returns:
+ main.TRUE on success OR
+ main.FALSE if the set was not changed OR
+ main.ERROR on error
+ """
+ try:
+ cmdStr = "set-test-remove "
+ if clear:
+ cmdStr += "-c " + str( setName )
+ elif retain:
+ cmdStr += "-r " + str( setName ) + " " + str( values )
+ else:
+ cmdStr += str( setName ) + " " + str( values )
+ output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
+ assert "Error executing command" not in output
+ main.log.info( self.name + ": " + output )
+ if clear:
+ pattern = "Set " + str( setName ) + " cleared"
+ if re.search( pattern, output ):
+ return main.TRUE
+ elif retain:
+ positivePattern = str( setName ) + " was pruned to contain " +\
+ "only elements of set \[(.*)\]"
+ negativePattern = str( setName ) + " was not changed by " +\
+ "retaining only elements of the set " +\
+ "\[(.*)\]"
+ if re.search( positivePattern, output ):
+ return main.TRUE
+ elif re.search( negativePattern, output ):
+ return main.FALSE
+ else:
+ positivePattern = "\[(.*)\] was removed from the set " +\
+ str( setName )
+ if ( len( values.split() ) == 1 ):
+ negativePattern = "\[(.*)\] was not in set " +\
+ str( setName )
+ else:
+ negativePattern = "No element of \[(.*)\] was in set " +\
+ str( setName )
+ if re.search( positivePattern, output ):
+ return main.TRUE
+ elif re.search( negativePattern, output ):
+ return main.FALSE
+ main.log.error( self.name + ": setTestRemove did not" +
+ " match expected output" )
+ main.log.debug( self.name + " expected: " + pattern )
+ main.log.debug( self.name + " actual: " + repr( output ) )
+ return main.ERROR
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-remove' command: " +
+ str( output ) )
+ return main.ERROR
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return main.ERROR
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanup()
+ main.exit()
+
+ def setTestGet( self, setName, values="" ):
+ """
+ CLI command to get the elements in a distributed set.
+ Required arguments:
+ setName - The name of the set to remove from.
+ Optional arguments:
+ values - The value(s) to check if in the set, space seperated.
+ returns:
+ main.ERROR on error OR
+ A list of elements in the set if no optional arguments are
+ supplied OR
+ A tuple containing the list then:
+ main.FALSE if the given values are not in the set OR
+ main.TRUE if the given values are in the set OR
+ """
+ try:
+ values = str( values ).strip()
+ setName = str( setName ).strip()
+ length = len( values.split() )
+ containsCheck = None
+ # Patterns to match
+ setPattern = "\[(.*)\]"
+ pattern = "Items in set " + setName + ":\n" + setPattern
+ containsTrue = "Set " + setName + " contains the value " + values
+ containsFalse = "Set " + setName + " did not contain the value " +\
+ values
+ containsAllTrue = "Set " + setName + " contains the the subset " +\
+ setPattern
+ containsAllFalse = "Set " + setName + " did not contain the the" +\
+ " subset " + setPattern
+
+ cmdStr = "set-test-get "
+ cmdStr += setName + " " + values
+ output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
+ assert "Error executing command" not in output
+ main.log.info( self.name + ": " + output )
+
+ if length == 0:
+ match = re.search( pattern, output )
+ else: # if given values
+ if length == 1: # Contains output
+ patternTrue = pattern + "\n" + containsTrue
+ patternFalse = pattern + "\n" + containsFalse
+ else: # ContainsAll output
+ patternTrue = pattern + "\n" + containsAllTrue
+ patternFalse = pattern + "\n" + containsAllFalse
+ matchTrue = re.search( patternTrue, output )
+ matchFalse = re.search( patternFalse, output )
+ if matchTrue:
+ containsCheck = main.TRUE
+ match = matchTrue
+ elif matchFalse:
+ containsCheck = main.FALSE
+ match = matchFalse
+ else:
+ main.log.error( self.name + " setTestGet did not match " +\
+ "expected output" )
+ main.log.debug( self.name + " expected: " + pattern )
+ main.log.debug( self.name + " actual: " + repr( output ) )
+ match = None
+ if match:
+ setMatch = match.group( 1 )
+ if setMatch == '':
+ setList = []
+ else:
+ setList = setMatch.split( ", " )
+ if length > 0:
+ return ( setList, containsCheck )
+ else:
+ return setList
+ else: # no match
+ main.log.error( self.name + ": setTestGet did not" +
+ " match expected output" )
+ main.log.debug( self.name + " expected: " + pattern )
+ main.log.debug( self.name + " actual: " + repr( output ) )
+ return main.ERROR
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-get' command: " +
+ str( output ) )
+ return main.ERROR
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return main.ERROR
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanup()
+ main.exit()
+
+ def setTestSize( self, setName ):
+ """
+ CLI command to get the elements in a distributed set.
+ Required arguments:
+ setName - The name of the set to remove from.
+ returns:
+ The integer value of the size returned or
+ None on error
+ """
+ try:
+ # TODO: Should this check against the number of elements returned
+ # and then return true/false based on that?
+ setName = str( setName ).strip()
+ # Patterns to match
+ setPattern = "\[(.*)\]"
+ pattern = "There are (\d+) items in set " + setName + ":\n" +\
+ setPattern
+ cmdStr = "set-test-get -s "
+ cmdStr += setName
+ output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
+ assert "Error executing command" not in output
+ main.log.info( self.name + ": " + output )
+ match = re.search( pattern, output )
+ if match:
+ setSize = int( match.group( 1 ) )
+ setMatch = match.group( 2 )
+ if len( setMatch.split() ) == setSize:
+ main.log.info( "The size returned by " + self.name +
+ " matches the number of elements in " +
+ "the returned set" )
+ else:
+ main.log.error( "The size returned by " + self.name +
+ " does not match the number of " +
+ "elements in the returned set." )
+ return setSize
+ else: # no match
+ main.log.error( self.name + ": setTestGet did not" +
+ " match expected output" )
+ main.log.debug( self.name + " expected: " + pattern )
+ main.log.debug( self.name + " actual: " + repr( output ) )
+ return None
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-get' command: " +
+ str( output ) )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanup()
+ main.exit()
+
+ def counters( self, jsonFormat=True ):
+ """
+ Command to list the various counters in the system.
+ returns:
+ if jsonFormat, a string of the json object returned by the cli
+ command
+ if not jsonFormat, the normal string output of the cli command
+ None on error
+ """
+ try:
+ counters = {}
+ cmdStr = "counters"
+ if jsonFormat:
+ cmdStr += " -j"
+ output = self.sendline( cmdStr )
+ assert "Error executing command" not in output
+ main.log.info( self.name + ": " + output )
+ return output
+ except AssertionError:
+ main.log.error( "Error in processing 'counters' command: " +
+ str( output ) )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanup()
+ main.exit()
+
+ def counterTestIncrement( self, counter, inMemory=False ):
+ """
+ CLI command to increment and get a distributed counter.
+ Required arguments:
+ counter - The name of the counter to increment.
+ Optional arguments:
+ inMemory - use in memory map for the counter
+ returns:
+ integer value of the counter or
+ None on Error
+ """
+ try:
+ counter = str( counter )
+ cmdStr = "counter-test-increment "
+ if inMemory:
+ cmdStr += "-i "
+ cmdStr += counter
+ output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
+ assert "Error executing command" not in output
+ main.log.info( self.name + ": " + output )
+ pattern = counter + " was incremented to (\d+)"
+ match = re.search( pattern, output )
+ if match:
+ return int( match.group( 1 ) )
+ else:
+ main.log.error( self.name + ": counterTestIncrement did not" +
+ " match expected output." )
+ main.log.debug( self.name + " expected: " + pattern )
+ main.log.debug( self.name + " actual: " + repr( output ) )
+ return None
+ except AssertionError:
+ main.log.error( "Error in processing 'counter-test-increment'" +
+ " command: " + str( output ) )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanup()
+ main.exit()
+
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 7d5ab5e..9d0a416 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -20,6 +20,7 @@
import time
import pexpect
import os.path
+from requests.models import Response
sys.path.append( "../" )
from drivers.common.clidriver import CLI
@@ -97,7 +98,28 @@
response = main.FALSE
return response
- def onosPackage( self ):
+ def getEpochMs( self ):
+ """
+ Returns milliseconds since epoch
+
+ When checking multiple nodes in a for loop,
+ around a hundred milliseconds of difference (ascending) is
+ generally acceptable due to calltime of the function.
+ Few seconds, however, is not and it means clocks
+ are off sync.
+ """
+ try:
+ self.handle.sendline( 'date +%s.%N' )
+ self.handle.expect( 'date \+\%s\.\%N' )
+ self.handle.expect( '\$' )
+ epochMs = self.handle.before
+ return epochMs
+ except Exception:
+ main.log.exception( 'Uncaught exception getting epoch time' )
+ main.cleanup()
+ main.exit()
+
+ def onosPackage( self, opTimeout=30 ):
"""
Produce a self-contained tar.gz file that can be deployed
and executed on any platform with Java 7 JRE.
@@ -105,7 +127,7 @@
try:
self.handle.sendline( "onos-package" )
self.handle.expect( "onos-package" )
- self.handle.expect( "tar.gz", timeout=30 )
+ self.handle.expect( "tar.gz", opTimeout )
handle = str( self.handle.before )
main.log.info( "onos-package command returned: " +
handle )
@@ -152,7 +174,7 @@
main.cleanup()
main.exit()
- def cleanInstall( self ):
+ def cleanInstall( self, mciTimeout=600 ):
"""
Runs mvn clean install in the root of the ONOS directory.
This will clean all ONOS artifacts then compile each module
@@ -179,7 +201,7 @@
'BUILD\sSUCCESS',
'onos\$', #TODO: fix this to be more generic?
'ONOS\$',
- pexpect.TIMEOUT ], timeout=600 )
+ pexpect.TIMEOUT ], mciTimeout )
if i == 0:
main.log.error( self.name + ":There is insufficient memory \
for the Java Runtime Environment to continue." )
@@ -483,6 +505,24 @@
main.cleanup()
main.exit()
+ def getBranchName( self ):
+ main.log.info( "self.home = " )
+ main.log.info( self.home )
+ self.handle.sendline( "cd " + self.home )
+ self.handle.expect( self.home + "\$" )
+ self.handle.sendline( "git name-rev --name-only HEAD" )
+ self.handle.expect( "git name-rev --name-only HEAD" )
+ self.handle.expect( "\$" )
+
+ lines = self.handle.before.splitlines()
+ if lines[1] == "master":
+ return "master"
+ elif lines[1] == "onos-1.0":
+ return "onos-1.0"
+ else:
+ main.log.info( lines[1] )
+ return "unexpected ONOS branch for SDN-IP test"
+
def getVersion( self, report=False ):
"""
Writes the COMMIT number to the report to be parsed
@@ -508,13 +548,18 @@
for line in lines:
print line
if report:
+ main.log.wiki( "<blockquote>" )
for line in lines[ 2:-1 ]:
# Bracket replacement is for Wiki-compliant
# formatting. '<' or '>' are interpreted
# as xml specific tags that cause errors
line = line.replace( "<", "[" )
line = line.replace( ">", "]" )
- main.log.report( "\t" + line )
+ #main.log.wiki( "\t" + line )
+ main.log.wiki( line + "<br /> " )
+ main.log.summary( line )
+ main.log.wiki( "</blockquote>" )
+ main.log.summary("\n")
return lines[ 2 ]
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
@@ -562,6 +607,8 @@
# on here.
appString = "export ONOS_APPS=" + appString
mnString = "export OCN="
+ if mnIpAddrs == "":
+ mnString = ""
onosString = "export OC"
tempCount = 1
@@ -921,7 +968,7 @@
"""
try:
self.handle.sendline( "" )
- self.handle.expect( "\$" )
+ self.handle.expect( "\$", timeout=60 )
self.handle.sendline( "onos-uninstall " + str( nodeIp ) )
self.handle.expect( "\$" )
@@ -930,6 +977,9 @@
# onos-uninstall command does not return any text
return main.TRUE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": Timeout in onosUninstall" )
+ return main.FALSE
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -1492,13 +1542,18 @@
except Exception:
main.log.exception( "Copying files failed" )
- def checkLogs( self, onosIp ):
+ def checkLogs( self, onosIp, restart=False):
"""
runs onos-check-logs on the given onos node
+ If restart is True, use the old version of onos-check-logs which
+ does not print the full stacktrace, but shows the entire log file,
+ including across restarts
returns the response
"""
try:
cmd = "onos-check-logs " + str( onosIp )
+ if restart:
+ cmd += " old"
self.handle.sendline( cmd )
self.handle.expect( cmd )
self.handle.expect( "\$" )
@@ -1896,3 +1951,300 @@
main.cleanup()
main.exit()
+ def getOnosIps(self):
+
+ import os
+
+ # reads env for OC variables, also saves file with OC variables. If file and env conflict
+ # priority goes to env. If file has OCs that are not in the env, the file OCs are used.
+ # In other words, if the env is set, the test will use those values.
+
+ # returns a list of ip addresses for the onos nodes, will work with up to 7 nodes + OCN and OCI
+ # returns in format [ OC1 ip, OC2 ...ect. , OCN, OCI ]
+
+ envONOSIps = {}
+
+ x = 1
+ while True:
+ try:
+ temp = os.environ[ 'OC' + str(x) ]
+ except KeyError:
+ break
+ envONOSIps[ ("OC" + str(x)) ] = temp
+ x += 1
+
+ try:
+ temp = os.environ[ 'OCN' ]
+ envONOSIps[ "OCN" ] = temp
+ except KeyError:
+ main.log.info("OCN not set in env")
+
+ try:
+ temp = os.environ[ 'OCI' ]
+ envONOSIps[ "OCI" ] = temp
+ except:
+ main.log.error("OCI not set in env")
+
+ print(str(envONOSIps))
+
+ order = [ "OC1", "OC2", "OC3","OC4","OC5","OC6","OC7","OCN","OCI" ]
+ ONOSIps = []
+
+ try:
+ if os.path.exists("myIps"):
+ ipFile = open("myIps","r+")
+ else:
+ ipFile = open("myIps","w+")
+
+ fileONOSIps = ipFile.readlines()
+ ipFile.close()
+
+ print str(fileONOSIps)
+
+ if str(fileONOSIps) == "[]":
+ ipFile = open("myIps","w+")
+ for key in envONOSIps:
+ ipFile.write(key+ "=" + envONOSIps[key] + "\n")
+ ipFile.close()
+ for i in order:
+ if i in envONOSIps:
+ ONOSIps.append(envONOSIps[i])
+
+ return ONOSIps
+
+ else:
+ fileDict = {}
+ for line in fileONOSIps:
+ line = line.replace("\n","")
+ line = line.split("=")
+ key = line[0]
+ value = line[1]
+ fileDict[key] = value
+
+ for x in envONOSIps:
+ if x in fileDict:
+ if envONOSIps[x] == fileDict[x]:
+ continue
+ else:
+ fileDict[x] = envONOSIps[x]
+ else:
+ fileDict[x] = envONOSIps[x]
+
+ ipFile = open("myIps","w+")
+ for key in order:
+ if key in fileDict:
+ ipFile.write(key + "=" + fileDict[key] + "\n")
+ ONOSIps.append(fileDict[key])
+ ipFile.close()
+
+ return ONOSIps
+
+ except IOError as a:
+ main.log.error(a)
+
+ except Exception as a:
+ main.log.error(a)
+
+
+ def logReport(self, nodeIp, searchTerms, outputMode="s"):
+ '''
+ - accepts either a list or a string for "searchTerms" these
+ terms will be searched for in the log and have their
+ instances counted
+
+ - nodeIp is the ip of the node whos log is to be scanned
+
+ - output modes:
+ "s" - Simple. Quiet output mode that just prints
+ the occurences of each search term
+
+ "d" - Detailed. Prints number of occurences as well as the entire
+ line for each of the last 5 occurences
+
+ - returns total of the number of instances of all search terms
+ '''
+ main.log.info("========================== Log Report ===========================\n")
+
+ if type(searchTerms) is str:
+ searchTerms = [searchTerms]
+
+ logLines = [ [" "] for i in range(len(searchTerms)) ]
+
+ for term in range(len(searchTerms)):
+ logLines[term][0] = searchTerms[term]
+
+ totalHits = 0
+ for term in range(len(searchTerms)):
+ cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep " + searchTerms[term]
+ self.handle.sendline(cmd)
+ self.handle.expect(":~")
+ before = (self.handle.before).splitlines()
+
+ count = [searchTerms[term],0]
+
+ for line in before:
+ if searchTerms[term] in line and "grep" not in line:
+ count[1] += 1
+ if before.index(line) > ( len(before) - 7 ):
+ logLines[term].append(line)
+
+ main.log.info( str(count[0]) + ": " + str(count[1]) )
+ if term == len(searchTerms)-1:
+ print("\n")
+ totalHits += int(count[1])
+
+ if outputMode != "s" and outputMode != "S":
+ outputString = ""
+ for i in logLines:
+ outputString = i[0] + ": \n"
+ for x in range(1,len(i)):
+ outputString += ( i[x] + "\n" )
+
+ if outputString != (i[0] + ": \n"):
+ main.log.info(outputString)
+
+ main.log.info("================================================================\n")
+ return totalHits
+
+ def getOnosIpFromEnv(self):
+
+ import string
+
+ # returns a list of ip addresses for the onos nodes, will work with up to 7 nodes + OCN and OCI
+ # returns in format [ 'x', OC1 ip, OC2 i... ect. ... , ONN ip ]
+
+ self.handle.sendline("env| grep OC")
+ self.handle.expect(":~")
+ rawOutput = self.handle.before
+ print rawOutput
+ print "-----------------------------"
+ print repr(rawOutput)
+ mpa = dict.fromkeys(range(32))
+ translated = rawOutput.translate(mpa)
+ print translated
+
+
+ # create list with only the lines that have the needed IPs
+ unparsedIps = []
+
+ # remove preceeding or trailing lines
+ for line in rawOutput:
+ if "OC" in line and "=" in line:
+ unparsedIps.append(str(line))
+
+ # determine cluster size
+ clusterCount = 0
+ for line in unparsedIps:
+ line = str(line)
+ print line
+ temp = line.replace("OC","")
+ print("line index " + str(line.index("=")))
+ OCindex = temp[0]
+ for i in range(0, 7):
+ if OCindex == str(i) and i > clusterCount:
+ clusterCount == i
+ print(clusterCount)
+ # create list to hold ips such that OC1 is at list[1] and OCN and OCI are at the end (in that order)
+ ONOSIps = ["x"] * (clusterCount + 3)
+
+ # populate list
+ for line in unparsedIps:
+ main.log.info(line)##########
+ temp = str(line.replace("OC",""))
+ main.log.info(str(list(temp)))
+ OCindex = temp[0]
+ main.log.info(OCindex)############
+ if OCindex == "N":
+ ONOSIps[ clusterCount + 1 ] = temp.replace("N=","")
+
+ if OCindex == "I":
+ ONOSIps[ clusterCount + 2 ] = temp.replace("I=","")
+
+ else:
+ ONOSIps[ int(OCindex) ] = temp.replace((OCindex + "=") ,"")
+
+ # validate
+ for x in ONOSIps:
+ if ONOSIps.index(x) != 0 and x == "x":
+ main.log.error("ENV READ FAILURE, MISSING DATA: \n\n" + str(ONOSIps) + "\n\n")
+
+ return ONOSIps
+
+
+ def onosErrorLog(self, nodeIp):
+
+ cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep WARN"
+ self.handle.sendline(cmd)
+ self.handle.expect(":~")
+ before = (self.handle.before).splitlines()
+
+ warnings = []
+
+ for line in before:
+ if "WARN" in line and "grep" not in line:
+ warnings.append(line)
+ main.warnings[main.warnings[0]+1] = line
+ main.warnings[0] += 1
+ if main.warnings[0] >= 10:
+ main.warnings[0] = 0
+
+ cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep ERROR"
+ self.handle.sendline(cmd)
+ self.handle.expect(":~")
+ before = (self.handle.before).splitlines()
+
+ errors = []
+
+ for line in before:
+ if "ERROR" in line and "grep" not in line:
+ errors.append(line)
+ main.errors[main.errors[0]+1] = line
+ main.errors[0] += 1
+ if main.errors[0] >= 10:
+ main.errors[0] = 0
+
+ cmd = "onos-ssh " + nodeIp + " cat /opt/onos/log/karaf.log | grep Exept"
+ self.handle.sendline(cmd)
+ self.handle.expect(":~")
+ before = (self.handle.before).splitlines()
+
+ exceptions = []
+
+ for line in before:
+ if "Except" in line and "grep" not in line:
+ exceptions.append(line)
+ main.exceptions[main.errors[0]+1] = line
+ main.exceptions[0] += 1
+ if main.exceptions[0] >= 10:
+ main.exceptions[0] = 0
+
+
+
+ ################################################################
+
+ msg1 = "WARNINGS: \n"
+ for i in main.warnings:
+ if type(i) is not int:
+ msg1 += ( i + "\n")
+
+ msg2 = "ERRORS: \n"
+ for i in main.errors:
+ if type(i) is not int:
+ msg2 += ( i + "\n")
+
+ msg3 = "EXCEPTIONS: \n"
+ for i in main.exceptions:
+ if type(i) is not int:
+ msg3 += ( i + "\n")
+
+ main.log.info("===============================================================\n")
+ main.log.info( "Warnings: " + str(len(warnings)))
+ main.log.info( "Errors: " + str(len(errors)))
+ main.log.info( "Exceptions: " + str(len(exceptions)))
+ if len(warnings) > 0:
+ main.log.info(msg1)
+ if len(errors) > 0:
+ main.log.info(msg2)
+ if len(exceptions) > 0:
+ main.log.info(msg3)
+ main.log.info("===============================================================\n")
diff --git a/TestON/drivers/common/cli/quaggaclidriver.py b/TestON/drivers/common/cli/quaggaclidriver.py
index 1c63206..7e3df2b 100644
--- a/TestON/drivers/common/cli/quaggaclidriver.py
+++ b/TestON/drivers/common/cli/quaggaclidriver.py
@@ -40,9 +40,9 @@
ip_address="1.1.1.1",
port=self.port,
pwd=self.pwd )
- main.log.info( "connect parameters:" + str( self.user_name ) + ";"
- + str( self.ip_address ) + ";" + str( self.port )
- + ";" + str(self.pwd ) )
+ #main.log.info( "connect parameters:" + str( self.user_name ) + ";"
+ # + str( self.ip_address ) + ";" + str( self.port )
+ # + ";" + str(self.pwd ) )
if self.handle:
# self.handle.expect( "",timeout=10 )
@@ -186,7 +186,7 @@
return intents
# This method extracts all actual routes from ONOS CLI
- def extractActualRoutes( self, getRoutesResult ):
+ def extractActualRoutesOneDotZero( self, getRoutesResult ):
routesJsonObj = json.loads( getRoutesResult )
allRoutesActual = []
@@ -199,6 +199,18 @@
return sorted( allRoutesActual )
+ def extractActualRoutesMaster( self, getRoutesResult ):
+ routesJsonObj = json.loads( getRoutesResult )
+
+ allRoutesActual = []
+ for route in routesJsonObj['routes4']:
+ if route[ 'prefix' ] == '172.16.10.0/24':
+ continue
+ allRoutesActual.append(
+ route[ 'prefix' ] + "/" + route[ 'nextHop' ] )
+
+ return sorted( allRoutesActual )
+
# This method extracts all actual route intents from ONOS CLI
def extractActualRouteIntents( self, getIntentsResult ):
intents = []
@@ -207,8 +219,8 @@
intentsJsonObj = json.loads( getIntentsResult )
for intent in intentsJsonObj:
- if intent[ 'appId' ] != "org.onosproject.sdnip":
- continue
+ #if intent[ 'appId' ] != "org.onosproject.sdnip":
+ # continue
if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
and intent[ 'state' ] == 'INSTALLED':
egress = str( intent[ 'egress' ][ 'device' ] ) + ":" \
@@ -230,6 +242,26 @@
intents.append( intent )
return sorted( intents )
+ # This method calculates the MultiPointToSinglePointIntent number installed
+ def extractActualRouteIntentNum( self, getIntentsResult ):
+ intentsJsonObj = json.loads( getIntentsResult )
+ num = 0
+ for intent in intentsJsonObj:
+ if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
+ and intent[ 'state' ] == 'INSTALLED':
+ num = num + 1
+ return num
+
+ # This method calculates the PointToPointIntent number installed
+ def extractActualBgpIntentNum( self, getIntentsResult ):
+ intentsJsonObj = json.loads( getIntentsResult )
+ num = 0
+ for intent in intentsJsonObj:
+ if intent[ 'type' ] == "PointToPointIntent" \
+ and intent[ 'state' ] == 'INSTALLED':
+ num = num + 1
+ return num
+
# This method extracts all actual BGP intents from ONOS CLI
def extractActualBgpIntents( self, getIntentsResult ):
intents = []
@@ -238,8 +270,8 @@
intentsJsonObj = json.loads( getIntentsResult )
for intent in intentsJsonObj:
- if intent[ 'appId' ] != "org.onosproject.sdnip":
- continue
+ #if intent[ 'appId' ] != "org.onosproject.sdnip":
+ # continue
if intent[ 'type' ] == "PointToPointIntent" \
and "protocol=6" in str( intent[ 'selector' ] ):
ingress = str( intent[ 'ingress' ][ 'device' ] ) + ":" \
@@ -489,7 +521,7 @@
if routesAdded == numRoutes:
return main.TRUE
return main.FALSE
-
+
# Please use deleteRoutes method instead of this one!
def delRoute( self, net, numRoutes, routeRate ):
try:
@@ -562,7 +594,7 @@
child.expect( "Flow table show" )
count = 0
while True:
- i = child.expect( [ '17\d\.\d{1,3}\.\d{1,3}\.\d{1,3}',
+ i = child.expect( [ '17\d\.\d{1,3}\.\d{1,3}\.\d{1,3}',
'CLI#', pexpect.TIMEOUT ] )
if i == 0:
count = count + 1
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 44552ae..3b94ef7 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -81,7 +81,7 @@
while i == 5:
i = self.handle.expect( [
ssh_newkey,
- 'password:',
+ 'password:|Password:',
pexpect.EOF,
pexpect.TIMEOUT,
refused,
diff --git a/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py b/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py
new file mode 100644
index 0000000..3f41180
--- /dev/null
+++ b/TestON/tests/FuncIntent/Dependency/FuncIntentFunction.py
@@ -0,0 +1,967 @@
+"""
+ Wrapper functions for FuncIntent
+ This functions include Onosclidriver and Mininetclidriver driver functions
+ Author: kelvin@onlab.us
+"""
+def __init__( self ):
+ self.default = ''
+
+def hostIntent( main,
+ name,
+ host1,
+ host2,
+ host1Id="",
+ host2Id="",
+ mac1="",
+ mac2="",
+ vlan1="-1",
+ vlan2="-1",
+ sw1="",
+ sw2="",
+ expectedLink=0 ):
+ """
+ Description:
+ Verify add-host-intent
+ Steps:
+ - Discover hosts
+ - Add host intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of host intent to add eg. IPV4 | VLAN | Dualstack
+ host1 - Name of first host
+ host2 - Name of second host
+ Optional:
+ host1Id - ONOS id of the first host eg. 00:00:00:00:00:01/-1
+ host2Id - ONOS id of the second host
+ mac1 - Mac address of first host
+ mac2 - Mac address of the second host
+ vlan1 - Vlan tag of first host, defaults to -1
+ vlan2 - Vlan tag of second host, defaults to -1
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
+ """
+ import time
+
+ # Assert variables
+ assert main, "There is no main variable"
+ assert name, "variable name is empty"
+ assert host1 and host2, "You must specify hosts"
+
+ global itemName
+ itemName = name
+ h1Id = host1Id
+ h2Id = host2Id
+ h1Mac = mac1
+ h2Mac = mac2
+ vlan1 = vlan1
+ vlan2 = vlan2
+ hostNames = [ host1 , host2 ]
+ intentsId = []
+ stepResult = main.TRUE
+ pingResult = main.TRUE
+ intentResult = main.TRUE
+ removeIntentResult = main.TRUE
+ flowResult = main.TRUE
+ topoResult = main.TRUE
+ linkDownResult = main.TRUE
+ linkUpResult = main.TRUE
+
+ if main.hostsData:
+ if not h1Mac:
+ h1Mac = main.hostsData[ host1 ][ 'mac' ]
+ if not h2Mac:
+ h2Mac = main.hostsData[ host2 ][ 'mac' ]
+ if main.hostsData[ host1 ][ 'vlan' ] != '-1':
+ vlan1 = main.hostsData[ host1 ][ 'vlan' ]
+ if main.hostsData[ host2 ][ 'vlan' ] != '-1':
+ vlan2 = main.hostsData[ host2 ][ 'vlan' ]
+ if not h1Id:
+ h1Id = main.hostsData[ host1 ][ 'id' ]
+ if not h2Id:
+ h2Id = main.hostsData[ host2 ][ 'id' ]
+
+ assert h1Id and h2Id, "You must specify host IDs"
+ if not ( h1Id and h2Id ):
+ main.log.info( "There are no host IDs" )
+ return main.FALSE
+
+ # Discover hosts using arping
+ main.log.info( itemName + ": Discover host using arping" )
+ main.Mininet1.arping( host=host1 )
+ main.Mininet1.arping( host=host2 )
+ host1 = main.CLIs[ 0 ].getHost( mac=h1Mac )
+ host2 = main.CLIs[ 0 ].getHost( mac=h2Mac )
+
+ # Adding host intents
+ main.log.info( itemName + ": Adding host intents" )
+ intent1 = main.CLIs[ 0 ].addHostIntent( hostIdOne=h1Id,
+ hostIdTwo=h2Id )
+ intentsId.append( intent1 )
+ time.sleep( 5 )
+
+ # Check intents state
+ time.sleep( 30 )
+ intentResult = checkIntentState( main, intentsId )
+
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Ping hosts
+ firstPingResult = pingallHosts( main, hostNames )
+ if not firstPingResult:
+ main.log.debug( "First ping failed, there must be" +
+ " something wrong with ONOS performance" )
+
+ # Ping hosts again...
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ time.sleep( 5 )
+
+ # Test rerouting if these variables exist
+ if sw1 and sw2 and expectedLink:
+ # link down
+ linkDownResult = link( main, sw1, sw2, "down" )
+ intentResult = intentResult and checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, expectedLink )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
+ # link up
+ linkUpResult = link( main, sw1, sw2, "up" )
+ time.sleep( 5 )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, main.numLinks )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
+ stepResult = pingResult and linkDownResult and linkUpResult \
+ and intentResult and removeIntentResult
+
+ return stepResult
+
+def pointIntent( main,
+ name,
+ host1,
+ host2,
+ deviceId1="",
+ deviceId2="",
+ port1="",
+ port2="",
+ ethType="",
+ mac1="",
+ mac2="",
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ip1="",
+ ip2="",
+ tcp1="",
+ tcp2="",
+ sw1="",
+ sw2="",
+ expectedLink=0 ):
+
+ """
+ Description:
+ Verify add-point-intent
+ Steps:
+ - Get device ids | ports
+ - Add point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+ host1 - Name of first host
+ host2 - Name of second host
+ Optional:
+ deviceId1 - ONOS device id of the first switch, the same as the
+ location of the first host eg. of:0000000000000001/1,
+ located at device 1 port 1
+ deviceId2 - ONOS device id of the second switch
+ port1 - The port number where the first host is attached
+ port2 - The port number where the second host is attached
+ ethType - Ethernet type eg. IPV4, IPV6
+ mac1 - Mac address of first host
+ mac2 - Mac address of the second host
+ bandwidth - Bandwidth capacity
+ lambdaAlloc - Allocate lambda, defaults to False
+ ipProto - IP protocol
+ ip1 - IP address of first host
+ ip2 - IP address of second host
+ tcp1 - TCP port of first host
+ tcp2 - TCP port of second host
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
+ """
+
+ import time
+ assert main, "There is no main variable"
+ assert name, "variable name is empty"
+ assert host1 and host2, "You must specify hosts"
+
+ global itemName
+ itemName = name
+ host1 = host1
+ host2 = host2
+ hostNames = [ host1, host2 ]
+ intentsId = []
+
+ pingResult = main.TRUE
+ intentResult = main.TRUE
+ removeIntentResult = main.TRUE
+ flowResult = main.TRUE
+ topoResult = main.TRUE
+ linkDownResult = main.TRUE
+ linkUpResult = main.TRUE
+
+ # Adding bidirectional point intents
+ main.log.info( itemName + ": Adding point intents" )
+ intent1 = main.CLIs[ 0 ].addPointIntent( ingressDevice=deviceId1,
+ egressDevice=deviceId2,
+ portIngress=port1,
+ portEgress=port2,
+ ethType=ethType,
+ ethSrc=mac1,
+ ethDst=mac2,
+ bandwidth=bandwidth,
+ lambdaAlloc=lambdaAlloc,
+ ipProto=ipProto,
+ ipSrc=ip1,
+ ipDst=ip2,
+ tcpSrc=tcp1,
+ tcpDst=tcp2 )
+
+ intentsId.append( intent1 )
+ time.sleep( 5 )
+ intent2 = main.CLIs[ 0 ].addPointIntent( ingressDevice=deviceId2,
+ egressDevice=deviceId1,
+ portIngress=port2,
+ portEgress=port1,
+ ethType=ethType,
+ ethSrc=mac2,
+ ethDst=mac1,
+ bandwidth=bandwidth,
+ lambdaAlloc=lambdaAlloc,
+ ipProto=ipProto,
+ ipSrc=ip2,
+ ipDst=ip1,
+ tcpSrc=tcp2,
+ tcpDst=tcp1 )
+ intentsId.append( intent2 )
+
+ # Check intents state
+ time.sleep( 30 )
+ intentResult = checkIntentState( main, intentsId )
+
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Ping hosts
+ firstPingResult = pingallHosts( main, hostNames )
+ if not firstPingResult:
+ main.log.debug( "First ping failed, there must be" +
+ " something wrong with ONOS performance" )
+
+ # Ping hosts again...
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ time.sleep( 5 )
+
+ # Test rerouting if these variables exist
+ if sw1 and sw2 and expectedLink:
+ # link down
+ linkDownResult = link( main, sw1, sw2, "down" )
+ intentResult = intentResult and checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, expectedLink )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
+ # link up
+ linkUpResult = link( main, sw1, sw2, "up" )
+ time.sleep( 5 )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, main.numLinks )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
+ stepResult = pingResult and linkDownResult and linkUpResult \
+ and intentResult and removeIntentResult
+
+ return stepResult
+
+def singleToMultiIntent( main,
+ name,
+ hostNames,
+ devices="",
+ ports=None,
+ ethType="",
+ macs=None,
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ipAddresses="",
+ tcp="",
+ sw1="",
+ sw2="",
+ expectedLink=0 ):
+ """
+ Verify Single to Multi Point intents
+ NOTE:If main.hostsData is not defined, variables data should be passed in the
+ same order index wise. All devices in the list should have the same
+ format, either all the devices have its port or it doesn't.
+ eg. hostName = [ 'h1', 'h2' ,.. ]
+ devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
+ ports = [ '1', '1', ..]
+ ...
+ Description:
+ Verify add-single-to-multi-intent iterates through the list of given
+ host | devices and add intents
+ Steps:
+ - Get device ids | ports
+ - Add single to multi point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+ hostNames - List of host names
+ Optional:
+ devices - List of device ids in the same order as the hosts
+ in hostNames
+ ports - List of port numbers in the same order as the device in
+ devices
+ ethType - Ethernet type eg. IPV4, IPV6
+ macs - List of hosts mac address in the same order as the hosts in
+ hostNames
+ bandwidth - Bandwidth capacity
+ lambdaAlloc - Allocate lambda, defaults to False
+ ipProto - IP protocol
+ ipAddresses - IP addresses of host in the same order as the hosts in
+ hostNames
+ tcp - TCP ports in the same order as the hosts in hostNames
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
+ """
+
+ import time
+ import copy
+ assert main, "There is no main variable"
+ assert hostNames, "You must specify hosts"
+ assert devices or main.hostsData, "You must specify devices"
+
+ global itemName
+ itemName = name
+ tempHostsData = {}
+ intentsId = []
+
+ macsDict = {}
+ ipDict = {}
+ if hostNames and devices:
+ if len( hostNames ) != len( devices ):
+ main.log.debug( "hosts and devices does not have the same length" )
+ #print "len hostNames = ", len( hostNames )
+ #print "len devices = ", len( devices )
+ return main.FALSE
+ if ports:
+ if len( ports ) != len( devices ):
+ main.log.error( "Ports and devices does " +
+ "not have the same length" )
+ #print "len devices = ", len( devices )
+ #print "len ports = ", len( ports )
+ return main.FALSE
+ for i in range( len( devices ) ):
+ macsDict[ devices[ i ] ] = macs[ i ]
+ else:
+ main.log.info( "Device Ports are not specified" )
+ elif hostNames and not devices and main.hostsData:
+ devices = []
+ main.log.info( "singleToMultiIntent function is using main.hostsData" )
+ for host in hostNames:
+ devices.append( main.hostsData.get( host ).get( 'location' ) )
+ macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'mac' )
+ ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'ipAddresses' )
+ #print main.hostsData
+
+ #print 'host names = ', hostNames
+ #print 'devices = ', devices
+ #print "macsDict = ", macsDict
+
+ pingResult = main.TRUE
+ intentResult = main.TRUE
+ removeIntentResult = main.TRUE
+ flowResult = main.TRUE
+ topoResult = main.TRUE
+ linkDownResult = main.TRUE
+ linkUpResult = main.TRUE
+
+ devicesCopy = copy.copy( devices )
+ if ports:
+ portsCopy = copy.copy( ports )
+ main.log.info( itemName + ": Adding single point to multi point intents" )
+ # Adding bidirectional point intents
+ for i in range( len( devices ) ):
+ ingressDevice = devicesCopy[ i ]
+ egressDeviceList = copy.copy( devicesCopy )
+ egressDeviceList.remove( ingressDevice )
+ if ports:
+ portIngress = portsCopy[ i ]
+ portEgressList = copy.copy( portsCopy )
+ del portEgressList[ i ]
+ else:
+ portIngress = ""
+ portEgressList = None
+ if not macsDict:
+ srcMac = ""
+ else:
+ srcMac = macsDict[ ingressDevice ]
+ if srcMac == None:
+ main.log.debug( "There is no MAC in device - " + ingressDevice )
+ srcMac = ""
+
+ intentsId.append( main.CLIs[ 0 ].addSinglepointToMultipointIntent(
+ ingressDevice=ingressDevice,
+ egressDeviceList=egressDeviceList,
+ portIngress=portIngress,
+ portEgressList=portEgressList,
+ ethType=ethType,
+ ethSrc=srcMac,
+ bandwidth=bandwidth,
+ lambdaAlloc=lambdaAlloc,
+ ipProto=ipProto,
+ ipSrc="",
+ ipDst="",
+ tcpSrc="",
+ tcpDst="" ) )
+
+ pingResult = pingallHosts( main, hostNames )
+
+ # Check intents state
+ time.sleep( 30 )
+ intentResult = checkIntentState( main, intentsId )
+
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ # Ping hosts again...
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ time.sleep( 5 )
+
+ # Test rerouting if these variables exist
+ if sw1 and sw2 and expectedLink:
+ # link down
+ linkDownResult = link( main, sw1, sw2, "down" )
+ intentResult = intentResult and checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, expectedLink )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
+ # link up
+ linkUpResult = link( main, sw1, sw2, "up" )
+ time.sleep( 5 )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, main.numLinks )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
+ stepResult = pingResult and linkDownResult and linkUpResult \
+ and intentResult and removeIntentResult
+
+ return stepResult
+
+def multiToSingleIntent( main,
+ name,
+ hostNames,
+ devices="",
+ ports=None,
+ ethType="",
+ macs=None,
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ipAddresses="",
+ tcp="",
+ sw1="",
+ sw2="",
+ expectedLink=0 ):
+ """
+ Verify Single to Multi Point intents
+ NOTE:If main.hostsData is not defined, variables data should be passed in the
+ same order index wise. All devices in the list should have the same
+ format, either all the devices have its port or it doesn't.
+ eg. hostName = [ 'h1', 'h2' ,.. ]
+ devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
+ ports = [ '1', '1', ..]
+ ...
+ Description:
+ Verify add-multi-to-single-intent
+ Steps:
+ - Get device ids | ports
+ - Add single to multi point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ Required:
+ name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
+ hostNames - List of host names
+ Optional:
+ devices - List of device ids in the same order as the hosts
+ in hostNames
+ ports - List of port numbers in the same order as the device in
+ devices
+ ethType - Ethernet type eg. IPV4, IPV6
+ macs - List of hosts mac address in the same order as the hosts in
+ hostNames
+ bandwidth - Bandwidth capacity
+ lambdaAlloc - Allocate lambda, defaults to False
+ ipProto - IP protocol
+ ipAddresses - IP addresses of host in the same order as the hosts in
+ hostNames
+ tcp - TCP ports in the same order as the hosts in hostNames
+ sw1 - First switch to bring down & up for rerouting purpose
+ sw2 - Second switch to bring down & up for rerouting purpose
+ expectedLink - Expected link when the switches are down, it should
+ be two links lower than the links before the two
+ switches are down
+ """
+
+ import time
+ import copy
+ assert main, "There is no main variable"
+ assert hostNames, "You must specify hosts"
+ assert devices or main.hostsData, "You must specify devices"
+
+ global itemName
+ itemName = name
+ tempHostsData = {}
+ intentsId = []
+
+ macsDict = {}
+ ipDict = {}
+ if hostNames and devices:
+ if len( hostNames ) != len( devices ):
+ main.log.debug( "hosts and devices does not have the same length" )
+ #print "len hostNames = ", len( hostNames )
+ #print "len devices = ", len( devices )
+ return main.FALSE
+ if ports:
+ if len( ports ) != len( devices ):
+ main.log.error( "Ports and devices does " +
+ "not have the same length" )
+ #print "len devices = ", len( devices )
+ #print "len ports = ", len( ports )
+ return main.FALSE
+ for i in range( len( devices ) ):
+ macsDict[ devices[ i ] ] = macs[ i ]
+ else:
+ main.log.info( "Device Ports are not specified" )
+ elif hostNames and not devices and main.hostsData:
+ devices = []
+ main.log.info( "singleToMultiIntent function is using main.hostsData" )
+ for host in hostNames:
+ devices.append( main.hostsData.get( host ).get( 'location' ) )
+ macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'mac' )
+ ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
+ main.hostsData.get( host ).get( 'ipAddresses' )
+ #print main.hostsData
+
+ #print 'host names = ', hostNames
+ #print 'devices = ', devices
+ #print "macsDict = ", macsDict
+
+ pingResult = main.TRUE
+ intentResult = main.TRUE
+ removeIntentResult = main.TRUE
+ flowResult = main.TRUE
+ topoResult = main.TRUE
+ linkDownResult = main.TRUE
+ linkUpResult = main.TRUE
+
+ devicesCopy = copy.copy( devices )
+ if ports:
+ portsCopy = copy.copy( ports )
+ main.log.info( itemName + ": Adding single point to multi point intents" )
+ # Adding bidirectional point intents
+ for i in range( len( devices ) ):
+ egressDevice = devicesCopy[ i ]
+ ingressDeviceList = copy.copy( devicesCopy )
+ ingressDeviceList.remove( egressDevice )
+ if ports:
+ portEgress = portsCopy[ i ]
+ portIngressList = copy.copy( portsCopy )
+ del portIngressList[ i ]
+ else:
+ portEgress = ""
+ portIngressList = None
+ if not macsDict:
+ dstMac = ""
+ else:
+ dstMac = macsDict[ egressDevice ]
+ if dstMac == None:
+ main.log.debug( "There is no MAC in device - " + egressDevice )
+ dstMac = ""
+
+ intentsId.append( main.CLIs[ 0 ].addMultipointToSinglepointIntent(
+ ingressDeviceList=ingressDeviceList,
+ egressDevice=egressDevice,
+ portIngressList=portIngressList,
+ portEgress=portEgress,
+ ethType=ethType,
+ ethDst=dstMac,
+ bandwidth=bandwidth,
+ lambdaAlloc=lambdaAlloc,
+ ipProto=ipProto,
+ ipSrc="",
+ ipDst="",
+ tcpSrc="",
+ tcpDst="" ) )
+
+ pingResult = pingallHosts( main, hostNames )
+
+ # Check intents state
+ time.sleep( 30 )
+ intentResult = checkIntentState( main, intentsId )
+
+ # Check intents state again if first check fails...
+ if not intentResult:
+ intentResult = checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ # Ping hosts again...
+ pingResult = pingResult and pingallHosts( main, hostNames )
+ time.sleep( 5 )
+
+ # Test rerouting if these variables exist
+ if sw1 and sw2 and expectedLink:
+ # link down
+ linkDownResult = link( main, sw1, sw2, "down" )
+ intentResult = intentResult and checkIntentState( main, intentsId )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, expectedLink )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link down
+ if linkDownResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link down" )
+ else:
+ main.log.error( itemName + ": Failed to bring link down" )
+
+ # link up
+ linkUpResult = link( main, sw1, sw2, "up" )
+ time.sleep( 5 )
+
+ # Verify flows
+ checkFlowsState( main )
+
+ # Check OnosTopology
+ topoResult = checkTopology( main, main.numLinks )
+
+ # Ping hosts
+ pingResult = pingResult and pingallHosts( main, hostNames )
+
+ intentResult = checkIntentState( main, intentsId )
+
+ # Checks ONOS state in link up
+ if linkUpResult and topoResult and pingResult and intentResult:
+ main.log.info( itemName + ": Successfully brought link back up" )
+ else:
+ main.log.error( itemName + ": Failed to bring link back up" )
+
+ # Remove all intents
+ removeIntentResult = removeAllIntents( main, intentsId )
+
+ stepResult = pingResult and linkDownResult and linkUpResult \
+ and intentResult and removeIntentResult
+
+ return stepResult
+
+def pingallHosts( main, hostList, pingType="ipv4" ):
+ # Ping all host in the hosts list variable
+ print "Pinging : ", hostList
+ pingResult = main.TRUE
+ pingResult = main.Mininet1.pingallHosts( hostList, pingType )
+ return pingResult
+
+def getHostsData( main ):
+ """
+ Use fwd app and pingall to discover all the hosts
+ """
+ import json
+ activateResult = main.TRUE
+ appCheck = main.TRUE
+ getDataResult = main.TRUE
+ main.log.info( "Activating reactive forwarding app " )
+ activateResult = main.CLIs[ 0 ].activateApp( "org.onosproject.fwd" )
+
+ for i in range( main.numCtrls ):
+ appCheck = appCheck and main.CLIs[ i ].appToIDCheck()
+ if appCheck != main.TRUE:
+ main.log.warn( main.CLIs[ i ].apps() )
+ main.log.warn( main.CLIs[ i ].appIDs() )
+
+ pingResult = main.Mininet1.pingall()
+ hostsJson = json.loads( main.CLIs[ 0 ].hosts() )
+ hosts = main.Mininet1.getHosts()
+ for host in hosts:
+ main.hostsData[ host ] = {}
+ main.hostsData[ host ][ 'mac' ] = \
+ main.Mininet1.getMacAddress( host ).upper()
+ for hostj in hostsJson:
+ if main.hostsData[ host ][ 'mac' ] == hostj[ 'mac' ]:
+ main.hostsData[ host ][ 'id' ] = hostj[ 'id' ]
+ main.hostsData[ host ][ 'vlan' ] = hostj[ 'vlan' ]
+ main.hostsData[ host ][ 'location' ] = \
+ hostj[ 'location' ][ 'elementId' ] + '/' + \
+ hostj[ 'location' ][ 'port' ]
+ main.hostsData[ host ][ 'ipAddresses' ] = hostj[ 'ipAddresses' ]
+
+ main.log.info( "Deactivating reactive forwarding app " )
+ deactivateResult = main.CLIs[ 0 ].deactivateApp( "org.onosproject.fwd" )
+ if activateResult and deactivateResult and main.hostsData:
+ main.log.info( "Successfully used fwd app to discover hosts " )
+ getDataResult = main.TRUE
+ else:
+ main.log.info( "Failed to use fwd app to discover hosts " )
+ getDataResult = main.FALSE
+
+ print main.hostsData
+
+ return getDataResult
+
+def checkTopology( main, expectedLink ):
+ statusResult = main.TRUE
+ # Check onos topology
+ main.log.info( itemName + ": Checking ONOS topology " )
+
+ for i in range( main.numCtrls ):
+ topologyResult = main.CLIs[ i ].topology()
+ statusResult = main.ONOSbench.checkStatus( topologyResult,
+ main.numSwitch,
+ expectedLink )\
+ and statusResult
+ if not statusResult:
+ main.log.error( itemName + ": Topology mismatch" )
+ else:
+ main.log.info( itemName + ": Topology match" )
+ return statusResult
+
+def checkIntentState( main, intentsId ):
+
+ intentResult = main.TRUE
+
+ main.log.info( itemName + ": Checking intents state" )
+ for i in range( main.numCtrls ):
+ intentResult = intentResult and \
+ main.CLIs[ i ].checkIntentState( intentsId=intentsId )
+
+ return intentResult
+
+def checkFlowsState( main ):
+
+ main.log.info( itemName + ": Check flows state" )
+ checkFlowsResult = main.CLIs[ 0 ].checkFlowsState()
+ return checkFlowsResult
+
+def link( main, sw1, sw2, option):
+
+ # link down
+ main.log.info( itemName + ": Bring link " + option + "between " +
+ sw1 + " and " + sw2 )
+ linkResult = main.Mininet1.link( end1=sw1, end2=sw2, option=option )
+ return linkResult
+
+def removeAllIntents( main, intentsId ):
+ """
+ Remove all intents in the intentsId
+ """
+ import time
+ intentsRemaining = []
+ removeIntentResult = main.TRUE
+ # Remove intents
+ for intent in intentsId:
+ main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
+
+ time.sleep( 5 )
+ # Checks if there is remaining intents using intents()
+ intentsRemaining = main.CLIs[ 0 ].intents( jsonFormat=False )
+ # If there is remianing intents then remove intents should fail
+
+ if intentsRemaining:
+ main.log.info( itemName + ": There are " +
+ str( len( intentsRemaining ) ) + " intents remaining, "
+ + "failed to remove all the intents " )
+ removeIntentResult = main.FALSE
+ main.log.info( intentsRemaining )
+ else:
+ main.log.info( itemName + ": There are no intents remaining, " +
+ "successfully removed all the intents." )
+ removeIntentResult = main.TRUE
+ return removeIntentResult
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/FuncIntent/Dependency/__init__.py
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/FuncIntent/Dependency/__init__.py
diff --git a/TestON/tests/FuncIntent/FuncIntent.py b/TestON/tests/FuncIntent/FuncIntent.py
new file mode 100644
index 0000000..d5dc2a7
--- /dev/null
+++ b/TestON/tests/FuncIntent/FuncIntent.py
@@ -0,0 +1,580 @@
+
+# Testing the basic functionality of ONOS Next
+# For sanity and driver functionality excercises only.
+
+import time
+import json
+
+class FuncIntent:
+
+ def __init__( self ):
+ self.default = ''
+
+ def CASE10( self, main ):
+ import time
+ import os
+ import imp
+ """
+ Startup sequence:
+ cell <name>
+ onos-verify-cell
+ onos-remove-raft-log
+ git pull
+ mvn clean install
+ onos-package
+ onos-install -f
+ onos-wait-for-start
+ """
+ global init
+ global globalONOSip
+ try:
+ if type(init) is not bool:
+ init = False
+ except NameError:
+ init = False
+
+ main.wrapper = imp.load_source( 'FuncIntentFunction',
+ '/home/admin/ONLabTest/TestON/tests/' +
+ 'FuncIntent/Dependency/' +
+ 'FuncIntentFunction.py' )
+ #Local variables
+ cellName = main.params[ 'ENV' ][ 'cellName' ]
+ apps = main.params[ 'ENV' ][ 'cellApps' ]
+ gitBranch = main.params[ 'GIT' ][ 'branch' ]
+ benchIp = os.environ[ 'OCN' ]
+ benchUser = main.params[ 'BENCH' ][ 'user' ]
+ topology = main.params[ 'MININET' ][ 'topo' ]
+ main.numSwitch = int( main.params[ 'MININET' ][ 'switch' ] )
+ main.numLinks = int( main.params[ 'MININET' ][ 'links' ] )
+ main.numCtrls = main.params[ 'CTRL' ][ 'num' ]
+ main.ONOSport = []
+ main.hostsData = {}
+ PULLCODE = False
+ if main.params[ 'GIT' ][ 'pull' ] == 'True':
+ PULLCODE = True
+ main.case( "Setting up test environment" )
+ main.CLIs = []
+ for i in range( 1, int( main.numCtrls ) + 1 ):
+ main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ main.ONOSport.append( main.params[ 'CTRL' ][ 'port' + str( i ) ] )
+
+ # -- INIT SECTION, ONLY RUNS ONCE -- #
+ if init == False:
+ init = True
+
+ main.scale = ( main.params[ 'SCALE' ] ).split( "," )
+ main.numCtrls = int( main.scale[ 0 ] )
+
+ if PULLCODE:
+ main.step( "Git checkout and pull " + gitBranch )
+ main.ONOSbench.gitCheckout( gitBranch )
+ gitPullResult = main.ONOSbench.gitPull()
+ if gitPullResult == main.ERROR:
+ main.log.error( "Error pulling git branch" )
+ main.step( "Using mvn clean & install" )
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ stepResult = cleanInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully compiled " +
+ "latest ONOS",
+ onfail="Failed to compile " +
+ "latest ONOS" )
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
+
+ globalONOSip = main.ONOSbench.getOnosIps()
+
+ maxNodes = ( len(globalONOSip) - 2 )
+
+ main.numCtrls = int( main.scale[ 0 ] )
+ main.scale.remove( main.scale[ 0 ] )
+
+ main.ONOSip = []
+ for i in range( maxNodes ):
+ main.ONOSip.append( globalONOSip[i] )
+
+ #kill off all onos processes
+ main.log.info( "Safety check, killing all ONOS processes" +
+ " before initiating enviornment setup" )
+ for i in range(maxNodes):
+ main.ONOSbench.onosDie( globalONOSip[ i ] )
+
+ print "NODE COUNT = ", main.numCtrls
+ main.log.info( "Creating cell file" )
+ cellIp = []
+ for i in range( main.numCtrls ):
+ cellIp.append( str( main.ONOSip[ i ] ) )
+ print cellIp
+ main.ONOSbench.createCellFile( benchIp, cellName, "",
+ str( apps ), *cellIp )
+
+ main.step( "Apply cell to environment" )
+ cellResult = main.ONOSbench.setCell( cellName )
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " + \
+ "environment",
+ onfail="Failed to apply cell to environment " )
+
+ main.step( "Creating ONOS package" )
+ packageResult = main.ONOSbench.onosPackage()
+ stepResult = packageResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully created ONOS package",
+ onfail="Failed to create ONOS package" )
+
+ main.step( "Uninstalling ONOS package" )
+ onosUninstallResult = main.TRUE
+ for i in range( main.numCtrls ):
+ onosUninstallResult = onosUninstallResult and \
+ main.ONOSbench.onosUninstall( nodeIp=main.ONOSip[ i ] )
+ stepResult = onosUninstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully uninstalled ONOS package",
+ onfail="Failed to uninstall ONOS package" )
+ time.sleep( 5 )
+ main.step( "Installing ONOS package" )
+ onosInstallResult = main.TRUE
+ for i in range( main.numCtrls ):
+ onosInstallResult = onosInstallResult and \
+ main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+ stepResult = onosInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully installed ONOS package",
+ onfail="Failed to install ONOS package" )
+
+ time.sleep( 20 )
+ main.step( "Starting ONOS service" )
+ stopResult = main.TRUE
+ startResult = main.TRUE
+ onosIsUp = main.TRUE
+ for i in range( main.numCtrls ):
+ onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+ if onosIsUp == main.TRUE:
+ main.log.report( "ONOS instance is up and ready" )
+ else:
+ main.log.report( "ONOS instance may not be up, stop and " +
+ "start ONOS again " )
+ for i in range( main.numCtrls ):
+ stopResult = stopResult and \
+ main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ for i in range( main.numCtrls ):
+ startResult = startResult and \
+ main.ONOSbench.onosStart( main.ONOSip[ i ] )
+ stepResult = onosIsUp and stopResult and startResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+
+ main.step( "Start ONOS cli" )
+ cliResult = main.TRUE
+ for i in range( main.numCtrls ):
+ cliResult = cliResult and \
+ main.CLIs[i].startOnosCli( main.ONOSip[ i ] )
+ stepResult = cliResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully start ONOS cli",
+ onfail="Failed to start ONOS cli" )
+
+ def CASE9( self, main ):
+ '''
+ Report errors/warnings/exceptions
+ '''
+ main.log.info( "Error report: \n" )
+ main.ONOSbench.logReport( globalONOSip[0],
+ [ "INFO", "FOLLOWER", "WARN", "flow", "ERROR" , "Except" ],
+ "s" )
+ #main.ONOSbench.logReport( globalONOSip[1], [ "INFO" ], "d" )
+
+ def CASE11( self, main ):
+ """
+ Start mininet
+ """
+ main.log.report( "Start Mininet topology" )
+ main.log.case( "Start Mininet topology" )
+
+ main.step( "Starting Mininet Topology" )
+ topoResult = main.Mininet1.startNet( topoFile=topology )
+ stepResult = topoResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully loaded topology",
+ onfail="Failed to load topology" )
+ # Exit if topology did not load properly
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
+ def CASE12( self, main ):
+ """
+ Assign mastership to controllers
+ """
+ import re
+
+ main.case( "Assign switches to controllers" )
+ main.step( "Assigning switches to controllers" )
+ assignResult = main.TRUE
+ for i in range( 1, ( main.numSwitch + 1 ) ):
+ main.Mininet1.assignSwController( sw=str( i ),
+ count=1,
+ ip1=main.ONOSip[ 0 ],
+ port1=main.ONOSport[ 0 ] )
+ for i in range( 1, ( main.numSwitch + 1 ) ):
+ response = main.Mininet1.getSwController( "s" + str( i ) )
+ print( "Response is " + str( response ) )
+ if re.search( "tcp:" + main.ONOSip[ 0 ], response ):
+ assignResult = assignResult and main.TRUE
+ else:
+ assignResult = main.FALSE
+ stepResult = assignResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully assigned switches" +
+ "to controller",
+ onfail="Failed to assign switches to " +
+ "controller" )
+ def CASE13( self, main ):
+ """
+ Discover all hosts and store its data to a dictionary
+ """
+ main.case( "Discover all hosts" )
+
+ stepResult = main.TRUE
+ main.step( "Discover all hosts using pingall " )
+ stepResult = main.wrapper.getHostsData( main )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully discovered hosts",
+ onfail="Failed to discover hosts" )
+
+ def CASE1001( self, main ):
+ """
+ Add host intents between 2 host:
+ - Discover hosts
+ - Add host intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ """
+ import time
+ import json
+ import re
+
+ # Assert variables - These variable's name|format must be followed
+ # if you want to use the wrapper function
+ assert main, "There is no main"
+ assert main.CLIs, "There is no main.CLIs"
+ assert main.Mininet1, "Mininet handle should be named Mininet1"
+ assert main.numSwitch, "Placed the total number of switch topology in \
+ main.numSwitch"
+
+ main.case( "Add host intents between 2 host" )
+
+ stepResult = main.TRUE
+ main.step( "IPV4: Add host intents between h1 and h9" )
+ stepResult = main.wrapper.hostIntent( main,
+ name='IPV4',
+ host1='h1',
+ host2='h9',
+ host1Id='00:00:00:00:00:01/-1',
+ host2Id='00:00:00:00:00:09/-1',
+ sw1='s5',
+ sw2='s2',
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4: Add host intent successful",
+ onfail="IPV4: Add host intent failed" )
+ stepResult = main.TRUE
+
+ main.step( "DUALSTACK1: Add host intents between h3 and h11" )
+ stepResult = main.wrapper.hostIntent( main,
+ name='DUALSTACK',
+ host1='h3',
+ host2='h11',
+ host1Id='00:00:00:00:00:03/-1',
+ host2Id='00:00:00:00:00:0B/-1',
+ sw1='s5',
+ sw2='s2',
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="DUALSTACK1: Add host intent" +
+ " successful",
+ onfail="DUALSTACK1: Add host intent failed" )
+
+ stepResult = main.TRUE
+ main.step( "DUALSTACK2: Add host intents between h1 and h9" )
+ stepResult = main.wrapper.hostIntent( main,
+ name='DUALSTACK2',
+ host1='h1',
+ host2='h11',
+ sw1='s5',
+ sw2='s2',
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="DUALSTACK2: Add host intent" +
+ " successful",
+ onfail="DUALSTACK2: Add host intent failed" )
+
+ def CASE1002( self, main ):
+ """
+ Add point intents between 2 hosts:
+ - Get device ids | ports
+ - Add point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ """
+ import time
+ import json
+ import re
+
+ # Assert variables - These variable's name|format must be followed
+ # if you want to use the wrapper function
+ assert main, "There is no main"
+ assert main.CLIs, "There is no main.CLIs"
+ assert main.Mininet1, "Mininet handle should be named Mininet1"
+ assert main.numSwitch, "Placed the total number of switch topology in \
+ main.numSwitch"
+
+ main.case( "Add point intents between 2 devices" )
+
+ stepResult = main.TRUE
+ main.step( "IPV4: Add point intents between h1 and h9" )
+ stepResult = main.wrapper.pointIntent(
+ main,
+ name="IPV4",
+ host1="h1",
+ host2="h9",
+ deviceId1="of:0000000000000005/1",
+ deviceId2="of:0000000000000006/1",
+ port1="",
+ port2="",
+ ethType="IPV4",
+ mac1="00:00:00:00:00:01",
+ mac2="00:00:00:00:00:09",
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ip1="",
+ ip2="",
+ tcp1="",
+ tcp2="",
+ sw1="s5",
+ sw2="s2",
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4: Add point intent successful",
+ onfail="IPV4: Add point intent failed" )
+
+ stepResult = main.TRUE
+ main.step( "DUALSTACK1: Add point intents between h1 and h9" )
+ stepResult = main.wrapper.pointIntent(
+ main,
+ name="DUALSTACK1",
+ host1="h3",
+ host2="h11",
+ deviceId1="of:0000000000000005",
+ deviceId2="of:0000000000000006",
+ port1="3",
+ port2="3",
+ ethType="IPV4",
+ mac1="00:00:00:00:00:03",
+ mac2="00:00:00:00:00:0B",
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ip1="",
+ ip2="",
+ tcp1="",
+ tcp2="",
+ sw1="s5",
+ sw2="s2",
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="DUALSTACK1: Add point intent" +
+ " successful",
+ onfail="DUALSTACK1: Add point intent failed" )
+
+ def CASE1003( self, main ):
+ """
+ Add single point to multi point intents
+ - Get device ids
+ - Add single point to multi point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ """
+ assert main, "There is no main"
+ assert main.CLIs, "There is no main.CLIs"
+ assert main.Mininet1, "Mininet handle should be named Mininet1"
+ assert main.numSwitch, "Placed the total number of switch topology in \
+ main.numSwitch"
+
+ main.case( "Add single point to multi point intents between devices" )
+
+ stepResult = main.TRUE
+ main.step( "IPV4: Add single point to multi point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ devices = [ 'of:0000000000000005/8', 'of:0000000000000006/8', \
+ 'of:0000000000000007/8' ]
+ macs = [ '00:00:00:00:00:08', '00:00:00:00:00:10', '00:00:00:00:00:18' ]
+ stepResult = main.wrapper.singleToMultiIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ devices=devices,
+ ports=None,
+ ethType="IPV4",
+ macs=macs,
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ipAddresses="",
+ tcp="",
+ sw1="s5",
+ sw2="s2",
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4: Successfully added single point"
+ + " to multi point intents",
+ onfail="IPV4: Failed to add single point" +
+ " to multi point intents" )
+
+ main.step( "IPV4_2: Add single point to multi point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ stepResult = main.wrapper.singleToMultiIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ ethType="IPV4",
+ lambdaAlloc=False )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4_2: Successfully added single point"
+ + " to multi point intents",
+ onfail="IPV4_2: Failed to add single point" +
+ " to multi point intents" )
+ def CASE1004( self, main ):
+ """
+ Add multi point to single point intents
+ - Get device ids
+ - Add multi point to single point intents
+ - Check intents
+ - Verify flows
+ - Ping hosts
+ - Reroute
+ - Link down
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Link up
+ - Verify flows
+ - Check topology
+ - Ping hosts
+ - Remove intents
+ """
+ assert main, "There is no main"
+ assert main.CLIs, "There is no main.CLIs"
+ assert main.Mininet1, "Mininet handle should be named Mininet1"
+ assert main.numSwitch, "Placed the total number of switch topology in \
+ main.numSwitch"
+
+ main.case( "Add multi point to single point intents between devices" )
+
+ stepResult = main.TRUE
+ main.step( "IPV4: Add multi point to single point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ devices = [ 'of:0000000000000005/8', 'of:0000000000000006/8', \
+ 'of:0000000000000007/8' ]
+ macs = [ '00:00:00:00:00:08', '00:00:00:00:00:10', '00:00:00:00:00:18' ]
+ stepResult = main.wrapper.multiToSingleIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ devices=devices,
+ ports=None,
+ ethType="IPV4",
+ macs=macs,
+ bandwidth="",
+ lambdaAlloc=False,
+ ipProto="",
+ ipAddresses="",
+ tcp="",
+ sw1="s5",
+ sw2="s2",
+ expectedLink=18 )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4: Successfully added multi point"
+ + " to single point intents",
+ onfail="IPV4: Failed to add multi point" +
+ " to single point intents" )
+
+ main.step( "IPV4_2: Add multi point to single point intents" )
+ hostNames = [ 'h8', 'h16', 'h24' ]
+ stepResult = main.wrapper.multiToSingleIntent(
+ main,
+ name="IPV4",
+ hostNames=hostNames,
+ ethType="IPV4",
+ lambdaAlloc=False )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="IPV4_2: Successfully added multi point"
+ + " to single point intents",
+ onfail="IPV4_2: Failed to add multi point" +
+ " to single point intents" )
diff --git a/TestON/tests/FuncIntent/FuncIntent.topo b/TestON/tests/FuncIntent/FuncIntent.topo
new file mode 100755
index 0000000..e6613de
--- /dev/null
+++ b/TestON/tests/FuncIntent/FuncIntent.topo
@@ -0,0 +1,52 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ <home>~/ONOS</home>
+ </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli3>
+
+ <Mininet1>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>MininetCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </Mininet1>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/FuncIntent/__init__.py
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/FuncIntent/__init__.py
diff --git a/TestON/tests/FuncPlatform/Dependency/App.py b/TestON/tests/FuncPlatform/Dependency/App.py
new file mode 100644
index 0000000..5410c32
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/App.py
@@ -0,0 +1,68 @@
+"""
+Methods related to application interaction
+
+"""
+
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def activate( apps, nodeToActivateFrom=0 ):
+ """
+ Activate specified applications from node specified
+
+ Ex) apps = ['metrics', 'fwd']
+ nodeToActivateFrom = range( 0, nodes )
+ """
+ if isinstance( apps, ( int, basestring ) ):
+ main.log.error( 'Please pass in a list of strings for args' )
+ return main.FALSE
+
+ if not isinstance( nodeToActivateFrom, ( int ) ) or \
+ nodeToActivateFrom < 0:
+ main.log.error( 'Incorrect node specified' )
+ return main.FALSE
+
+ # TODO: Start log capture and listen for exceptions
+ # and errors. Also investigate possible keywords
+ # to listen for when activating applications
+
+
+ for app in apps:
+ # Check if app str in appList is in the main scope
+ # definition main.appList
+ if app not in main.appList:
+ main.log.error( 'Invalid app name given' )
+ return main.FALSE
+
+ try:
+ # NOTE: assumes node 1 is always activating application
+ appOutput = main.CLIs[nodeToActivateFrom].activateApp(
+ main.appList[app] )
+ except KeyError:
+ main.log.error( 'There was an error with the key '+
+ str(app) + '. Check the appList dictionary' )
+ return main.FALSE
+ except Exception:
+ main.log.error( 'Uncaught exception error while ' +
+ 'activating applications: ' + str(app) )
+ return main.FALSE
+
+ return main.TRUE
+
+def deactivate( apps, nodeToDeactivateFrom=0 ):
+ """
+ Deactivate specified applications from node specified
+
+ """
+ main.log.report( 'deactivate implment me' )
+
+def isAppInstallSuccess():
+ """
+ Check the app list across clusters to determine
+ that apps have been installed successfully
+
+ """
+
+ main.log.report( 'isAppInstallSuccess implement me' )
+
diff --git a/TestON/tests/FuncPlatform/Dependency/Logger.py b/TestON/tests/FuncPlatform/Dependency/Logger.py
new file mode 100644
index 0000000..22d3f61
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Logger.py
@@ -0,0 +1,57 @@
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def checkOnosLog( nodeIp, option='',
+ outputType=0):
+ """
+ Listens to the log for any Errors and Exceptions.
+
+ Runs 'onos-check-logs <option>'
+ This script only returns if there are any errors
+ or exceptions
+
+ outputType
+ 0: Return output of log
+ 1: Return (#Errors, #Exceptions, #Warn)
+
+ """
+ if not isinstance( option, basestring ):
+ main.log.error( 'Incorrect grep format specified' )
+ return main.FALSE
+
+ try:
+ main.log.info( 'Starting Onos-log listening for '+
+ str(option) )
+ cmd = 'onos-check-logs ' + str(nodeIp) + ' old'
+ if outputType == 0:
+ main.ONOSbench.handle.sendline( cmd )
+ main.ONOSbench.handle.expect( cmd )
+ main.ONOSbench.handle.expect('\$')
+ logResult = main.ONOSbench.handle.before
+ return logResult
+ elif outputType == 1:
+ # Important in assertion criteria
+ # to determine how much warn / error is
+ # acceptable
+ return 'Implement option 1'
+ else:
+ main.log.error( 'Incorrect outputType specified' )
+ return main.FALSE
+
+ except Exception:
+ main.log.exception( self.name + ': Uncaught exception' )
+ main.cleanup()
+ main.exit()
+
+def setLogLevel( level ):
+ """
+ Set the log level of onos
+ """
+ main.log.info( 'setLogLevel implement me' )
+
+def getLogReport( nodeIp, searchTerms ):
+ """
+ Refer to CLI driver for 'logReport'
+ """
+ main.log.info( 'getLogReport - implement me!' )
diff --git a/TestON/tests/FuncPlatform/Dependency/Shutdown.py b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
new file mode 100644
index 0000000..ba77bbd
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
@@ -0,0 +1,19 @@
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def killOnosNodes( nodeIps ):
+ """
+ Kill all components of Onos on
+ given list of ips
+
+ Ex) nodeIps = ['10.0.0.1', '10.0.0.2']
+ """
+ killResult = main.TRUE
+
+ for node in nodeIps:
+ killResult = killResult and main.ONOSbench.onosDie( node )
+ if killResult == main.TRUE:
+ main.log.info( str(node) + ' was killed' )
+
+ return killResult
diff --git a/TestON/tests/FuncPlatform/Dependency/Startup.py b/TestON/tests/FuncPlatform/Dependency/Startup.py
new file mode 100644
index 0000000..f3f90a9
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Startup.py
@@ -0,0 +1,219 @@
+
+"""
+Startup related methods for ONOS
+
+Guidelines:
+ * Group sequential functionalities together
+ * Methods should not be dependent on platform
+ * Return main.TRUE on success or comprehensive error message
+ on failure (TBD)
+ * All methods should be consistent in expected behavior
+"""
+import time
+import json
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def gitPullAndMci( branchName, commitLog=False ):
+ """
+ Pull from branch repository specified and compile changes
+ If commitLog is True, report commit information
+
+ Any errors / warnings will be handled by respective
+ driver function calls
+ """
+ co = main.ONOSbench.gitCheckout( branchName )
+ gp = main.ONOSbench.gitPull()
+ ci = main.ONOSbench.cleanInstall()
+
+ if co and gp and ci == main.TRUE:
+ if commitLog:
+ main.log.report( 'Commit information - ' )
+ main.ONOSbench.getVersion(report=True)
+
+ return main.TRUE
+
+ else:
+ # TODO: Comprehensive error message
+ return 'git pull and mci failed'
+
+def initOnosStartupSequence( cellName, appStr, benchIp, mnIp, onosIps ):
+ """
+ Startup sequence includes the following:
+ * Create cell file
+ * Set cell variables on ONOS bench
+ * Verify cell
+ * Create ONOS package
+ * Force install ONOS package
+ * Start ONOS service
+ * Start ONOS cli
+
+ Also verifies that Onos is up and running by
+ 'isup' driver function which executs
+ 'onos-wait-for-start'
+ """
+
+ # NOTE: leave out create cell file until bug addressed
+ #cf = main.ONOSbench.createCellFile( benchIp, cellName, mnIp,
+ # str(appStr), *onosIps )
+ numNodes = len(onosIps)
+
+ sc = main.ONOSbench.setCell( cellName )
+ vc = main.ONOSbench.verifyCell()
+ op = main.ONOSbench.onosPackage()
+ for addr in onosIps:
+ oi = main.ONOSbench.onosInstall( node = addr )
+
+ time.sleep( 5 )
+
+ iu = main.TRUE
+ for node in onosIps:
+ iu = iu and main.ONOSbench.isup( node )
+
+ cli = main.TRUE
+ for node in range( 0, numNodes ):
+ cli = cli and main.CLIs[node].startOnosCli( onosIps[node] )
+
+ # Check if all nodes are discovered correctly using
+ # 'nodes' command in Onos Cli
+ na = main.TRUE
+ try:
+ nodeCmdJson = json.loads( main.CLIs[0].nodes() )
+ for node in nodeCmdJson:
+ if node['state'] != 'ACTIVE':
+ main.log.warn( str( node['id'] ) +
+ ' Node is not in ACTIVE state.' )
+ na = main.FALSE
+ if na != main.FALSE:
+ main.log.info( 'All nodes discovered successfully' )
+ except Exception:
+ main.log.error( 'nodes command did not execute properly' )
+ return main.FALSE
+
+ if sc and vc and op and oi and iu and cli and na == main.TRUE:
+ return main.TRUE
+ else:
+ return main.FALSE
+
+def installOnosFromTar( wgetAddr, nodeIps ):
+ """
+ Install Onos directly from tar.gz file.
+ Due to the nature of the specific steps required
+ to startup Onos in this fashion, all commands
+ required to start Onos from tar.gz will be
+ grouped in this method.
+
+ 1) wget latest onos tar.gz on onos node
+ 2) untar package
+ 3) specify onos-config cluster
+ 4) start onos via onos-service
+ 5) form onos cluster using onos-form-cluster
+ 6) check for successful startup
+
+ Specify the download link for the tar.gz.
+ Provide a list of nodeIps
+
+ Ex) wgetAddr = 'https://mytargzdownload.com/file.tar.gz'
+ nodeIps = ['10.0.0.1', '10.0.0.2']
+ """
+ if isinstance( nodeIps, ( int, basestring ) ):
+ main.log.error( 'Please pass in a list of string nodes' )
+ return main.FALSE
+
+ # Obtain filename from provided address
+ # assumes that filename is separated by '/' character
+ f_name = ''
+ addr = str( wgetAddr ).split('/')
+ for phrase in addr:
+ if 'tar.gz' in phrase:
+ f_name = str( phrase )
+ main.log.info( 'Using ' + f_name + ' as file' )
+
+ clusterCount = len( nodeIps )
+
+ main.log.info( 'Initiating Onos installation sequence ' +
+ 'using tar.gz ... This may take a few minutes' )
+
+ for node in range( 0, clusterCount ):
+ # Use the wgetAddr to download a new tar.gz from server
+ try:
+ main.ONOSnode[node].handle.sendline( 'wget ' + wgetAddr )
+ main.ONOSnode[node].handle.expect( 'saved' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ main.log.info( 'Successfully downloaded tar.gz ' +
+ 'on node: ' + str( main.ONOSips[node] ) )
+ except Exception:
+ # NOTE: Additional exception may be appropriate
+ main.log.error( 'Uncaught exception while ' +
+ 'downloading Onos tar.gz: ' +
+ main.ONOSnode[node].handle.before )
+ return main.FALSE
+
+ for node in range( 0, clusterCount ):
+ # Untar files on all nodes, then enter the
+ # newly created directory
+ try:
+ main.ONOSnode[node].handle.sendline( 'tar zxvf ' + f_name )
+ # Verbose output of tar will contain some onos returns
+ main.ONOSnode[node].handle.expect( 'onos' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ # NOTE: Making an assumption here:
+ # the directory created by tar file has a name that
+ # starts with 'onos-' followed by version number
+ # '1.2.0'. If this is NOT true, this method must
+ # be changed to enter the correct directory.
+ # The directory name is currently dynamic
+ # and depends on the day at which you downloaded
+ # the tar file
+ # Enter onos- wildcard to disregard its version / date
+ # suffix
+ main.ONOSnode[node].handle.sendline( 'cd onos-*' )
+ main.ONOSnode[node].handle.expect( '\$' )
+
+ main.ONOSnode[node].handle.sendline( 'pwd' )
+ main.ONOSnode[node].handle.expect( 'pwd' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ pwd = main.ONOSnode[node].handle.before
+ if 'onos' in str(pwd):
+ main.log.info( 'tar zxvf ' + f_name + ' successful ' +
+ 'on node ' + str( main.ONOSips[node] ) )
+
+ except Exception:
+ main.log.error( 'Uncaught exception while executing ' +
+ 'tar zxvf of ' + f_name + ': ' +
+ main.ONOSnode[node].handle.before +
+ main.ONOSnode[node].handle.after)
+ return main.FALSE
+
+ for node in range( 0, clusterCount ):
+ try:
+ main.ONOSnode[node].handle.sendline( 'bin/onos-service '+
+ 'server &' )
+ # Send some extra characters to run the process
+ main.ONOSnode[node].handle.sendline( '' )
+ main.ONOSnode[node].handle.sendline( '' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ except Exception:
+ main.log.error( 'Uncaught exception while executing ' +
+ 'onos-service server command ' +
+ str( main.ONOSnode[node].handle.before ) )
+ return main.FALSE
+
+ iu = main.TRUE
+ for node in nodeIps:
+ iu = iu and main.ONOSbench.isup( node )
+
+ if iu == main.TRUE:
+ return main.TRUE
+ else:
+ return main.FALSE
+
+def addAndStartOnosNode( nodeIps ):
+ """
+ A scale-out scenario that adds specified list of
+ nodes and starts those instances.
+
+ Ex) nodeIps = ['10.0.0.2', '10.0.0.3', 10.0.0.4']
+ """
+ main.log.info( 'addAndStartOnosNode implement me!' )
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.params b/TestON/tests/FuncPlatform/FuncPlatform.params
new file mode 100644
index 0000000..3895660
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.params
@@ -0,0 +1,63 @@
+<PARAMS>
+ <testcases>1,2,3,4</testcases>
+ <DEP>
+ <startupSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Startup.py
+ </startupSrc>
+ <startupClassName>
+ Startup
+ </startupClassName>
+
+ <appSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/App.py
+ </appSrc>
+ <appClassName>
+ App
+ </appClassName>
+
+ <logSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Logger.py
+ </logSrc>
+ <logClassName>
+ Log
+ </logClassName>
+
+ <shutdownSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Shutdown.py
+ </shutdownSrc>
+ <shutdownClassName>
+ Shutdown
+ </shutdownClassName>
+
+ <targz>
+ http://downloads.onosproject.org/nightly/onos-1.2.0.latest-NIGHTLY.tar.gz
+ </targz>
+ </DEP>
+
+ <CTRL>
+ <num>3</num>
+ <port>6633</port>
+ <ip1>10.128.174.1</ip1>
+ <ip2>10.128.174.2</ip2>
+ <ip3>10.128.174.3</ip3>
+ </CTRL>
+
+ <MN>
+ <ip>10.128.10.90</ip>
+ </MN>
+
+ <BENCH>
+ <ip>10.128.174.10</ip>
+ </BENCH>
+
+ <CELL>
+ <name>func_plat</name>
+ <appStr>"drivers,openflow"</appStr>
+ </CELL>
+
+ <GIT>
+ <branchName>master</branchName>
+ <pull>off</pull>
+ </GIT>
+
+</PARAMS>
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.py b/TestON/tests/FuncPlatform/FuncPlatform.py
new file mode 100644
index 0000000..60b1aa0
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.py
@@ -0,0 +1,194 @@
+"""
+FuncPlatform
+
+A functional test designed to test the environment and
+gather information on startup -> shutdown related issues.
+
+Future works may also include security mode startup /
+shutdown check and cfg get and set.
+
+Abstracting the collection of commands that go hand in hand
+should allow easy rearrangement of steps to replicate or
+create scenarios.
+For example:
+ CASE N - Represents a particular scenario
+ Steps - Represents abstraction methods called from
+ dependency
+ 1. Bring ONOS 1 up
+ 2. Activate application X
+ 3. Activate application Y
+ 4. Deactivate application X
+
+The ideal platform test script should have incredible
+robustness to possible exceptions and report the most
+useful error messages.
+
+contributers to contact for help:
+andrew@onlab.us
+"""
+
+class FuncPlatform:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ """
+ Main scope initialization case
+ Must include to run any other test cases
+ """
+ import imp
+
+ # NOTE: Hardcoded application name subject to change
+ # closely monitor and make changes when necessary
+ # (or implement ways to dynamically get names)
+ main.appList = {
+ 'bgprouter' : 'org.onosproject.bgprouter',
+ 'config' : 'org.onosproject.config',
+ 'cordfabric' : 'org.onosproject.cordfabric',
+ 'demo' : 'org.onosproject.demo',
+ 'distributedprimitives' : 'org.onosproject.distributedprimitives',
+ 'election' : 'org.onosproject.election',
+ 'flowrule' : 'org.onosproject.flowrule',
+ 'fwd' : 'org.onosproject.fwd',
+ 'intentperf' : 'org.onosproject.intentperf',
+ 'messagingperf' : 'org.onosproject.messagingperf',
+ 'metrics' : 'org.onosproject.metrics',
+ 'mobility' : 'org.onosproject.mobility',
+ 'netconf' : 'org.onosproject.netconf',
+ 'null' : 'org.onosproject.null',
+ 'optical' : 'org.onosproject.optical',
+ 'pcep' : 'org.onosproject.pcep',
+ 'proxyarp' : 'org.onosproject.proxyarp',
+ 'reactive.routing' : 'org.onosproject.reactive.routing',
+ 'sdnip' : 'org.onosproject.sdnip',
+ 'segmentrouting' : 'org.onosproject.segmentrouting',
+ 'tunnel' : 'org.onosproject.tunnel',
+ 'virtualbng' : 'org.onosproject.virtualbng',
+ 'xosintegration' : 'org.onosproject.xosintegration'
+ }
+ # List of ONOS ip's specififed in params
+ main.ONOSips = []
+ main.CLIs = []
+ main.ONOSnode = []
+
+ for node in range( 0, int(main.params['CTRL']['num']) ):
+ main.ONOSips.append( main.params['CTRL']['ip'+str(node+1)] )
+ main.CLIs.append(
+ getattr( main, 'ONOS' + str(node+1) + 'cli' ) )
+ main.ONOSnode.append(
+ getattr( main, 'ONOS' + str(node+1) ) )
+
+ # Application source and name definitions
+ startupSrc = main.params['DEP']['startupSrc']
+ startupClassName = main.params['DEP']['startupClassName']
+
+ appClassName = main.params['DEP']['appClassName']
+ appSrc = main.params['DEP']['appSrc']
+
+ logClassName = main.params['DEP']['logClassName']
+ logSrc = main.params['DEP']['logSrc']
+
+ shutdownClassName = main.params['DEP']['shutdownClassName']
+ shutdownSrc = main.params['DEP']['shutdownSrc']
+
+ # Importing dependency class(es)
+ # Refer to source files in Dependency folder to
+ # make changes to its respective methods
+ # Be weary of naming collisions
+ try:
+ main.startup = imp.load_source( startupClassName, startupSrc )
+ main.app = imp.load_source( appClassName, appSrc )
+ main.onosLog = imp.load_source( logClassName, logSrc )
+ main.shutdown = imp.load_source( shutdownClassName, shutdownSrc )
+ except ImportError:
+ main.log.error( 'Error importing class file(s). Please ' +
+ 'check file location' )
+ main.cleanup()
+ main.exit()
+
+ def CASE2( self, main ):
+ import time
+
+ cellName = main.params['CELL']['name']
+ appStr = main.params['CELL']['appStr']
+ benchIp = main.params['BENCH']['ip']
+ branchName = main.params['GIT']['branchName']
+ gitPull = main.params['GIT']['pull']
+ mnIp = main.params['MN']['ip']
+
+ main.case( 'Setup environment and install ONOS' )
+ if gitPull == 'on':
+ main.step( 'Git pull and clean install' )
+ gitPullResult = main.startup.gitPullAndMci( branchName )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=gitPullResult,
+ onpass='Git pull and install successful',
+ onfail='Git pull and install failed: ' +
+ str(gitPullResult) )
+
+ main.step( 'Initiate ONOS startup sequence' )
+ startupResult = main.startup.initOnosStartupSequence(
+ cellName, appStr, benchIp, mnIp, main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=startupResult,
+ onpass='ONOS startup sequence successful',
+ onfail='ONOS startup sequence failed: ' +
+ str(startupResult) )
+
+ def CASE3( self, main ):
+ import time
+
+ main.case( 'Activate applications and check installation' )
+
+ # NOTE: Test only
+ # Unceremoniously kill onos 2
+ main.ONOSbench.onosDie( '10.128.174.2' )
+
+ time.sleep( 30 )
+
+ main.step( 'Sample Onos log check' )
+ logResult = main.onosLog.checkOnosLog( main.ONOSips[0] )
+ main.log.info( logResult )
+ # TODO: Define pass criteria
+ utilities.assert_equals( expect=main.TRUE,
+ actual=main.TRUE,
+ onpass= 'Logging successful',
+ onfail= 'Logging failed ' )
+
+ # Sample app activation
+ main.step( 'Activating applications metrics and fwd' )
+ appList = ['metrics', 'fwd']
+ appResult = main.app.activate( appList )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=appResult,
+ onpass= 'App activation of ' + str(appList) + ' successful',
+ onfail= 'App activation failed ' + str(appResult) )
+
+ def CASE4( self, main ):
+ """
+ Download ONOS tar.gz built from latest nightly
+ (following tutorial on wiki) and run ONOS directly on the
+ instance
+ """
+ import imp
+
+ targz = main.params['DEP']['targz']
+ clusterCount = main.params['CTRL']['num']
+
+ main.case( 'Install ONOS from onos.tar.gz file' )
+
+ main.step( 'Killing all ONOS instances previous started' )
+ killResult = main.shutdown.killOnosNodes( main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual = killResult,
+ onpass = 'All Onos nodes successfully killed',
+ onfail = 'Onos nodes were not successfully killed' )
+
+ main.step( 'Starting ONOS using tar.gz on all nodes' )
+ installResult = main.startup.installOnosFromTar( targz, main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual = installResult,
+ onpass= 'Onos tar.gz installation successful',
+ onfail= 'Onos tar.gz installation failed' )
+
+
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.topo b/TestON/tests/FuncPlatform/FuncPlatform.topo
new file mode 100644
index 0000000..f19e8f8
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.topo
@@ -0,0 +1,82 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOSbench>
+
+ <ONOS1>
+ <host>10.128.174.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.174.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.174.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3>
+
+ <ONOS1cli>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1cli>
+
+ <ONOS2cli>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2cli>
+
+ <ONOS3cli>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3cli>
+
+ <MininetOvs>
+ <host>10.128.10.90</host>
+ <user>admin</user>
+ <password></password>
+ <type>MininetCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ <arg1> --custom topo-perf-2sw.py </arg1>
+ <arg2> --arp --mac --topo mytopo </arg2>
+ <arg3> </arg3>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </MininetOvs>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/FuncStartTemplate/FuncStartTemplate.params b/TestON/tests/FuncStartTemplate/FuncStartTemplate.params
new file mode 100755
index 0000000..310e5f8
--- /dev/null
+++ b/TestON/tests/FuncStartTemplate/FuncStartTemplate.params
@@ -0,0 +1,34 @@
+<PARAMS>
+
+ <testcases>10,9,10,9</testcases>
+
+ <SCALE>3,1</SCALE>
+ <availableNodes>3</availableNodes>
+ <ENV>
+ <cellName>single_func</cellName>
+ <cellApps>drivers,openflow,proxyarp,mobility</cellApps>
+ </ENV>
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+ <CTRL>
+ <num>3</num>
+ <ip1>OC1</ip1>
+ <port1>6633</port1>
+ <ip2>OC2</ip2>
+ <port2>6633</port2>
+ <ip3>OC3</ip3>
+ <port3>6633</port3>
+ </CTRL>
+ <BENCH>
+ <user>admin</user>
+ <ip1>OCN</ip1>
+ </BENCH>
+ <MININET>
+ <switch>7</switch>
+ <links>20</links>
+ <topo>~/mininet/custom/newFuncTopo.py</topo>
+ </MININET>
+
+</PARAMS>
diff --git a/TestON/tests/FuncStartTemplate/FuncStartTemplate.py b/TestON/tests/FuncStartTemplate/FuncStartTemplate.py
new file mode 100644
index 0000000..4beefae
--- /dev/null
+++ b/TestON/tests/FuncStartTemplate/FuncStartTemplate.py
@@ -0,0 +1,231 @@
+
+# Testing the basic functionality of ONOS Next
+# For sanity and driver functionality excercises only.
+
+import time
+import json
+
+time.sleep( 1 )
+
+class FuncStartTemplate:
+
+ def __init__( self ):
+ self.default = ''
+
+ def CASE10( self, main ):
+ import time
+ import os
+ """
+ Startup sequence:
+ cell <name>
+ onos-verify-cell
+ onos-remove-raft-log
+ git pull
+ mvn clean install
+ onos-package
+ onos-install -f
+ onos-wait-for-start
+ """
+ global init
+ global globalONOSip
+ try:
+ if type(init) is not bool:
+ init = False
+ except NameError:
+ init = False
+
+ #Local variables
+ cellName = main.params[ 'ENV' ][ 'cellName' ]
+ apps = main.params[ 'ENV' ][ 'cellApps' ]
+ gitBranch = main.params[ 'GIT' ][ 'branch' ]
+ benchIp = os.environ[ 'OCN' ]
+ benchUser = main.params[ 'BENCH' ][ 'user' ]
+ topology = main.params[ 'MININET' ][ 'topo' ]
+ main.numSwitch = int( main.params[ 'MININET' ][ 'switch' ] )
+ main.numLinks = int( main.params[ 'MININET' ][ 'links' ] )
+ main.numCtrls = main.params[ 'CTRL' ][ 'num' ]
+ PULLCODE = False
+ if main.params[ 'GIT' ][ 'pull' ] == 'True':
+ PULLCODE = True
+ main.case( "Setting up test environment" )
+ main.CLIs = []
+ for i in range( 1, int( main.numCtrls ) + 1 ):
+ main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+
+ # -- INIT SECTION, ONLY RUNS ONCE -- #
+ if init == False:
+ init = True
+
+ main.ONOSport = []
+ main.scale = ( main.params[ 'SCALE' ] ).split( "," )
+ main.numCtrls = int( main.scale[ 0 ] )
+
+ if PULLCODE:
+ main.step( "Git checkout and pull " + gitBranch )
+ main.ONOSbench.gitCheckout( gitBranch )
+ gitPullResult = main.ONOSbench.gitPull()
+ if gitPullResult == main.ERROR:
+ main.log.error( "Error pulling git branch" )
+ main.step( "Using mvn clean & install" )
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ stepResult = cleanInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully compiled " +
+ "latest ONOS",
+ onfail="Failed to compile " +
+ "latest ONOS" )
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
+
+ globalONOSip = main.ONOSbench.getOnosIps()
+
+ maxNodes = ( len(globalONOSip) - 2 )
+
+ main.numCtrls = int( main.scale[ 0 ] )
+ main.scale.remove( main.scale[ 0 ] )
+
+ main.ONOSip = []
+ for i in range( maxNodes ):
+ main.ONOSip.append( globalONOSip[i] )
+
+
+
+ #kill off all onos processes
+ main.log.info( "Safety check, killing all ONOS processes" +
+ " before initiating enviornment setup" )
+ for i in range(maxNodes):
+ main.ONOSbench.onosDie( globalONOSip[ i ] )
+
+ """
+ main.step( "Apply cell to environment" )
+ cellResult = main.ONOSbench.setCell( cellName )
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " + \
+ "environment",
+ onfail="Failed to apply cell to environment " )
+ """
+ """main.step( "Removing raft logs" )
+ removeRaftResult = main.ONOSbench.onosRemoveRaftLogs()
+ stepResult = removeRaftResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully removed raft logs",
+ onfail="Failed to remove raft logs" )
+ """
+
+ print "NODE COUNT = ", main.numCtrls
+ main.log.info( "Creating cell file" )
+ cellIp = []
+ for i in range( main.numCtrls ):
+ cellIp.append( str( main.ONOSip[ i ] ) )
+ print cellIp
+ main.ONOSbench.createCellFile( benchIp, cellName, "",
+ str( apps ), *cellIp )
+
+ main.step( "Apply cell to environment" )
+ cellResult = main.ONOSbench.setCell( cellName )
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " + \
+ "environment",
+ onfail="Failed to apply cell to environment " )
+
+ main.step( "Creating ONOS package" )
+ packageResult = main.ONOSbench.onosPackage()
+ stepResult = packageResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully created ONOS package",
+ onfail="Failed to create ONOS package" )
+
+ main.step( "Uninstalling ONOS package" )
+ onosUninstallResult = main.TRUE
+ for i in range( main.numCtrls ):
+ onosUninstallResult = onosUninstallResult and \
+ main.ONOSbench.onosUninstall( nodeIp=main.ONOSip[ i ] )
+ stepResult = onosUninstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully uninstalled ONOS package",
+ onfail="Failed to uninstall ONOS package" )
+ time.sleep( 5 )
+ main.step( "Installing ONOS package" )
+ onosInstallResult = main.TRUE
+ for i in range( main.numCtrls ):
+ onosInstallResult = onosInstallResult and \
+ main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+ stepResult = onosInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully installed ONOS package",
+ onfail="Failed to install ONOS package" )
+
+ time.sleep( 20 )
+ main.step( "Starting ONOS service" )
+ stopResult = main.TRUE
+ startResult = main.TRUE
+ onosIsUp = main.TRUE
+ for i in range( main.numCtrls ):
+ onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+ if onosIsUp == main.TRUE:
+ main.log.report( "ONOS instance is up and ready" )
+ else:
+ main.log.report( "ONOS instance may not be up, stop and " +
+ "start ONOS again " )
+ for i in range( main.numCtrls ):
+ stopResult = stopResult and \
+ main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ for i in range( main.numCtrls ):
+ startResult = startResult and \
+ main.ONOSbench.onosStart( main.ONOSip[ i ] )
+ stepResult = onosIsUp and stopResult and startResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+
+ main.step( "Start ONOS cli" )
+ cliResult = main.TRUE
+ for i in range( main.numCtrls ):
+ cliResult = cliResult and \
+ main.CLIs[i].startOnosCli( main.ONOSip[ i ] )
+ stepResult = cliResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully start ONOS cli",
+ onfail="Failed to start ONOS cli" )
+
+ def CASE9( self, main ):
+ '''
+ Report errors/warnings/exceptions
+ '''
+ main.log.info("Error report: \n")
+ main.ONOSbench.logReport( globalONOSip[0], [ "INFO","FOLLOWER","WARN","flow","ERROR","Except" ], "s" )
+ #main.ONOSbench.logReport( globalONOSip[1], [ "INFO" ], "d" )
+
+ def CASE11( self, main ):
+ """
+ Start mininet
+ """
+ main.log.report( "Start Mininet topology" )
+ main.log.case( "Start Mininet topology" )
+
+ main.step( "Starting Mininet Topology" )
+ topoResult = main.Mininet1.startNet( topoFile=topology )
+ stepResult = topoResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully loaded topology",
+ onfail="Failed to load topology" )
+ # Exit if topology did not load properly
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+
diff --git a/TestON/tests/FuncStartTemplate/FuncStartTemplate.topo b/TestON/tests/FuncStartTemplate/FuncStartTemplate.topo
new file mode 100755
index 0000000..2321da0
--- /dev/null
+++ b/TestON/tests/FuncStartTemplate/FuncStartTemplate.topo
@@ -0,0 +1,52 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ <home>~/onos</home>
+ </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli3>
+
+ <Mininet1>
+ <host>OCN</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>MininetCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </Mininet1>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/FuncStartTemplate/__init__.py
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/FuncStartTemplate/__init__.py
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 123fd0f..390facf 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -1,11 +1,12 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
- <Git>False</Git>
+ <Git> False </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 1205b11..50bf06f 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -18,6 +18,8 @@
CASE13: Clean up
CASE14: start election app on all onos nodes
CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
"""
@@ -44,9 +46,12 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
+ main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -90,36 +95,73 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
+ # Make sure ONOS is DEAD
+ main.log.info( "Killing any ONOS processes" )
+ killResults = main.TRUE
+ for node in nodes:
+ killed = main.ONOSbench.onosKill( node.ip_address )
+ killResults = killResults and killed
+
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
main.step( "Starting Mininet" )
- main.Mininet1.startNet( )
+ mnResult = main.Mininet1.startNet( )
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet Started",
+ onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
- if gitPullResult == main.ERROR:
- main.log.error( "Error pulling git branch" )
+ # values of 1 or 3 are good
+ utilities.assert_lesser( expect=0, actual=gitPullResult,
+ onpass="Git pull successful",
+ onfail="Git pull failed" )
+ main.ONOSbench.getVersion( report=True )
- main.step( "Using mvn clean & install" )
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
- main.ONOSbench.getVersion( report=True )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
+ # GRAPHS
+ # NOTE: important params here:
+ # job = name of Jenkins job
+ # Plot Name = Plot-HA, only can be used if multiple plots
+ # index = The number of the graph under plot name
+ job = "HAClusterRestart"
+ plotName = "Plot-HA"
+ graphs = '<ac:structured-macro ac:name="html">\n'
+ graphs += '<ac:plain-text-body><![CDATA[\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
+ 'noborder="0" width="500" height="300" scrolling="yes" ' +\
+ 'seamless="seamless"></iframe>\n'
+ graphs += ']]></ac:plain-text-body>\n'
+ graphs += '</ac:structured-macro>\n'
+ main.log.wiki(graphs)
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
+ utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+ onpass="ONOS package successful",
+ onfail="ONOS package failed" )
main.step( "Installing ONOS package" )
onosInstallResult = main.TRUE
@@ -127,6 +169,9 @@
tmpResult = main.ONOSbench.onosInstall( options="-f",
node=node.ip_address )
onosInstallResult = onosInstallResult and tmpResult
+ utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+ onpass="ONOS install successful",
+ onfail="ONOS install failed" )
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
@@ -134,12 +179,15 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
+ utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+ onpass="ONOS startup successful",
+ onfail="ONOS startup failed" )
main.log.step( "Starting ONOS CLI sessions" )
cliResults = main.TRUE
@@ -154,14 +202,19 @@
for t in threads:
t.join()
cliResults = cliResults and t.result
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS cli startup successful",
+ onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ main.step( "App Ids check" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -174,22 +227,15 @@
for t in threads:
t.join()
appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults and appCheck)
-
- utilities.assert_equals( expect=main.TRUE, actual=case1Result,
- onpass="Test startup successful",
- onfail="Test startup NOT successful" )
-
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -198,6 +244,7 @@
Assign mastership to controllers
"""
import re
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -211,8 +258,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -244,16 +295,18 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
+
+ main.step( "Assign mastership of switches to specific controllers" )
# Manually assign mastership to the controller we want
roleCall = main.TRUE
- roleCheck = main.TRUE
+
+ ipList = [ ]
+ deviceList = []
try:
for i in range( 1, 29 ): # switches 1 through 28
# set up correct variables:
@@ -297,16 +350,8 @@
# TODO: make this controller dynamic
roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
ip )
- # Check assignment
- master = main.ONOScli1.getRole( deviceId ).get( 'master' )
- if ip in master:
- roleCheck = roleCheck and main.TRUE
- else:
- roleCheck = roleCheck and main.FALSE
- main.log.error( "Error, controller " + ip + " is not" +
- " master " + "of device " +
- str( deviceId ) + ". Master is " +
- repr( master ) + "." )
+ ipList.append( ip )
+ deviceList.append( deviceId )
except ( AttributeError, AssertionError ):
main.log.exception( "Something is wrong with ONOS device view" )
main.log.info( main.ONOScli1.devices() )
@@ -316,6 +361,24 @@
onpass="Re-assigned switch mastership to designated controller",
onfail="Something wrong with deviceRole calls" )
+ main.step( "Check mastership was correctly assigned" )
+ roleCheck = main.TRUE
+ # NOTE: This is due to the fact that device mastership change is not
+ # atomic and is actually a multi step process
+ time.sleep( 5 )
+ for i in range( len( ipList ) ):
+ ip = ipList[i]
+ deviceId = deviceList[i]
+ # Check assignment
+ master = main.ONOScli1.getRole( deviceId ).get( 'master' )
+ if ip in master:
+ roleCheck = roleCheck and main.TRUE
+ else:
+ roleCheck = roleCheck and main.FALSE
+ main.log.error( "Error, controller " + ip + " is not" +
+ " master " + "of device " +
+ str( deviceId ) + ". Master is " +
+ repr( master ) + "." )
utilities.assert_equals(
expect=main.TRUE,
actual=roleCheck,
@@ -332,8 +395,6 @@
"""
Assign intents
"""
- # FIXME: we must reinstall intents until we have a persistant
- # datastore!
import time
import json
assert numControllers, "numControllers not defined"
@@ -341,17 +402,23 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
+ # NOTE: we must reinstall intents until we have a persistant intent
+ # datastore!
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
- main.log.info( "Install reactive forwarding app" )
- appResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+ main.step( "Install reactive forwarding app" )
+ installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=installResults,
+ onpass="Install fwd successful",
+ onfail="Install fwd failed" )
- # FIXME: add this to asserts
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -364,13 +431,15 @@
for t in threads:
t.join()
appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -380,14 +449,23 @@
expect=main.TRUE,
actual=pingResult,
onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ onfail="Reactive Pingall failed, " +
+ "one or more ping pairs failed" )
time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+ main.log.info( "Time for pingall: %2f seconds" %
+ ( time2 - time1 ) )
+ # timeout for fwd flows
+ time.sleep( 11 )
# uninstall onos-app-fwd
- main.log.info( "Uninstall reactive forwarding app" )
- appResults = appResults and CLIs[0].deactivateApp( "org.onosproject.fwd" )
+ main.step( "Uninstall reactive forwarding app" )
+ uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+ onpass="Uninstall fwd successful",
+ onfail="Uninstall fwd failed" )
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -397,18 +475,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- # timeout for fwd flows
- time.sleep( 11 )
-
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -451,7 +526,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -461,7 +539,6 @@
pass # intent submitted is in onos
else:
intentAddResult = False
- # FIXME: DEBUG
if intentAddResult:
intentStop = time.time()
else:
@@ -546,35 +623,49 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
+ main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
- onosIds = main.ONOScli1.getAllIntentsId()
+ correct = True
main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
- main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
- if sorted(onosIds) == sorted(intentIds):
+ for cli in CLIs:
+ onosIds = []
+ ids = cli.getAllIntentsId()
+ onosIds.append( ids )
+ main.log.debug( "Intents in " + cli.name + ": " +
+ str( sorted( onosIds ) ) )
+ if sorted( ids ) != sorted( intentIds ):
+ # IF intents are missing
+ correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ correct = False
+ break
+ if correct:
break
else:
time.sleep(1)
- # FIXME: DEBUG
if not intentStop:
intentStop = time.time()
+ global gossipTime
gossipTime = intentStop - intentStart
main.log.info( "It took about " + str( gossipTime ) +
- " seconds for all intents to appear on ONOS1" )
+ " seconds for all intents to appear in each node" )
# FIXME: make this time configurable/calculate based off of number of
# nodes and gossip rounds
utilities.assert_greater_equals(
- expect=30, actual=gossipTime,
+ expect=40, actual=gossipTime,
onpass="ECM anti-entropy for intents worked within " +
"expected time",
onfail="Intent ECM anti-entropy took too long" )
- if gossipTime <= 30:
+ if gossipTime <= 40:
intentAddResult = True
if not intentAddResult or "key" in pendingMap:
@@ -677,9 +768,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -692,7 +785,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -704,17 +797,17 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
- installedCheck = True
- if PingResult is not main.TRUE:
+ main.step( "Check Intent state" )
+ installedCheck = False
+ count = 0
+ while not installedCheck and count < 40:
+ installedCheck = True
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
@@ -730,71 +823,92 @@
intentStates.append( ( intentId, state ) )
except ( ValueError, TypeError ):
main.log.exception( "Error parsing intents." )
+ # Print states
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +
+ "INSTALLED state" )
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
+ time.sleep( 60 )
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if the state of " +
- "intents change" )
- time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
@@ -870,6 +984,39 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ # the wait is above
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.error(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
def CASE5( self, main ):
"""
@@ -885,7 +1032,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -933,7 +1079,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -947,7 +1093,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -995,7 +1141,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1008,34 +1154,63 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ if intentsResults:
+ # Try to make it easy to figure out what is happening
+ #
+ # Intent ONOS1 ONOS2 ...
+ # 0x01 INSTALLED INSTALLING
+ # ... ... ...
+ # ... ... ...
+ title = " Id"
+ for n in range( numControllers ):
+ title += " " * 10 + "ONOS" + str( n + 1 )
+ main.log.warn( title )
+ # get all intent keys in the cluster
+ keys = []
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id', "Error" ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End table view
+
if intentsResults and not consistentIntents:
+ # print the json objects
n = len(ONOSIntents)
- main.log.warn( "ONOS" + str( n ) + " intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.debug( "ONOS" + str( n ) + " intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
for i in range( numControllers ):
if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
- main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
else:
- main.log.warn( nodes[ i ].name + " intents match ONOS" +
- str( n ) + " intents" )
+ main.log.debug( nodes[ i ].name + " intents match ONOS" +
+ str( n ) + " intents" )
elif intentsResults and consistentIntents:
intentCheck = main.TRUE
intentState = ONOSIntents[ 0 ]
@@ -1067,7 +1242,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1091,7 +1266,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1219,6 +1394,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -1246,6 +1429,7 @@
# Compare json objects for hosts and dataplane clusters
# hosts
+ main.step( "Host view is consistent across ONOS nodes" )
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
@@ -1253,14 +1437,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1272,11 +1456,12 @@
onpass="Hosts view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of hosts" )
+ main.step( "Each host has an IP address" )
ipResult = main.TRUE
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1287,6 +1472,7 @@
onfail="The ip of at least one host is missing" )
# Strongly connected clusters of devices
+ main.step( "Cluster view is consistent across ONOS nodes" )
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
controllerStr = str( controller + 1 )
@@ -1294,12 +1480,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1311,6 +1497,7 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
+ main.step( "Cluster view correct across ONOS nodes" )
try:
numClusters = len( json.loads( clusters[ 0 ] ) )
except ( ValueError, TypeError ):
@@ -1381,30 +1568,43 @@
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- finalAssert = main.TRUE
- finalAssert = ( finalAssert and topoResult and flowCheck
- and intentCheck and consistentMastership
- and mastershipCheck and rolesNotNull )
- utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
- onpass="State check successful",
- onfail="State check NOT successful" )
-
def CASE6( self, main ):
"""
The Failure case.
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Restart entire ONOS cluster" )
+
+ # Reset non-persistent variables
+ try:
+ iCounterValue = 0
+ except NameError:
+ main.log.error( "iCounterValue not defined, setting to 0" )
+ iCounterValue = 0
+
main.case( "Restart entire ONOS cluster" )
+ # FIXME: DEBUG
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ # FIXME: DEBUG
+
main.step( "Killing ONOS nodes" )
killResults = main.TRUE
+ killTime = time.time()
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
killResults = killResults and killed
+ utilities.assert_equals( expect=main.TRUE, actual=killResults,
+ onpass="ONOS nodes killed",
+ onfail="ONOS kill unsuccessful" )
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
@@ -1412,10 +1612,13 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
+ utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+ onpass="ONOS restarted",
+ onfail="ONOS restart NOT successful" )
main.log.step( "Starting ONOS CLI sessions" )
cliResults = main.TRUE
@@ -1430,11 +1633,32 @@
for t in threads:
t.join()
cliResults = cliResults and t.result
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS cli started",
+ onfail="ONOS clis did not restart" )
- caseResults = main.TRUE and onosIsupResult and cliResults
- utilities.assert_equals( expect=main.TRUE, actual=caseResults,
- onpass="ONOS restart successful",
- onfail="ONOS restart NOT successful" )
+ # Grab the time of restart so we chan check how long the gossip
+ # protocol has had time to work
+ main.restartTime = time.time() - killTime
+ main.log.debug( "Restart time: " + str( main.restartTime ) )
+
+ # FIXME: revisit test plan for election with madan
+ # Rerun for election on restarted nodes
+ runResults = main.TRUE
+ for cli in CLIs:
+ run = CLIs[0].electionTestRun()
+ if run != main.TRUE:
+ main.log.error( "Error running for election on " + cli.name )
+ runResults = runResults and run
+ utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ onpass="Reran for election",
+ onfail="Failed to rerun for election" )
+
+ # TODO: Make this configurable
+ time.sleep( 60 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1468,6 +1692,7 @@
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
+ main.step( "Read device roles from ONOS" )
ONOSMastership = []
mastershipCheck = main.FALSE
consistentMastership = True
@@ -1486,7 +1711,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1500,7 +1725,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1522,6 +1747,7 @@
elif rolesResults and not consistentMastership:
mastershipCheck = main.TRUE
+ '''
description2 = "Compare switch roles from before failure"
main.step( description2 )
try:
@@ -1547,15 +1773,13 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed" )
+ '''
# NOTE: we expect mastership to change on controller failure
- mastershipCheck = mastershipCheck and consistentMastership
main.step( "Get the intents and compare across all nodes" )
ONOSIntents = []
@@ -1577,7 +1801,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1590,10 +1814,38 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
+
+ # Try to make it easy to figure out what is happening
+ #
+ # Intent ONOS1 ONOS2 ...
+ # 0x01 INSTALLED INSTALLING
+ # ... ... ...
+ # ... ... ...
+ title = " ID"
+ for n in range( numControllers ):
+ title += " " * 10 + "ONOS" + str( n + 1 )
+ main.log.warn( title )
+ # get all intent keys in the cluster
+ keys = []
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id' ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End table view
+
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1630,22 +1882,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1668,16 +1939,14 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
- main.step( "Check the continuous pings to ensure that no packets " +
- "were dropped during component failure" )
+ # main.step( "Check the continuous pings to ensure that no packets " +
+ # "were dropped during component failure" )
main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
main.params[ 'TESTONIP' ] )
LossInPings = main.FALSE
@@ -1695,16 +1964,18 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
- utilities.assert_equals(
- expect=main.FALSE,
- actual=LossInPings,
- onpass="No Loss of connectivity",
- onfail="Loss of dataplane connectivity detected" )
+ main.log.info( "No loss of dataplane connectivity" )
+ # utilities.assert_equals(
+ # expect=main.FALSE,
+ # actual=LossInPings,
+ # onpass="No Loss of connectivity",
+ # onfail="Loss of dataplane connectivity detected" )
+
# NOTE: Since intents are not persisted with IntnentStore,
# we expect loss in dataplane connectivity
LossInPings = main.FALSE
+ main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leaderList = []
leaderResult = main.TRUE
@@ -1712,40 +1983,26 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
- # error in response
- main.log.report( "Something is wrong with " +
+ # error in response
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( cli.name +
- " shows no leader for the election-app was" +
- " elected after the old one died" )
+ main.log.error( cli.name +
+ " shows no leader for the election-app." )
leaderResult = main.FALSE
if len( set( leaderList ) ) != 1:
leaderResult = main.FALSE
main.log.error(
"Inconsistent view of leader for the election test app" )
# TODO: print the list
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
onpass="Leadership election passed",
onfail="Something went wrong with Leadership election" )
- result = ( mastershipCheck and intentCheck and FlowTables and
- ( not LossInPings ) and rolesNotNull and leaderResult )
- result = int( result )
- if result == main.TRUE:
- main.log.report( "Constant State Tests Passed" )
- utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Constant State Tests Passed",
- onfail="Constant state tests failed" )
-
def CASE8( self, main ):
"""
Compare topo
@@ -1763,15 +2020,24 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1784,11 +2050,12 @@
main.step( "Collecting topology information from ONOS" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and elapsed < 120:
count += 1
if count > 1:
# TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
+ time.sleep( 1 )
cliStart = time.time()
devices = []
threads = []
@@ -1822,9 +2089,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -1839,6 +2106,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -1866,6 +2141,7 @@
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
+ print "Elapsed time: " + str( elapsed )
print "CLI time: " + str( cliTime )
for controller in range( numControllers ):
@@ -1930,6 +2206,7 @@
# Compare json objects for hosts and dataplane clusters
# hosts
+ main.step( "Hosts view is consistent across all ONOS nodes" )
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
@@ -1937,13 +2214,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1956,6 +2233,7 @@
onfail="ONOS nodes have different views of hosts" )
# Strongly connected clusters of devices
+ main.step( "Clusters view is consistent across all ONOS nodes" )
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
controllerStr = str( controller + 1 )
@@ -1963,13 +2241,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1981,6 +2259,8 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
+ main.step( "Topology view is correct and consistent across all " +
+ "ONOS nodes" )
try:
numClusters = len( json.loads( clusters[ 0 ] ) )
except ( ValueError, TypeError ):
@@ -2011,12 +2291,11 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
nodesOutput = []
+ nodeResults = main.TRUE
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].nodes,
@@ -2033,9 +2312,10 @@
try:
current = json.loads( i )
for node in current:
+ currentResult = main.FALSE
if node['ip'] in ips: # node in nodes() output is in cell
if node['state'] == 'ACTIVE':
- pass # as it should be
+ currentResult = main.TRUE
else:
main.log.error( "Error in ONOS node availability" )
main.log.error(
@@ -2044,9 +2324,13 @@
indent=4,
separators=( ',', ': ' ) ) )
break
+ nodeResults = nodeResults and currentResult
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
def CASE9( self, main ):
"""
@@ -2064,7 +2348,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2093,7 +2376,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2121,14 +2403,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2167,11 +2448,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2220,9 +2499,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2269,26 +2546,84 @@
time.sleep( 10 )
main.step( "Stopping Mininet" )
- main.Mininet1.stopNet()
+ mnResult = main.Mininet1.stopNet()
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet stopped",
+ onfail="MN cleanup NOT successful" )
main.step( "Checking ONOS Logs for errors" )
for node in nodes:
print colors[ 'purple' ] + "Checking logs for errors on " + \
node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address )
+ print main.ONOSbench.checkLogs( node.ip_address, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
- # TODO: actually check something here
- utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
- onpass="Test cleanup successful",
- onfail="Test cleanup NOT successful" )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
def CASE14( self, main ):
"""
start election app on all onos nodes
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
+ main.case("Start Leadership Election app")
+ main.step( "Install leadership election app" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ cli.electionTestRun()
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ if leader is None or leader == main.FALSE:
+ main.log.error( cli.name + ": Leader for the election app " +
+ "should be an ONOS node, instead got '" +
+ str( leader ) + "'" )
+ leaderResult = main.FALSE
+ leaders.append( leader )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=leaderResult,
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
+ def CASE15( self, main ):
+ """
+ Check that Leadership Election is still functional
+ """
import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
@@ -2297,51 +2632,46 @@
assert nodes, "nodes not defined"
leaderResult = main.TRUE
- main.log.info( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ description = "Check that Leadership Election is still functional"
+ main.case( description )
+ # NOTE: Need to re-run since being a canidate is not persistant
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
leaders = []
for cli in CLIs:
+ cli.electionTestRun()
+ for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
leaders.append( leader )
- if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
- main.log.error( "Results of electionTestLeader is order of CLIs:" +
- str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
- def CASE15( self, main ):
- """
- Check that Leadership Election is still functional
- """
- assert numControllers, "numControllers not defined"
- assert main, "main not defined"
- assert utilities.assert_equals, "utilities.assert_equals not defined"
- assert CLIs, "CLIs not defined"
- assert nodes, "nodes not defined"
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
- leaderResult = main.TRUE
- description = "Check that Leadership Election is still functional"
- main.log.report( description )
- main.case( description )
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2357,8 +2687,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2367,19 +2697,20 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
elif leaderN is None:
# node may not have recieved the event yet
+ time.sleep(7)
leaderN = cli.electionTestLeader()
leaderList.pop()
leaderList.append( leaderN )
@@ -2390,17 +2721,12 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2418,18 +2744,1110 @@
actual=runResult,
onpass="App re-ran for election",
onfail="App failed to run for election" )
- if consistentLeader == main.TRUE:
- afterRun = main.ONOScli1.electionTestLeader()
- # verify leader didn't just change
- if afterRun == leaderList[ 0 ]:
- leaderResult = main.TRUE
- else:
- leaderResult = main.FALSE
- # TODO: assert on run and withdraw results?
+
+ main.step( "Leader did not change when old leader re-ran" )
+ afterRun = main.ONOScli1.electionTestLeader()
+ # verify leader didn't just change
+ if afterRun == leaderList[ 0 ]:
+ afterResult = main.TRUE
+ else:
+ afterResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
+ actual=afterResult,
+ onpass="Old leader successfully re-ran for election",
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
+
+ def CASE16( self, main ):
+ """
+ Install Distributed Primitives app
+ """
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
+ # Variables for the distributed primitives tests
+ global pCounterName
+ global iCounterName
+ global pCounterValue
+ global iCounterValue
+ global onosSet
+ global onosSetName
+ pCounterName = "TestON-Partitions"
+ iCounterName = "TestON-inMemory"
+ pCounterValue = 0
+ iCounterValue = 0
+ onosSet = set([])
+ onosSetName = "TestON-set"
+
+ description = "Install Primitives app"
+ main.case( description )
+ main.step( "Install Primitives app" )
+ appName = "org.onosproject.distributedprimitives"
+ appResults = CLIs[0].activateApp( appName )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=appResults,
+ onpass="Primitives app activated",
+ onfail="Primitives app not activated" )
+ time.sleep( 5 ) # To allow all nodes to activate
+
+ def CASE17( self, main ):
+ """
+ Check for basic functionality with distributed primitives
+ """
+ # Make sure variables are defined/set
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert pCounterName, "pCounterName not defined"
+ assert iCounterName, "iCounterName not defined"
+ assert onosSetName, "onosSetName not defined"
+ # NOTE: assert fails if value is 0/None/Empty/False
+ try:
+ pCounterValue
+ except NameError:
+ main.log.error( "pCounterValue not defined, setting to 0" )
+ pCounterValue = 0
+ try:
+ iCounterValue
+ except NameError:
+ main.log.error( "iCounterValue not defined, setting to 0" )
+ iCounterValue = 0
+ try:
+ onosSet
+ except NameError:
+ main.log.error( "onosSet not defined, setting to empty Set" )
+ onosSet = set([])
+ # Variables for the distributed primitives tests. These are local only
+ addValue = "a"
+ addAllValue = "a b c d e f"
+ retainValue = "c d e f"
+
+ description = "Check for basic functionality with distributed " +\
+ "primitives"
+ main.case( description )
+ main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+ # DISTRIBUTED ATOMIC COUNTERS
+ main.step( "Increment and get a default counter on each node" )
+ pCounters = []
+ threads = []
+ addedPValues = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="counterIncrement-" + str( i ),
+ args=[ pCounterName ] )
+ pCounterValue += 1
+ addedPValues.append( pCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ pCounters.append( t.result )
+ # Check that counter incremented numController times
+ pCounterResults = True
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=pCounterResults,
+ onpass="Default counter incremented",
+ onfail="Error incrementing default" +
+ " counter" )
+
+ main.step( "Increment and get an in memory counter on each node" )
+ iCounters = []
+ addedIValues = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="icounterIncrement-" + str( i ),
+ args=[ iCounterName ],
+ kwargs={ "inMemory": True } )
+ iCounterValue += 1
+ addedIValues.append( iCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ iCounters.append( t.result )
+ # Check that counter incremented numController times
+ iCounterResults = True
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=iCounterResults,
+ onpass="In memory counter incremented",
+ onfail="Error incrementing in memory" +
+ " counter" )
+
+ main.step( "Check counters are consistant across nodes" )
+ onosCounters = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCounters.append( t.result )
+ tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+ if all( tmp ):
+ main.log.info( "Counters are consistent across all nodes" )
+ consistentCounterResults = main.TRUE
+ else:
+ main.log.error( "Counters are not consistent across all nodes" )
+ consistentCounterResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=consistentCounterResults,
+ onpass="ONOS counters are consistent " +
+ "across nodes",
+ onfail="ONOS Counters are inconsistent " +
+ "across nodes" )
+
+ main.step( "Counters we added have the correct values" )
+ correctResults = main.TRUE
+ for i in range( numControllers ):
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
+ try:
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
+ except AttributeError, e:
+ main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+ "is not as expected" )
+ correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=correctResults,
+ onpass="Added counters are correct",
+ onfail="Added counters are incorrect" )
+ # DISTRIBUTED SETS
+ main.step( "Distributed Set get" )
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=getResults,
+ onpass="Set elements are correct",
+ onfail="Set elements are incorrect" )
+
+ main.step( "Distributed Set size" )
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=sizeResults,
+ onpass="Set sizes are correct",
+ onfail="Set sizes are incorrect" )
+
+ main.step( "Distributed Set add()" )
+ onosSet.add( addValue )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAdd-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addResults = main.FALSE
+ else:
+ # unexpected result
+ addResults = main.FALSE
+ if addResults != main.TRUE:
+ main.log.error( "Error executing set add" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addResults = addResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addResults,
+ onpass="Set add correct",
+ onfail="Set add was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set contains()" )
+ containsResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContains-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsResponses.append( t.result )
+
+ containsResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsResults,
+ onpass="Set contains is functional",
+ onfail="Set contains failed" )
+
+ main.step( "Distributed Set containsAll()" )
+ containsAllResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContainsAll-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addAllValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsAllResponses.append( t.result )
+
+ containsAllResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsAllResults,
+ onpass="Set containsAll is functional",
+ onfail="Set containsAll failed" )
+
+ main.step( "Distributed Set remove()" )
+ onosSet.remove( addValue )
+ removeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemove-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeResults = main.TRUE
+ for i in range( numControllers ):
+ if removeResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeResults = main.FALSE
+ else:
+ # unexpected result
+ removeResults = main.FALSE
+ if removeResults != main.TRUE:
+ main.log.error( "Error executing set remove" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeResults = removeResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeResults,
+ onpass="Set remove correct",
+ onfail="Set remove was incorrect" )
+
+ main.step( "Distributed Set removeAll()" )
+ onosSet.difference_update( addAllValue.split() )
+ removeAllResponses = []
+ threads = []
+ try:
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemoveAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeAllResponses.append( t.result )
+ except Exception, e:
+ main.log.exception(e)
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeAllResults = main.TRUE
+ for i in range( numControllers ):
+ if removeAllResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeAllResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeAllResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeAllResults = main.FALSE
+ else:
+ # unexpected result
+ removeAllResults = main.FALSE
+ if removeAllResults != main.TRUE:
+ main.log.error( "Error executing set removeAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeAllResults = removeAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeAllResults,
+ onpass="Set removeAll correct",
+ onfail="Set removeAll was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set clear()" )
+ onosSet.clear()
+ clearResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestClear-" + str( i ),
+ args=[ onosSetName, " "], # Values doesn't matter
+ kwargs={ "clear": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ clearResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ clearResults = main.TRUE
+ for i in range( numControllers ):
+ if clearResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif clearResponses[ i ] == main.FALSE:
+ # Nothing set, probably fine
+ pass
+ elif clearResponses[ i ] == main.ERROR:
+ # Error in execution
+ clearResults = main.FALSE
+ else:
+ # unexpected result
+ clearResults = main.FALSE
+ if clearResults != main.TRUE:
+ main.log.error( "Error executing set clear" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ clearResults = clearResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=clearResults,
+ onpass="Set clear correct",
+ onfail="Set clear was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set retain()" )
+ onosSet.intersection_update( retainValue.split() )
+ retainResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRetain-" + str( i ),
+ args=[ onosSetName, retainValue ],
+ kwargs={ "retain": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ retainResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ retainResults = main.TRUE
+ for i in range( numControllers ):
+ if retainResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif retainResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif retainResponses[ i ] == main.ERROR:
+ # Error in execution
+ retainResults = main.FALSE
+ else:
+ # unexpected result
+ retainResults = main.FALSE
+ if retainResults != main.TRUE:
+ main.log.error( "Error executing set retain" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " +
+ str( size ) + " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ retainResults = retainResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=retainResults,
+ onpass="Set retain correct",
+ onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index 595b78c..956d889 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -1,11 +1,12 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git>False</Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 43dd995..af96bb5 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -18,6 +18,8 @@
CASE13: Clean up
CASE14: start election app on all onos nodes
CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
"""
@@ -44,9 +46,12 @@
start cli sessions
start tcpdump
"""
- main.log.report(
- "ONOS HA test: Restart minority of ONOS nodes - initialization" )
+ main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
+ "initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -90,36 +95,73 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
+ # Make sure ONOS is DEAD
+ main.log.info( "Killing any ONOS processes" )
+ killResults = main.TRUE
+ for node in nodes:
+ killed = main.ONOSbench.onosKill( node.ip_address )
+ killResults = killResults and killed
+
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
main.step( "Starting Mininet" )
- main.Mininet1.startNet( )
+ mnResult = main.Mininet1.startNet( )
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet Started",
+ onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
- if gitPullResult == main.ERROR:
- main.log.error( "Error pulling git branch" )
+ # values of 1 or 3 are good
+ utilities.assert_lesser( expect=0, actual=gitPullResult,
+ onpass="Git pull successful",
+ onfail="Git pull failed" )
+ main.ONOSbench.getVersion( report=True )
- main.step( "Using mvn clean & install" )
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
- main.ONOSbench.getVersion( report=True )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
+ # GRAPHS
+ # NOTE: important params here:
+ # job = name of Jenkins job
+ # Plot Name = Plot-HA, only can be used if multiple plots
+ # index = The number of the graph under plot name
+ job = "HAMinorityRestart"
+ plotName = "Plot-HA"
+ graphs = '<ac:structured-macro ac:name="html">\n'
+ graphs += '<ac:plain-text-body><![CDATA[\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
+ 'noborder="0" width="500" height="300" scrolling="yes" ' +\
+ 'seamless="seamless"></iframe>\n'
+ graphs += ']]></ac:plain-text-body>\n'
+ graphs += '</ac:structured-macro>\n'
+ main.log.wiki(graphs)
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
+ utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+ onpass="ONOS package successful",
+ onfail="ONOS package failed" )
main.step( "Installing ONOS package" )
onosInstallResult = main.TRUE
@@ -127,6 +169,9 @@
tmpResult = main.ONOSbench.onosInstall( options="-f",
node=node.ip_address )
onosInstallResult = onosInstallResult and tmpResult
+ utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+ onpass="ONOS install successful",
+ onfail="ONOS install failed" )
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
@@ -134,12 +179,15 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
+ utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+ onpass="ONOS startup successful",
+ onfail="ONOS startup failed" )
main.log.step( "Starting ONOS CLI sessions" )
cliResults = main.TRUE
@@ -154,14 +202,19 @@
for t in threads:
t.join()
cliResults = cliResults and t.result
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS cli startup successful",
+ onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ main.step( "App Ids check" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -174,22 +227,15 @@
for t in threads:
t.join()
appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults and appCheck)
-
- utilities.assert_equals( expect=main.TRUE, actual=case1Result,
- onpass="Test startup successful",
- onfail="Test startup NOT successful" )
-
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -198,6 +244,7 @@
Assign mastership to controllers
"""
import re
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -211,8 +258,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -244,16 +295,18 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
+
+ main.step( "Assign mastership of switches to specific controllers" )
# Manually assign mastership to the controller we want
roleCall = main.TRUE
- roleCheck = main.TRUE
+
+ ipList = [ ]
+ deviceList = []
try:
for i in range( 1, 29 ): # switches 1 through 28
# set up correct variables:
@@ -297,16 +350,8 @@
# TODO: make this controller dynamic
roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
ip )
- # Check assignment
- master = main.ONOScli1.getRole( deviceId ).get( 'master' )
- if ip in master:
- roleCheck = roleCheck and main.TRUE
- else:
- roleCheck = roleCheck and main.FALSE
- main.log.error( "Error, controller " + ip + " is not" +
- " master " + "of device " +
- str( deviceId ) + ". Master is " +
- repr( master ) + "." )
+ ipList.append( ip )
+ deviceList.append( deviceId )
except ( AttributeError, AssertionError ):
main.log.exception( "Something is wrong with ONOS device view" )
main.log.info( main.ONOScli1.devices() )
@@ -316,6 +361,24 @@
onpass="Re-assigned switch mastership to designated controller",
onfail="Something wrong with deviceRole calls" )
+ main.step( "Check mastership was correctly assigned" )
+ roleCheck = main.TRUE
+ # NOTE: This is due to the fact that device mastership change is not
+ # atomic and is actually a multi step process
+ time.sleep( 5 )
+ for i in range( len( ipList ) ):
+ ip = ipList[i]
+ deviceId = deviceList[i]
+ # Check assignment
+ master = main.ONOScli1.getRole( deviceId ).get( 'master' )
+ if ip in master:
+ roleCheck = roleCheck and main.TRUE
+ else:
+ roleCheck = roleCheck and main.FALSE
+ main.log.error( "Error, controller " + ip + " is not" +
+ " master " + "of device " +
+ str( deviceId ) + ". Master is " +
+ repr( master ) + "." )
utilities.assert_equals(
expect=main.TRUE,
actual=roleCheck,
@@ -339,17 +402,21 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
- main.log.info( "Install reactive forwarding app" )
- appResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+ main.step( "Install reactive forwarding app" )
+ installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=installResults,
+ onpass="Install fwd successful",
+ onfail="Install fwd failed" )
- # FIXME: add this to asserts
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -362,13 +429,15 @@
for t in threads:
t.join()
appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -378,14 +447,23 @@
expect=main.TRUE,
actual=pingResult,
onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ onfail="Reactive Pingall failed, " +
+ "one or more ping pairs failed" )
time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+ main.log.info( "Time for pingall: %2f seconds" %
+ ( time2 - time1 ) )
+ # timeout for fwd flows
+ time.sleep( 11 )
# uninstall onos-app-fwd
- main.log.info( "Uninstall reactive forwarding app" )
- appResults = appResults and CLIs[0].deactivateApp( "org.onosproject.fwd" )
+ main.step( "Uninstall reactive forwarding app" )
+ uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+ onpass="Uninstall fwd successful",
+ onfail="Uninstall fwd failed" )
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -395,18 +473,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- # timeout for fwd flows
- time.sleep( 11 )
-
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -449,7 +524,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -544,35 +622,48 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
+ main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
- onosIds = main.ONOScli1.getAllIntentsId()
+ correct = True
main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
- main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
- if sorted(onosIds) == sorted(intentIds):
+ for cli in CLIs:
+ onosIds = []
+ ids = cli.getAllIntentsId()
+ onosIds.append( ids )
+ main.log.debug( "Intents in " + cli.name + ": " +
+ str( sorted( onosIds ) ) )
+ if sorted( ids ) != sorted( intentIds ):
+ correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ correct = False
+ break
+ if correct:
break
else:
time.sleep(1)
- # FIXME: DEBUG
if not intentStop:
intentStop = time.time()
+ global gossipTime
gossipTime = intentStop - intentStart
main.log.info( "It took about " + str( gossipTime ) +
- " seconds for all intents to appear on ONOS1" )
+ " seconds for all intents to appear in each node" )
# FIXME: make this time configurable/calculate based off of number of
# nodes and gossip rounds
utilities.assert_greater_equals(
- expect=30, actual=gossipTime,
+ expect=40, actual=gossipTime,
onpass="ECM anti-entropy for intents worked within " +
"expected time",
onfail="Intent ECM anti-entropy took too long" )
- if gossipTime <= 30:
+ if gossipTime <= 40:
intentAddResult = True
if not intentAddResult or "key" in pendingMap:
@@ -675,9 +766,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -690,7 +783,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -702,17 +795,17 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
- installedCheck = True
- if PingResult is not main.TRUE:
+ main.step( "Check Intent state" )
+ installedCheck = False
+ count = 0
+ while not installedCheck and count < 40:
+ installedCheck = True
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
@@ -728,66 +821,88 @@
intentStates.append( ( intentId, state ) )
except ( ValueError, TypeError ):
main.log.exception( "Error parsing intents." )
+ # Print states
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -868,6 +983,39 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ # the wait is above
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.error(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
def CASE5( self, main ):
"""
@@ -883,7 +1031,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -931,7 +1078,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -945,7 +1092,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -993,7 +1140,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1006,34 +1153,63 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ if intentsResults:
+ # Try to make it easy to figure out what is happening
+ #
+ # Intent ONOS1 ONOS2 ...
+ # 0x01 INSTALLED INSTALLING
+ # ... ... ...
+ # ... ... ...
+ title = " Id"
+ for n in range( numControllers ):
+ title += " " * 10 + "ONOS" + str( n + 1 )
+ main.log.warn( title )
+ # get all intent keys in the cluster
+ keys = []
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id', "Error" ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End table view
+
if intentsResults and not consistentIntents:
+ # print the json objects
n = len(ONOSIntents)
- main.log.warn( "ONOS" + str( n ) + " intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.debug( "ONOS" + str( n ) + " intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
for i in range( numControllers ):
if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
- main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
else:
- main.log.warn( nodes[ i ].name + " intents match ONOS" +
- str( n ) + " intents" )
+ main.log.debug( nodes[ i ].name + " intents match ONOS" +
+ str( n ) + " intents" )
elif intentsResults and consistentIntents:
intentCheck = main.TRUE
intentState = ONOSIntents[ 0 ]
@@ -1065,7 +1241,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1089,7 +1265,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1244,6 +1420,7 @@
# Compare json objects for hosts and dataplane clusters
# hosts
+ main.step( "Host view is consistent across ONOS nodes" )
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
@@ -1251,14 +1428,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1270,11 +1447,12 @@
onpass="Hosts view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of hosts" )
+ main.step( "Each host has an IP address" )
ipResult = main.TRUE
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1285,6 +1463,7 @@
onfail="The ip of at least one host is missing" )
# Strongly connected clusters of devices
+ main.step( "Cluster view is consistent across ONOS nodes" )
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
controllerStr = str( controller + 1 )
@@ -1292,12 +1471,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1309,6 +1488,7 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
+ main.step( "Cluster view correct across ONOS nodes" )
try:
numClusters = len( json.loads( clusters[ 0 ] ) )
except ( ValueError, TypeError ):
@@ -1379,14 +1559,6 @@
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- finalAssert = main.TRUE
- finalAssert = ( finalAssert and topoResult and flowCheck
- and intentCheck and consistentMastership
- and mastershipCheck and rolesNotNull )
- utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
- onpass="State check successful",
- onfail="State check NOT successful" )
-
def CASE6( self, main ):
"""
The Failure case.
@@ -1397,15 +1569,21 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Killing 3 ONOS nodes" )
main.case( "Restart minority of ONOS nodes" )
- # TODO: Randomize these nodes
+ main.step( "Killing 3 ONOS nodes" )
+ killTime = time.time()
+ # TODO: Randomize these nodes or base this on partitions
# TODO: use threads in this case
- main.ONOSbench.onosKill( nodes[0].ip_address )
+ killResults = main.ONOSbench.onosKill( nodes[0].ip_address )
time.sleep( 10 )
- main.ONOSbench.onosKill( nodes[1].ip_address )
+ killResults = killResults and\
+ main.ONOSbench.onosKill( nodes[1].ip_address )
time.sleep( 10 )
- main.ONOSbench.onosKill( nodes[2].ip_address )
+ killResults = killResults and\
+ main.ONOSbench.onosKill( nodes[2].ip_address )
+ utilities.assert_equals( expect=main.TRUE, actual=killResults,
+ onpass="ONOS Killed successfully",
+ onfail="ONOS kill NOT successful" )
main.step( "Checking if ONOS is up yet" )
count = 0
@@ -1417,19 +1595,39 @@
onosIsupResult = onos1Isup and onos2Isup and onos3Isup
count = count + 1
# TODO: if it becomes an issue, we can retry this step a few times
+ utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+ onpass="ONOS restarted successfully",
+ onfail="ONOS restart NOT successful" )
+ main.step( "Restarting ONOS CLIs" )
cliResult1 = main.ONOScli1.startOnosCli( nodes[0].ip_address )
cliResult2 = main.ONOScli2.startOnosCli( nodes[1].ip_address )
cliResult3 = main.ONOScli3.startOnosCli( nodes[2].ip_address )
cliResults = cliResult1 and cliResult2 and cliResult3
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS cli restarted",
+ onfail="ONOS cli did not restart" )
# Grab the time of restart so we chan check how long the gossip
# protocol has had time to work
- main.restartTime = time.time()
- caseResults = main.TRUE and onosIsupResult and cliResults
- utilities.assert_equals( expect=main.TRUE, actual=caseResults,
- onpass="ONOS restart successful",
- onfail="ONOS restart NOT successful" )
+ main.restartTime = time.time() - killTime
+ main.log.debug( "Restart time: " + str( main.restartTime ) )
+ '''
+ # FIXME: revisit test plan for election with madan
+ # Rerun for election on restarted nodes
+ run1 = CLIs[0].electionTestRun()
+ run2 = CLIs[1].electionTestRun()
+ run3 = CLIs[2].electionTestRun()
+ runResults = run1 and run2 and run3
+ utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ onpass="Reran for election",
+ onfail="Failed to rerun for election" )
+ '''
+ # TODO: MAke this configurable. Also, we are breaking the above timer
+ time.sleep( 60 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1463,8 +1661,8 @@
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
+ main.step( "Read device roles from ONOS" )
ONOSMastership = []
- mastershipCheck = main.FALSE
consistentMastership = True
rolesResults = True
threads = []
@@ -1481,7 +1679,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1495,7 +1693,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1514,9 +1712,9 @@
sort_keys=True,
indent=4,
separators=( ',', ': ' ) ) )
- elif rolesResults and not consistentMastership:
- mastershipCheck = main.TRUE
+ # NOTE: we expect mastership to change on controller failure
+ '''
description2 = "Compare switch roles from before failure"
main.step( description2 )
try:
@@ -1542,15 +1740,12 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed" )
- # NOTE: we expect mastership to change on controller failure
- mastershipCheck = consistentMastership
+ '''
main.step( "Get the intents and compare across all nodes" )
ONOSIntents = []
@@ -1572,7 +1767,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1585,10 +1780,38 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
+
+ # Try to make it easy to figure out what is happening
+ #
+ # Intent ONOS1 ONOS2 ...
+ # 0x01 INSTALLED INSTALLING
+ # ... ... ...
+ # ... ... ...
+ title = " ID"
+ for n in range( numControllers ):
+ title += " " * 10 + "ONOS" + str( n + 1 )
+ main.log.warn( title )
+ # get all intent keys in the cluster
+ keys = []
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id' ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End table view
+
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1624,22 +1847,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1662,8 +1904,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1689,36 +1929,37 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
+ main.log.info( "No loss of dataplane connectivity" )
utilities.assert_equals(
expect=main.FALSE,
actual=LossInPings,
onpass="No Loss of connectivity",
onfail="Loss of dataplane connectivity detected" )
+ main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leaderList = []
# FIXME: make sure this matches nodes that were restarted
restarted = [ nodes[0].ip_address, nodes[1].ip_address,
nodes[2].ip_address ]
-
+
leaderResult = main.TRUE
for cli in CLIs:
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
- # error in response
- main.log.report( "Something is wrong with " +
+ # error in response
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( cli.name +
+ main.log.error( cli.name +
" shows no leader for the election-app was" +
" elected after the old one died" )
leaderResult = main.FALSE
elif leaderN in restarted:
- main.log.report( cli.name + " shows " + str( leaderN ) +
+ main.log.error( cli.name + " shows " + str( leaderN ) +
" as leader for the election-app, but it " +
"was restarted" )
leaderResult = main.FALSE
@@ -1727,25 +1968,12 @@
main.log.error(
"Inconsistent view of leader for the election test app" )
# TODO: print the list
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
onpass="Leadership election passed",
onfail="Something went wrong with Leadership election" )
- result = ( mastershipCheck and intentCheck and FlowTables and
- ( not LossInPings ) and rolesNotNull and leaderResult )
- result = int( result )
- if result == main.TRUE:
- main.log.report( "Constant State Tests Passed" )
- utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Constant State Tests Passed",
- onfail="Constant state tests failed" )
-
def CASE8( self, main ):
"""
Compare topo
@@ -1763,15 +1991,24 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1822,9 +2059,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -1930,6 +2167,7 @@
# Compare json objects for hosts and dataplane clusters
# hosts
+ main.step( "Hosts view is consistent across all ONOS nodes" )
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
@@ -1937,13 +2175,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1956,6 +2194,7 @@
onfail="ONOS nodes have different views of hosts" )
# Strongly connected clusters of devices
+ main.step( "Clusters view is consistent across all ONOS nodes" )
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
controllerStr = str( controller + 1 )
@@ -1963,13 +2202,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1981,6 +2220,8 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
+ main.step( "Topology view is correct and consistent across all " +
+ "ONOS nodes" )
try:
numClusters = len( json.loads( clusters[ 0 ] ) )
except ( ValueError, TypeError ):
@@ -2011,12 +2252,11 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
nodesOutput = []
+ nodeResults = main.TRUE
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].nodes,
@@ -2033,9 +2273,10 @@
try:
current = json.loads( i )
for node in current:
+ currentResult = main.FALSE
if node['ip'] in ips: # node in nodes() output is in cell
if node['state'] == 'ACTIVE':
- pass # as it should be
+ currentResult = main.TRUE
else:
main.log.error( "Error in ONOS node availability" )
main.log.error(
@@ -2044,9 +2285,13 @@
indent=4,
separators=( ',', ': ' ) ) )
break
+ nodeResults = nodeResults and currentResult
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
def CASE9( self, main ):
"""
@@ -2064,7 +2309,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2093,7 +2337,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2121,14 +2364,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2167,11 +2409,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2220,9 +2460,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2269,26 +2507,84 @@
time.sleep( 10 )
main.step( "Stopping Mininet" )
- main.Mininet1.stopNet()
+ mnResult = main.Mininet1.stopNet()
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet stopped",
+ onfail="MN cleanup NOT successful" )
main.step( "Checking ONOS Logs for errors" )
for node in nodes:
print colors[ 'purple' ] + "Checking logs for errors on " + \
node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address )
+ print main.ONOSbench.checkLogs( node.ip_address, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
- # TODO: actually check something here
- utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
- onpass="Test cleanup successful",
- onfail="Test cleanup NOT successful" )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
def CASE14( self, main ):
"""
start election app on all onos nodes
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
+ main.case("Start Leadership Election app")
+ main.step( "Install leadership election app" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ cli.electionTestRun()
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ if leader is None or leader == main.FALSE:
+ main.log.error( cli.name + ": Leader for the election app " +
+ "should be an ONOS node, instead got '" +
+ str( leader ) + "'" )
+ leaderResult = main.FALSE
+ leaders.append( leader )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=leaderResult,
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
+ def CASE15( self, main ):
+ """
+ Check that Leadership Election is still functional
+ """
import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
@@ -2297,51 +2593,31 @@
assert nodes, "nodes not defined"
leaderResult = main.TRUE
- main.log.info( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ description = "Check that Leadership Election is still functional"
+ main.case( description )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
leaders = []
for cli in CLIs:
leader = cli.electionTestLeader()
- if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
- "should be an ONOS node, instead got '" +
- str( leader ) + "'" )
- leaderResult = main.FALSE
leaders.append( leader )
if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
+ sameLeader = main.FALSE
main.log.error( "Results of electionTestLeader is order of CLIs:" +
str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
- def CASE15( self, main ):
- """
- Check that Leadership Election is still functional
- """
- assert numControllers, "numControllers not defined"
- assert main, "main not defined"
- assert utilities.assert_equals, "utilities.assert_equals not defined"
- assert CLIs, "CLIs not defined"
- assert nodes, "nodes not defined"
-
- leaderResult = main.TRUE
- description = "Check that Leadership Election is still functional"
- main.log.report( description )
- main.case( description )
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2357,8 +2633,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2367,19 +2643,20 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
elif leaderN is None:
# node may not have recieved the event yet
+ time.sleep(7)
leaderN = cli.electionTestLeader()
leaderList.pop()
leaderList.append( leaderN )
@@ -2390,17 +2667,12 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2418,18 +2690,1110 @@
actual=runResult,
onpass="App re-ran for election",
onfail="App failed to run for election" )
- if consistentLeader == main.TRUE:
- afterRun = main.ONOScli1.electionTestLeader()
- # verify leader didn't just change
- if afterRun == leaderList[ 0 ]:
- leaderResult = main.TRUE
- else:
- leaderResult = main.FALSE
- # TODO: assert on run and withdraw results?
+
+ main.step( "Leader did not change when old leader re-ran" )
+ afterRun = main.ONOScli1.electionTestLeader()
+ # verify leader didn't just change
+ if afterRun == leaderList[ 0 ]:
+ afterResult = main.TRUE
+ else:
+ afterResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
+ actual=afterResult,
+ onpass="Old leader successfully re-ran for election",
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
+
+ def CASE16( self, main ):
+ """
+ Install Distributed Primitives app
+ """
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
+ # Variables for the distributed primitives tests
+ global pCounterName
+ global iCounterName
+ global pCounterValue
+ global iCounterValue
+ global onosSet
+ global onosSetName
+ pCounterName = "TestON-Partitions"
+ iCounterName = "TestON-inMemory"
+ pCounterValue = 0
+ iCounterValue = 0
+ onosSet = set([])
+ onosSetName = "TestON-set"
+
+ description = "Install Primitives app"
+ main.case( description )
+ main.step( "Install Primitives app" )
+ appName = "org.onosproject.distributedprimitives"
+ appResults = CLIs[0].activateApp( appName )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=appResults,
+ onpass="Primitives app activated",
+ onfail="Primitives app not activated" )
+ time.sleep( 5 ) # To allow all nodes to activate
+
+ def CASE17( self, main ):
+ """
+ Check for basic functionality with distributed primitives
+ """
+ # Make sure variables are defined/set
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert pCounterName, "pCounterName not defined"
+ assert iCounterName, "iCounterName not defined"
+ assert onosSetName, "onosSetName not defined"
+ # NOTE: assert fails if value is 0/None/Empty/False
+ try:
+ pCounterValue
+ except NameError:
+ main.log.error( "pCounterValue not defined, setting to 0" )
+ pCounterValue = 0
+ try:
+ iCounterValue
+ except NameError:
+ main.log.error( "iCounterValue not defined, setting to 0" )
+ iCounterValue = 0
+ try:
+ onosSet
+ except NameError:
+ main.log.error( "onosSet not defined, setting to empty Set" )
+ onosSet = set([])
+ # Variables for the distributed primitives tests. These are local only
+ addValue = "a"
+ addAllValue = "a b c d e f"
+ retainValue = "c d e f"
+
+ description = "Check for basic functionality with distributed " +\
+ "primitives"
+ main.case( description )
+ main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+ # DISTRIBUTED ATOMIC COUNTERS
+ main.step( "Increment and get a default counter on each node" )
+ pCounters = []
+ threads = []
+ addedPValues = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="counterIncrement-" + str( i ),
+ args=[ pCounterName ] )
+ pCounterValue += 1
+ addedPValues.append( pCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ pCounters.append( t.result )
+ # Check that counter incremented numController times
+ pCounterResults = True
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=pCounterResults,
+ onpass="Default counter incremented",
+ onfail="Error incrementing default" +
+ " counter" )
+
+ main.step( "Increment and get an in memory counter on each node" )
+ iCounters = []
+ addedIValues = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="icounterIncrement-" + str( i ),
+ args=[ iCounterName ],
+ kwargs={ "inMemory": True } )
+ iCounterValue += 1
+ addedIValues.append( iCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ iCounters.append( t.result )
+ # Check that counter incremented numController times
+ iCounterResults = True
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=iCounterResults,
+ onpass="In memory counter incremented",
+ onfail="Error incrementing in memory" +
+ " counter" )
+
+ main.step( "Check counters are consistant across nodes" )
+ onosCounters = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCounters.append( t.result )
+ tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+ if all( tmp ):
+ main.log.info( "Counters are consistent across all nodes" )
+ consistentCounterResults = main.TRUE
+ else:
+ main.log.error( "Counters are not consistent across all nodes" )
+ consistentCounterResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=consistentCounterResults,
+ onpass="ONOS counters are consistent " +
+ "across nodes",
+ onfail="ONOS Counters are inconsistent " +
+ "across nodes" )
+
+ main.step( "Counters we added have the correct values" )
+ correctResults = main.TRUE
+ for i in range( numControllers ):
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
+ try:
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
+ except AttributeError, e:
+ main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+ "is not as expected" )
+ correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=correctResults,
+ onpass="Added counters are correct",
+ onfail="Added counters are incorrect" )
+ # DISTRIBUTED SETS
+ main.step( "Distributed Set get" )
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=getResults,
+ onpass="Set elements are correct",
+ onfail="Set elements are incorrect" )
+
+ main.step( "Distributed Set size" )
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=sizeResults,
+ onpass="Set sizes are correct",
+ onfail="Set sizes are incorrect" )
+
+ main.step( "Distributed Set add()" )
+ onosSet.add( addValue )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAdd-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addResults = main.FALSE
+ else:
+ # unexpected result
+ addResults = main.FALSE
+ if addResults != main.TRUE:
+ main.log.error( "Error executing set add" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addResults = addResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addResults,
+ onpass="Set add correct",
+ onfail="Set add was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set contains()" )
+ containsResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContains-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsResponses.append( t.result )
+
+ containsResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsResults,
+ onpass="Set contains is functional",
+ onfail="Set contains failed" )
+
+ main.step( "Distributed Set containsAll()" )
+ containsAllResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContainsAll-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addAllValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsAllResponses.append( t.result )
+
+ containsAllResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsAllResults,
+ onpass="Set containsAll is functional",
+ onfail="Set containsAll failed" )
+
+ main.step( "Distributed Set remove()" )
+ onosSet.remove( addValue )
+ removeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemove-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeResults = main.TRUE
+ for i in range( numControllers ):
+ if removeResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeResults = main.FALSE
+ else:
+ # unexpected result
+ removeResults = main.FALSE
+ if removeResults != main.TRUE:
+ main.log.error( "Error executing set remove" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeResults = removeResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeResults,
+ onpass="Set remove correct",
+ onfail="Set remove was incorrect" )
+
+ main.step( "Distributed Set removeAll()" )
+ onosSet.difference_update( addAllValue.split() )
+ removeAllResponses = []
+ threads = []
+ try:
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemoveAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeAllResponses.append( t.result )
+ except Exception, e:
+ main.log.exception(e)
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeAllResults = main.TRUE
+ for i in range( numControllers ):
+ if removeAllResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeAllResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeAllResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeAllResults = main.FALSE
+ else:
+ # unexpected result
+ removeAllResults = main.FALSE
+ if removeAllResults != main.TRUE:
+ main.log.error( "Error executing set removeAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeAllResults = removeAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeAllResults,
+ onpass="Set removeAll correct",
+ onfail="Set removeAll was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set clear()" )
+ onosSet.clear()
+ clearResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestClear-" + str( i ),
+ args=[ onosSetName, " "], # Values doesn't matter
+ kwargs={ "clear": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ clearResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ clearResults = main.TRUE
+ for i in range( numControllers ):
+ if clearResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif clearResponses[ i ] == main.FALSE:
+ # Nothing set, probably fine
+ pass
+ elif clearResponses[ i ] == main.ERROR:
+ # Error in execution
+ clearResults = main.FALSE
+ else:
+ # unexpected result
+ clearResults = main.FALSE
+ if clearResults != main.TRUE:
+ main.log.error( "Error executing set clear" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ clearResults = clearResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=clearResults,
+ onpass="Set clear correct",
+ onfail="Set clear was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set retain()" )
+ onosSet.intersection_update( retainValue.split() )
+ retainResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRetain-" + str( i ),
+ args=[ onosSetName, retainValue ],
+ kwargs={ "retain": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ retainResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ retainResults = main.TRUE
+ for i in range( numControllers ):
+ if retainResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif retainResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif retainResponses[ i ] == main.ERROR:
+ # Error in execution
+ retainResults = main.FALSE
+ else:
+ # unexpected result
+ retainResults = main.FALSE
+ if retainResults != main.TRUE:
+ main.log.error( "Error executing set retain" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " +
+ str( size ) + " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ retainResults = retainResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=retainResults,
+ onpass="Set retain correct",
+ onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
index e1421cd..f82aba3 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
@@ -25,6 +25,7 @@
<Git> False </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
index 3c4541c..d2a2ec1 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
@@ -120,12 +120,27 @@
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
- main.step( "Using mvn clean & install" )
+ main.step( "Using mvn clean and install" )
cleanInstallResult = main.ONOSbench.cleanInstall()
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+ # GRAPHS
+ # NOTE: important params here:
+ # job = name of Jenkins job
+ # Plot Name = Plot-HA, only can be used if multiple plots
+ # index = The number of the graph under plot name
+ job = "HANetworkPartition"
+ graphs = '<ac:structured-macro ac:name="html">\n'
+ graphs += '<ac:plain-text-body><![CDATA[\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+ '/plot/getPlot?index=0&width=500&height=300"' +\
+ 'noborder="0" width="500" height="300" scrolling="yes" '+\
+ 'seamless="seamless"></iframe>\n'
+ graphs += ']]></ac:plain-text-body>\n'
+ graphs += '</ac:structured-macro>\n'
+ main.log.wiki(graphs)
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
@@ -202,12 +217,13 @@
cliResults = cliResult1 and cliResult2 and cliResult3 and\
cliResult4 and cliResult5 and cliResult6 and cliResult7
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
case1Result = ( cleanInstallResult and packageResult and
cellResult and verifyResult and onosInstallResult
@@ -218,6 +234,7 @@
onfail="Test startup NOT successful" )
if case1Result == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -1414,7 +1431,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host[ 'ips' ] == []:
+ if host[ 'ipAddresses' ] == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index ee4374d..9106d68 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,11 +1,31 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ #List of test cases:
+ #CASE1: Compile ONOS and push it to the test machines
+ #CASE2: Assign mastership to controllers
+ #CASE3: Assign intents
+ #CASE4: Ping across added host intents
+ #CASE5: Reading state of ONOS
+ #CASE6: The Failure case. Since this is the Sanity test, we do nothing.
+ #CASE7: Check state after control plane failure
+ #CASE8: Compare topo
+ #CASE9: Link s3-s28 down
+ #CASE10: Link s3-s28 up
+ #CASE11: Switch down
+ #CASE12: Switch up
+ #CASE13: Clean up
+ #CASE14: start election app on all onos nodes
+ #CASE15: Check that Leadership Election is still functional
+ #CASE16: Install Distributed Primitives app
+ #CASE17: Check for basic functionality with distributed primitives
+ #1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13
+ <testcases>1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git> True </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 20f6cc8..60c438e 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -19,6 +19,8 @@
CASE13: Clean up
CASE14: start election app on all onos nodes
CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
"""
@@ -45,8 +47,11 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA Sanity test - initialization" )
+ main.log.info( "ONOS HA Sanity test - initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -90,36 +95,73 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
+ # Make sure ONOS is DEAD
+ main.log.info( "Killing any ONOS processes" )
+ killResults = main.TRUE
+ for node in nodes:
+ killed = main.ONOSbench.onosKill( node.ip_address )
+ killResults = killResults and killed
+
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
main.step( "Starting Mininet" )
- main.Mininet1.startNet( )
+ mnResult = main.Mininet1.startNet( )
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet Started",
+ onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
- if gitPullResult == main.ERROR:
- main.log.error( "Error pulling git branch" )
+ # values of 1 or 3 are good
+ utilities.assert_lesser( expect=0, actual=gitPullResult,
+ onpass="Git pull successful",
+ onfail="Git pull failed" )
+ main.ONOSbench.getVersion( report=True )
- main.step( "Using mvn clean & install" )
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
- main.ONOSbench.getVersion( report=True )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
+ # GRAPHS
+ # NOTE: important params here:
+ # job = name of Jenkins job
+ # Plot Name = Plot-HA, only can be used if multiple plots
+ # index = The number of the graph under plot name
+ job = "HASanity"
+ plotName = "Plot-HA"
+ graphs = '<ac:structured-macro ac:name="html">\n'
+ graphs += '<ac:plain-text-body><![CDATA[\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
+ 'noborder="0" width="500" height="300" scrolling="yes" ' +\
+ 'seamless="seamless"></iframe>\n'
+ graphs += ']]></ac:plain-text-body>\n'
+ graphs += '</ac:structured-macro>\n'
+ main.log.wiki(graphs)
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
+ utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+ onpass="ONOS package successful",
+ onfail="ONOS package failed" )
main.step( "Installing ONOS package" )
onosInstallResult = main.TRUE
@@ -127,6 +169,9 @@
tmpResult = main.ONOSbench.onosInstall( options="-f",
node=node.ip_address )
onosInstallResult = onosInstallResult and tmpResult
+ utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+ onpass="ONOS install successful",
+ onfail="ONOS install failed" )
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
@@ -134,12 +179,15 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
+ utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
+ onpass="ONOS startup successful",
+ onfail="ONOS startup failed" )
main.log.step( "Starting ONOS CLI sessions" )
cliResults = main.TRUE
@@ -154,14 +202,19 @@
for t in threads:
t.join()
cliResults = cliResults and t.result
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS cli startup successful",
+ onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ main.step( "App Ids check" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -174,22 +227,15 @@
for t in threads:
t.join()
appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults and appCheck)
-
- utilities.assert_equals( expect=main.TRUE, actual=case1Result,
- onpass="Test startup successful",
- onfail="Test startup NOT successful" )
-
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -198,6 +244,7 @@
Assign mastership to controllers
"""
import re
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -211,8 +258,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -244,16 +295,19 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
+ # FIXME: Check topo here
+
+ main.step( "Assign mastership of switches to specific controllers" )
# Manually assign mastership to the controller we want
roleCall = main.TRUE
- roleCheck = main.TRUE
+
+ ipList = [ ]
+ deviceList = []
try:
for i in range( 1, 29 ): # switches 1 through 28
# set up correct variables:
@@ -297,16 +351,8 @@
# TODO: make this controller dynamic
roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
ip )
- # Check assignment
- master = main.ONOScli1.getRole( deviceId ).get( 'master' )
- if ip in master:
- roleCheck = roleCheck and main.TRUE
- else:
- roleCheck = roleCheck and main.FALSE
- main.log.error( "Error, controller " + ip + " is not" +
- " master " + "of device " +
- str( deviceId ) + ". Master is " +
- repr( master ) + "." )
+ ipList.append( ip )
+ deviceList.append( deviceId )
except ( AttributeError, AssertionError ):
main.log.exception( "Something is wrong with ONOS device view" )
main.log.info( main.ONOScli1.devices() )
@@ -316,6 +362,24 @@
onpass="Re-assigned switch mastership to designated controller",
onfail="Something wrong with deviceRole calls" )
+ main.step( "Check mastership was correctly assigned" )
+ roleCheck = main.TRUE
+ # NOTE: This is due to the fact that device mastership change is not
+ # atomic and is actually a multi step process
+ time.sleep( 5 )
+ for i in range( len( ipList ) ):
+ ip = ipList[i]
+ deviceId = deviceList[i]
+ # Check assignment
+ master = main.ONOScli1.getRole( deviceId ).get( 'master' )
+ if ip in master:
+ roleCheck = roleCheck and main.TRUE
+ else:
+ roleCheck = roleCheck and main.FALSE
+ main.log.error( "Error, controller " + ip + " is not" +
+ " master " + "of device " +
+ str( deviceId ) + ". Master is " +
+ repr( master ) + "." )
utilities.assert_equals(
expect=main.TRUE,
actual=roleCheck,
@@ -339,17 +403,21 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
- main.log.info( "Install reactive forwarding app" )
- appResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+ main.step( "Install reactive forwarding app" )
+ installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=installResults,
+ onpass="Install fwd successful",
+ onfail="Install fwd failed" )
- # FIXME: add this to asserts
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -362,13 +430,15 @@
for t in threads:
t.join()
appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -378,14 +448,28 @@
expect=main.TRUE,
actual=pingResult,
onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ onfail="Reactive Pingall failed, " +
+ "one or more ping pairs failed" )
time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+ main.log.info( "Time for pingall: %2f seconds" %
+ ( time2 - time1 ) )
+ # timeout for fwd flows
+ time.sleep( 11 )
# uninstall onos-app-fwd
- main.log.info( "Uninstall reactive forwarding app" )
- appResults = appResults and CLIs[0].deactivateApp( "org.onosproject.fwd" )
+ main.step( "Uninstall reactive forwarding app" )
+ uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+ onpass="Uninstall fwd successful",
+ onfail="Uninstall fwd failed" )
+ '''
+ main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
+ import time
+ time.sleep(60)
+ '''
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -395,18 +479,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
- onpass="App Ids seem to be correct",
- onfail="Something is wrong with app Ids" )
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- # timeout for fwd flows
- time.sleep( 11 )
-
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -449,7 +530,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -544,35 +628,49 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
- for i in range(100):
- onosIds = main.ONOScli1.getAllIntentsId()
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
+ main.step( "Intent Anti-Entropy dispersion" )
+ for i in range(80):
+ correct = True
main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
- main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
- if sorted(onosIds) == sorted(intentIds):
+ for cli in CLIs:
+ onosIds = []
+ ids = cli.getAllIntentsId()
+ onosIds.append( ids )
+ main.log.debug( "Intents in " + cli.name + ": " +
+ str( sorted( onosIds ) ) )
+ if sorted( ids ) != sorted( intentIds ):
+ main.log.debug( "Set of intent IDs doesn't match" )
+ correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ main.log.warn( "Intent " + intent[ 'id' ] +
+ " is " + intent[ 'state' ] )
+ correct = False
+ break
+ if correct:
break
- else:
- time.sleep(1)
- # FIXME: DEBUG
if not intentStop:
intentStop = time.time()
+ global gossipTime
gossipTime = intentStop - intentStart
main.log.info( "It took about " + str( gossipTime ) +
- " seconds for all intents to appear on ONOS1" )
+ " seconds for all intents to appear in each node" )
# FIXME: make this time configurable/calculate based off of number of
# nodes and gossip rounds
utilities.assert_greater_equals(
- expect=30, actual=gossipTime,
+ expect=40, actual=gossipTime,
onpass="ECM anti-entropy for intents worked within " +
"expected time",
onfail="Intent ECM anti-entropy took too long" )
- if gossipTime <= 30:
+ if gossipTime <= 40:
intentAddResult = True
if not intentAddResult or "key" in pendingMap:
@@ -675,9 +773,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -690,7 +790,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -702,17 +802,17 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
- installedCheck = True
- if PingResult is not main.TRUE:
+ main.step( "Check Intent state" )
+ installedCheck = False
+ count = 0
+ while not installedCheck and count < 40:
+ installedCheck = True
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
@@ -728,66 +828,88 @@
intentStates.append( ( intentId, state ) )
except ( ValueError, TypeError ):
main.log.exception( "Error parsing intents." )
+ # Print states
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -868,6 +990,39 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ # the wait is above
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.error(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
def CASE5( self, main ):
"""
@@ -883,7 +1038,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -931,7 +1085,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -945,7 +1099,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -993,7 +1147,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1006,34 +1160,68 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ if intentsResults:
+ # Try to make it easy to figure out what is happening
+ #
+ # Intent ONOS1 ONOS2 ...
+ # 0x01 INSTALLED INSTALLING
+ # ... ... ...
+ # ... ... ...
+ title = " Id"
+ for n in range( numControllers ):
+ title += " " * 10 + "ONOS" + str( n + 1 )
+ main.log.warn( title )
+ keys = []
+ try:
+ # Get the set of all intent keys
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ # For each intent key, print the state on each node
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id', "Error" ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End of intent state table
+ except ValueError as e:
+ main.log.exception( e )
+ main.log.debug( "nodeStr was: " + repr( nodeStr ) )
+
if intentsResults and not consistentIntents:
+ # print the json objects
n = len(ONOSIntents)
- main.log.warn( "ONOS" + str( n ) + " intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.debug( "ONOS" + str( n ) + " intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
for i in range( numControllers ):
if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
- main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
else:
- main.log.warn( nodes[ i ].name + " intents match ONOS" +
- str( n ) + " intents" )
+ main.log.debug( nodes[ i ].name + " intents match ONOS" +
+ str( n ) + " intents" )
elif intentsResults and consistentIntents:
intentCheck = main.TRUE
intentState = ONOSIntents[ 0 ]
@@ -1065,7 +1253,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1089,7 +1277,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1244,6 +1432,7 @@
# Compare json objects for hosts and dataplane clusters
# hosts
+ main.step( "Host view is consistent across ONOS nodes" )
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
@@ -1251,14 +1440,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1270,11 +1459,12 @@
onpass="Hosts view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of hosts" )
+ main.step( "Each host has an IP address" )
ipResult = main.TRUE
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1285,6 +1475,7 @@
onfail="The ip of at least one host is missing" )
# Strongly connected clusters of devices
+ main.step( "Cluster view is consistent across ONOS nodes" )
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
controllerStr = str( controller + 1 )
@@ -1292,12 +1483,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1309,6 +1500,7 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
+ main.step( "Cluster view correct across ONOS nodes" )
try:
numClusters = len( json.loads( clusters[ 0 ] ) )
except ( ValueError, TypeError ):
@@ -1379,14 +1571,6 @@
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- finalAssert = main.TRUE
- finalAssert = ( finalAssert and topoResult and flowCheck
- and intentCheck and consistentMastership
- and mastershipCheck and rolesNotNull )
- utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
- onpass="State check successful",
- onfail="State check NOT successful" )
-
def CASE6( self, main ):
"""
The Failure case. Since this is the Sanity test, we do nothing.
@@ -1397,7 +1581,7 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Wait 60 seconds instead of inducing a failure" )
+ main.case( "Wait 60 seconds instead of inducing a failure" )
time.sleep( 60 )
utilities.assert_equals(
expect=main.TRUE,
@@ -1437,6 +1621,7 @@
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
+ main.step( "Read device roles from ONOS" )
ONOSMastership = []
mastershipCheck = main.FALSE
consistentMastership = True
@@ -1455,7 +1640,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1469,7 +1654,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1516,8 +1701,6 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -1545,7 +1728,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1558,10 +1741,38 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
+
+ # Try to make it easy to figure out what is happening
+ #
+ # Intent ONOS1 ONOS2 ...
+ # 0x01 INSTALLED INSTALLING
+ # ... ... ...
+ # ... ... ...
+ title = " ID"
+ for n in range( numControllers ):
+ title += " " * 10 + "ONOS" + str( n + 1 )
+ main.log.warn( title )
+ # get all intent keys in the cluster
+ keys = []
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id' ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End table view
+
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1597,22 +1808,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1635,8 +1865,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1662,13 +1890,14 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
+ main.log.info( "No loss of dataplane connectivity" )
utilities.assert_equals(
expect=main.FALSE,
actual=LossInPings,
onpass="No Loss of connectivity",
onfail="Loss of dataplane connectivity detected" )
+ main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
# NOTE: this only works for the sanity test. In case of failures,
# leader will likely change
@@ -1684,34 +1913,21 @@
pass
elif leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( cli.name + " sees " + str( leaderN ) +
+ main.log.error( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
onpass="Leadership election passed",
onfail="Something went wrong with Leadership election" )
- result = ( mastershipCheck and intentCheck and FlowTables and
- ( not LossInPings ) and rolesNotNull and leaderResult )
- result = int( result )
- if result == main.TRUE:
- main.log.report( "Constant State Tests Passed" )
- utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Constant State Tests Passed",
- onfail="Constant state tests failed" )
-
def CASE8( self, main ):
"""
Compare topo
@@ -1729,15 +1945,24 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1788,9 +2013,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -1896,6 +2121,7 @@
# Compare json objects for hosts and dataplane clusters
# hosts
+ main.step( "Hosts view is consistent across all ONOS nodes" )
consistentHostsResult = main.TRUE
for controller in range( len( hosts ) ):
controllerStr = str( controller + 1 )
@@ -1903,13 +2129,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1922,6 +2148,7 @@
onfail="ONOS nodes have different views of hosts" )
# Strongly connected clusters of devices
+ main.step( "Clusters view is consistent across all ONOS nodes" )
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
controllerStr = str( controller + 1 )
@@ -1929,13 +2156,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1947,6 +2174,8 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
+ main.step( "Topology view is correct and consistent across all " +
+ "ONOS nodes" )
try:
numClusters = len( json.loads( clusters[ 0 ] ) )
except ( ValueError, TypeError ):
@@ -1977,12 +2206,11 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
nodesOutput = []
+ nodeResults = main.TRUE
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].nodes,
@@ -1999,9 +2227,10 @@
try:
current = json.loads( i )
for node in current:
+ currentResult = main.FALSE
if node['ip'] in ips: # node in nodes() output is in cell
if node['state'] == 'ACTIVE':
- pass # as it should be
+ currentResult = main.TRUE
else:
main.log.error( "Error in ONOS node availability" )
main.log.error(
@@ -2010,9 +2239,13 @@
indent=4,
separators=( ',', ': ' ) ) )
break
+ nodeResults = nodeResults and currentResult
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
def CASE9( self, main ):
"""
@@ -2030,7 +2263,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2059,7 +2291,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2087,14 +2318,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2133,11 +2363,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2186,9 +2414,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2235,7 +2461,10 @@
time.sleep( 10 )
main.step( "Stopping Mininet" )
- main.Mininet1.stopNet()
+ mnResult = main.Mininet1.stopNet()
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet stopped",
+ onfail="MN cleanup NOT successful" )
main.step( "Checking ONOS Logs for errors" )
for node in nodes:
@@ -2246,15 +2475,70 @@
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
- # TODO: actually check something here
- utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
- onpass="Test cleanup successful",
- onfail="Test cleanup NOT successful" )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ labels = "Gossip Intents"
+ data = str( gossipTime )
+ timerLog.write( labels + "\n" + data )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
def CASE14( self, main ):
"""
start election app on all onos nodes
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
+ main.case("Start Leadership Election app")
+ main.step( "Install leadership election app" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ cli.electionTestRun()
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ if leader is None or leader == main.FALSE:
+ main.log.error( cli.name + ": Leader for the election app " +
+ "should be an ONOS node, instead got '" +
+ str( leader ) + "'" )
+ leaderResult = main.FALSE
+ leaders.append( leader )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=leaderResult,
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
+ def CASE15( self, main ):
+ """
+ Check that Leadership Election is still functional
+ """
import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
@@ -2263,51 +2547,31 @@
assert nodes, "nodes not defined"
leaderResult = main.TRUE
- main.log.info( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ description = "Check that Leadership Election is still functional"
+ main.case( description )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
leaders = []
for cli in CLIs:
leader = cli.electionTestLeader()
- if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
- "should be an ONOS node, instead got '" +
- str( leader ) + "'" )
- leaderResult = main.FALSE
leaders.append( leader )
if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
+ sameLeader = main.FALSE
main.log.error( "Results of electionTestLeader is order of CLIs:" +
str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
- def CASE15( self, main ):
- """
- Check that Leadership Election is still functional
- """
- assert numControllers, "numControllers not defined"
- assert main, "main not defined"
- assert utilities.assert_equals, "utilities.assert_equals not defined"
- assert CLIs, "CLIs not defined"
- assert nodes, "nodes not defined"
-
- leaderResult = main.TRUE
- description = "Check that Leadership Election is still functional"
- main.log.report( description )
- main.case( description )
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2323,8 +2587,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2333,19 +2597,20 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
elif leaderN is None:
# node may not have recieved the event yet
+ time.sleep(7)
leaderN = cli.electionTestLeader()
leaderList.pop()
leaderList.append( leaderN )
@@ -2356,17 +2621,12 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2384,18 +2644,1110 @@
actual=runResult,
onpass="App re-ran for election",
onfail="App failed to run for election" )
- if consistentLeader == main.TRUE:
- afterRun = main.ONOScli1.electionTestLeader()
- # verify leader didn't just change
- if afterRun == leaderList[ 0 ]:
- leaderResult = main.TRUE
- else:
- leaderResult = main.FALSE
- # TODO: assert on run and withdraw results?
+
+ main.step( "Leader did not change when old leader re-ran" )
+ afterRun = main.ONOScli1.electionTestLeader()
+ # verify leader didn't just change
+ if afterRun == leaderList[ 0 ]:
+ afterResult = main.TRUE
+ else:
+ afterResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
+ actual=afterResult,
+ onpass="Old leader successfully re-ran for election",
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
+
+ def CASE16( self, main ):
+ """
+ Install Distributed Primitives app
+ """
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
+ # Variables for the distributed primitives tests
+ global pCounterName
+ global iCounterName
+ global pCounterValue
+ global iCounterValue
+ global onosSet
+ global onosSetName
+ pCounterName = "TestON-Partitions"
+ iCounterName = "TestON-inMemory"
+ pCounterValue = 0
+ iCounterValue = 0
+ onosSet = set([])
+ onosSetName = "TestON-set"
+
+ description = "Install Primitives app"
+ main.case( description )
+ main.step( "Install Primitives app" )
+ appName = "org.onosproject.distributedprimitives"
+ appResults = CLIs[0].activateApp( appName )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=appResults,
+ onpass="Primitives app activated",
+ onfail="Primitives app not activated" )
+ time.sleep( 5 ) # To allow all nodes to activate
+
+ def CASE17( self, main ):
+ """
+ Check for basic functionality with distributed primitives
+ """
+ # Make sure variables are defined/set
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert pCounterName, "pCounterName not defined"
+ assert iCounterName, "iCounterName not defined"
+ assert onosSetName, "onosSetName not defined"
+ # NOTE: assert fails if value is 0/None/Empty/False
+ try:
+ pCounterValue
+ except NameError:
+ main.log.error( "pCounterValue not defined, setting to 0" )
+ pCounterValue = 0
+ try:
+ iCounterValue
+ except NameError:
+ main.log.error( "iCounterValue not defined, setting to 0" )
+ iCounterValue = 0
+ try:
+ onosSet
+ except NameError:
+ main.log.error( "onosSet not defined, setting to empty Set" )
+ onosSet = set([])
+ # Variables for the distributed primitives tests. These are local only
+ addValue = "a"
+ addAllValue = "a b c d e f"
+ retainValue = "c d e f"
+
+ description = "Check for basic functionality with distributed " +\
+ "primitives"
+ main.case( description )
+ main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+ # DISTRIBUTED ATOMIC COUNTERS
+ main.step( "Increment and get a default counter on each node" )
+ pCounters = []
+ threads = []
+ addedPValues = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="counterIncrement-" + str( i ),
+ args=[ pCounterName ] )
+ pCounterValue += 1
+ addedPValues.append( pCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ pCounters.append( t.result )
+ # Check that counter incremented numController times
+ pCounterResults = True
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=pCounterResults,
+ onpass="Default counter incremented",
+ onfail="Error incrementing default" +
+ " counter" )
+
+ main.step( "Increment and get an in memory counter on each node" )
+ iCounters = []
+ addedIValues = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="icounterIncrement-" + str( i ),
+ args=[ iCounterName ],
+ kwargs={ "inMemory": True } )
+ iCounterValue += 1
+ addedIValues.append( iCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ iCounters.append( t.result )
+ # Check that counter incremented numController times
+ iCounterResults = True
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=iCounterResults,
+ onpass="In memory counter incremented",
+ onfail="Error incrementing in memory" +
+ " counter" )
+
+ main.step( "Check counters are consistant across nodes" )
+ onosCounters = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCounters.append( t.result )
+ tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+ if all( tmp ):
+ main.log.info( "Counters are consistent across all nodes" )
+ consistentCounterResults = main.TRUE
+ else:
+ main.log.error( "Counters are not consistent across all nodes" )
+ consistentCounterResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=consistentCounterResults,
+ onpass="ONOS counters are consistent " +
+ "across nodes",
+ onfail="ONOS Counters are inconsistent " +
+ "across nodes" )
+
+ main.step( "Counters we added have the correct values" )
+ correctResults = main.TRUE
+ for i in range( numControllers ):
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
+ try:
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
+ except AttributeError, e:
+ main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+ "is not as expected" )
+ correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=correctResults,
+ onpass="Added counters are correct",
+ onfail="Added counters are incorrect" )
+ # DISTRIBUTED SETS
+ main.step( "Distributed Set get" )
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=getResults,
+ onpass="Set elements are correct",
+ onfail="Set elements are incorrect" )
+
+ main.step( "Distributed Set size" )
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=sizeResults,
+ onpass="Set sizes are correct",
+ onfail="Set sizes are incorrect" )
+
+ main.step( "Distributed Set add()" )
+ onosSet.add( addValue )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAdd-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addResults = main.FALSE
+ else:
+ # unexpected result
+ addResults = main.FALSE
+ if addResults != main.TRUE:
+ main.log.error( "Error executing set add" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addResults = addResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addResults,
+ onpass="Set add correct",
+ onfail="Set add was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set contains()" )
+ containsResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContains-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsResponses.append( t.result )
+
+ containsResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsResults,
+ onpass="Set contains is functional",
+ onfail="Set contains failed" )
+
+ main.step( "Distributed Set containsAll()" )
+ containsAllResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContainsAll-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addAllValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsAllResponses.append( t.result )
+
+ containsAllResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsAllResults,
+ onpass="Set containsAll is functional",
+ onfail="Set containsAll failed" )
+
+ main.step( "Distributed Set remove()" )
+ onosSet.remove( addValue )
+ removeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemove-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeResults = main.TRUE
+ for i in range( numControllers ):
+ if removeResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeResults = main.FALSE
+ else:
+ # unexpected result
+ removeResults = main.FALSE
+ if removeResults != main.TRUE:
+ main.log.error( "Error executing set remove" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeResults = removeResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeResults,
+ onpass="Set remove correct",
+ onfail="Set remove was incorrect" )
+
+ main.step( "Distributed Set removeAll()" )
+ onosSet.difference_update( addAllValue.split() )
+ removeAllResponses = []
+ threads = []
+ try:
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemoveAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeAllResponses.append( t.result )
+ except Exception, e:
+ main.log.exception(e)
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeAllResults = main.TRUE
+ for i in range( numControllers ):
+ if removeAllResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeAllResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeAllResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeAllResults = main.FALSE
+ else:
+ # unexpected result
+ removeAllResults = main.FALSE
+ if removeAllResults != main.TRUE:
+ main.log.error( "Error executing set removeAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeAllResults = removeAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeAllResults,
+ onpass="Set removeAll correct",
+ onfail="Set removeAll was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set clear()" )
+ onosSet.clear()
+ clearResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestClear-" + str( i ),
+ args=[ onosSetName, " "], # Values doesn't matter
+ kwargs={ "clear": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ clearResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ clearResults = main.TRUE
+ for i in range( numControllers ):
+ if clearResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif clearResponses[ i ] == main.FALSE:
+ # Nothing set, probably fine
+ pass
+ elif clearResponses[ i ] == main.ERROR:
+ # Error in execution
+ clearResults = main.FALSE
+ else:
+ # unexpected result
+ clearResults = main.FALSE
+ if clearResults != main.TRUE:
+ main.log.error( "Error executing set clear" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ clearResults = clearResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=clearResults,
+ onpass="Set clear correct",
+ onfail="Set clear was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set retain()" )
+ onosSet.intersection_update( retainValue.split() )
+ retainResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRetain-" + str( i ),
+ args=[ onosSetName, retainValue ],
+ kwargs={ "retain": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ retainResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ retainResults = main.TRUE
+ for i in range( numControllers ):
+ if retainResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif retainResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif retainResponses[ i ] == main.ERROR:
+ # Error in execution
+ retainResults = main.FALSE
+ else:
+ # unexpected result
+ retainResults = main.FALSE
+ if retainResults != main.TRUE:
+ main.log.error( "Error executing set retain" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " +
+ str( size ) + " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ retainResults = retainResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=retainResults,
+ onpass="Set retain correct",
+ onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
index d3eb8dd..108ab88 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
@@ -16,13 +16,14 @@
#CASE15: Check that Leadership Election is still functional
#1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13
#extra hosts test 1,2,8,11,8,12,8
- <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,15,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git>False</Git>
<branch> master </branch>
<num_controllers> 1 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index d2f0bc9..8f3c604 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -18,6 +18,8 @@
CASE13: Clean up
CASE14: start election app on all onos nodes
CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
"""
@@ -44,9 +46,12 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS Single node cluster restart " +
+ main.log.info( "ONOS Single node cluster restart " +
"HA test - initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -90,69 +95,117 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
+ # Make sure ONOS is DEAD
+ main.log.info( "Killing any ONOS processes" )
+ killResults = main.TRUE
+ for node in nodes:
+ killed = main.ONOSbench.onosKill( node.ip_address )
+ killResults = killResults and killed
+
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
main.step( "Starting Mininet" )
- main.Mininet1.startNet( )
+ mnResult = main.Mininet1.startNet( )
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet Started",
+ onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
- if gitPullResult == main.ERROR:
- main.log.error( "Error pulling git branch" )
+ # values of 1 or 3 are good
+ utilities.assert_lesser( expect=0, actual=gitPullResult,
+ onpass="Git pull successful",
+ onfail="Git pull failed" )
+ main.ONOSbench.getVersion( report=True )
- main.step( "Using mvn clean & install" )
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
- main.ONOSbench.getVersion( report=True )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
+ # GRAPHS
+ # NOTE: important params here:
+ # job = name of Jenkins job
+ # Plot Name = Plot-HA, only can be used if multiple plots
+ # index = The number of the graph under plot name
+ job = "HASingleInstanceRestart"
+ plotName = "Plot-HA"
+ graphs = '<ac:structured-macro ac:name="html">\n'
+ graphs += '<ac:plain-text-body><![CDATA[\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
+ 'noborder="0" width="500" height="300" scrolling="yes" ' +\
+ 'seamless="seamless"></iframe>\n'
+ graphs += ']]></ac:plain-text-body>\n'
+ graphs += '</ac:structured-macro>\n'
+ main.log.wiki(graphs)
cellResult = main.ONOSbench.setCell( "SingleHA" )
verifyResult = main.ONOSbench.verifyCell()
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
+ utilities.assert_equals( expect=main.TRUE, actual=packageResult,
+ onpass="ONOS package successful",
+ onfail="ONOS package failed" )
main.step( "Installing ONOS package" )
- onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
+ onosInstallResult = main.ONOSbench.onosInstall( options="-f",
node=ONOS1Ip )
+ utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
+ onpass="ONOS install successful",
+ onfail="ONOS install failed" )
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
onos1Isup = main.ONOSbench.isup( ONOS1Ip )
if onos1Isup:
break
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
+ utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
+ onpass="ONOS startup successful",
+ onfail="ONOS startup failed" )
- cliResult = main.ONOScli1.startOnosCli( ONOS1Ip )
+ main.log.step( "Starting ONOS CLI sessions" )
+ cliResults = main.ONOScli1.startOnosCli( ONOS1Ip )
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS cli startup successful",
+ onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onos1InstallResult
- and onos1Isup and cliResult )
+ main.step( "App Ids check" )
+ appCheck = main.ONOScli1.appToIDCheck()
+ if appCheck != main.TRUE:
+ main.log.warn( CLIs[0].apps() )
+ main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- utilities.assert_equals( expect=main.TRUE, actual=case1Result,
- onpass="Test startup successful",
- onfail="Test startup NOT successful" )
-
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -172,8 +225,10 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device."
main.step( "Assign switches to controllers" )
for i in range( 1, 29 ):
@@ -193,13 +248,14 @@
else:
mastershipCheck = main.FALSE
if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
+ main.log.info( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
+ main.step( "Assign mastership of switches to specific controllers" )
roleCall = main.TRUE
roleCheck = main.TRUE
try:
@@ -237,7 +293,7 @@
roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
ip )
# Check assignment
- master = main.ONOScli1.getRole( deviceId ).get( 'master' )
+ master = main.ONOScli1.getRole( deviceId ).get( 'master' )
if ip in master:
roleCheck = roleCheck and main.TRUE
else:
@@ -255,6 +311,7 @@
onpass="Re-assigned switch mastership to designated controller",
onfail="Something wrong with deviceRole calls" )
+ main.step( "Check mastership was correctly assigned" )
utilities.assert_equals(
expect=main.TRUE,
actual=roleCheck,
@@ -276,24 +333,33 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- # FIXME: we must reinstall intents until we have a persistant
- # datastore!
- main.log.report( "Adding host intents" )
+ # NOTE: we must reinstall intents until we have a persistant intent
+ # datastore!
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
- main.log.info( "Install reactive forwarding app" )
- appResults = main.ONOScli1.activateApp( "org.onosproject.fwd" )
+ main.step( "Install reactive forwarding app" )
+ installResults = CLIs[0].activateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=installResults,
+ onpass="Install fwd successful",
+ onfail="Install fwd failed" )
- # FIXME: add this to asserts
+ main.step( "Check app ids" )
appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -303,22 +369,30 @@
expect=main.TRUE,
actual=pingResult,
onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ onfail="Reactive Pingall failed, " +
+ "one or more ping pairs failed" )
time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
-
+ main.log.info( "Time for pingall: %2f seconds" %
+ ( time2 - time1 ) )
+ # timeout for fwd flows
+ time.sleep( 11 )
# uninstall onos-app-fwd
- main.log.info( "Uninstall reactive forwarding app" )
- appResults = appResults and main.ONOScli1.deactivateApp( "org.onosproject.fwd" )
+ main.step( "Uninstall reactive forwarding app" )
+ uninstallResult = CLIs[0].deactivateApp( "org.onosproject.fwd" )
+ utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
+ onpass="Uninstall fwd successful",
+ onfail="Uninstall fwd failed" )
+
+ main.step( "Check app ids" )
appCheck2 = main.ONOScli1.appToIDCheck()
if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
+ onpass="App Ids seem to be correct",
+ onfail="Something is wrong with app Ids" )
- # timeout for fwd flows
- time.sleep( 11 )
-
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -360,7 +434,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -455,13 +532,42 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
+ main.step( "Intent Anti-Entropy dispersion" )
+ for i in range(100):
+ correct = True
+ main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
+ for cli in CLIs:
+ onosIds = []
+ ids = cli.getAllIntentsId()
+ onosIds.append( ids )
+ main.log.debug( "Intents in " + cli.name + ": " +
+ str( sorted( onosIds ) ) )
+ if sorted( ids ) != sorted( intentIds ):
+ correct = False
+ if correct:
+ break
+ else:
+ time.sleep(1)
+ if not intentStop:
+ intentStop = time.time()
+ global gossipTime
+ gossipTime = intentStop - intentStart
+ main.log.info( "It took about " + str( gossipTime ) +
+ " seconds for all intents to appear in each node" )
+ # FIXME: make this time configurable/calculate based off of number of
+ # nodes and gossip rounds
+ utilities.assert_greater_equals(
+ expect=40, actual=gossipTime,
+ onpass="ECM anti-entropy for intents worked within " +
+ "expected time",
+ onfail="Intent ECM anti-entropy took too long" )
+ if gossipTime <= 40:
+ intentAddResult = True
if not intentAddResult or "key" in pendingMap:
import time
@@ -561,9 +667,11 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -576,7 +684,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -588,92 +696,108 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -754,6 +878,39 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ # the wait is above
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.error(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
def CASE5( self, main ):
"""
@@ -766,7 +923,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -784,10 +940,11 @@
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
+ main.step( "Get the Mastership of each switch" )
ONOS1Mastership = main.ONOScli1.roles()
# TODO: Make this a meaningful check
if "Error" in ONOS1Mastership or not ONOS1Mastership:
- main.log.report( "Error in getting ONOS roles" )
+ main.log.error( "Error in getting ONOS roles" )
main.log.warn(
"ONOS1 mastership response: " +
repr( ONOS1Mastership ) )
@@ -802,7 +959,7 @@
ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
intentCheck = main.FALSE
if "Error" in ONOS1Intents or not ONOS1Intents:
- main.log.report( "Error in getting ONOS intents" )
+ main.log.error( "Error in getting ONOS intents" )
main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
else:
intentCheck = main.TRUE
@@ -813,7 +970,7 @@
flowCheck = main.FALSE
ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
if "Error" in ONOS1Flows or not ONOS1Flows:
- main.log.report( "Error in getting ONOS flows" )
+ main.log.error( "Error in getting ONOS flows" )
main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
else:
# TODO: Do a better check, maybe compare flows on switches?
@@ -832,17 +989,9 @@
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- temp = ()
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
+ temp = ( nodes[0], nodes[0].name, nodes[0].ip_address, 6633 )
ctrls.append( temp )
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Collecting topology information from ONOS" )
devices = []
@@ -855,18 +1004,30 @@
links.append( main.ONOScli1.links() )
clusters = []
clusters.append( main.ONOScli1.clusters() )
+
+ main.step( "Each host has an IP address" )
ipResult = main.TRUE
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=ipResult,
+ onpass="The ips of the hosts aren't empty",
+ onfail="The ip of at least one host is missing" )
# there should always only be one cluster
- numClusters = len( json.loads( clusters[ 0 ] ) )
+ main.step( "There is only one dataplane cluster" )
+ try:
+ numClusters = len( json.loads( clusters[ 0 ] ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing clusters[0]: " +
+ repr( clusters[ 0 ] ) )
clusterResults = main.FALSE
if numClusters == 1:
clusterResults = main.TRUE
@@ -945,13 +1106,6 @@
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- finalAssert = main.TRUE
- finalAssert = finalAssert and topoResult and flowCheck \
- and intentCheck and consistentMastership and rolesNotNull
- utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
- onpass="State check successful",
- onfail="State check NOT successful" )
-
def CASE6( self, main ):
"""
The Failure case.
@@ -961,10 +1115,22 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- main.log.report( "Restart ONOS node" )
- main.log.case( "Restart ONOS node" )
- main.ONOSbench.onosKill( ONOS1Ip )
+ # Reset non-persistent variables
+ try:
+ iCounterValue = 0
+ except NameError:
+ main.log.error( "iCounterValue not defined, setting to 0" )
+ iCounterValue = 0
+
+ main.case( "Restart ONOS node" )
+ main.caseExplaination = "Killing ONOS process and restart cli " +\
+ "sessions once onos is up."
+ main.step( "Killing ONOS processes" )
+ killResult = main.ONOSbench.onosKill( ONOS1Ip )
start = time.time()
+ utilities.assert_equals( expect=main.TRUE, actual=killResult,
+ onpass="ONOS Killed",
+ onfail="Error killing ONOS" )
main.step( "Checking if ONOS is up yet" )
count = 0
@@ -975,17 +1141,25 @@
break
else:
count = count + 1
+ utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
+ onpass="ONOS is back up",
+ onfail="ONOS failed to start" )
- cliResult = main.ONOScli1.startOnosCli( ONOS1Ip )
+ main.log.step( "Starting ONOS CLI sessions" )
+ cliResults = main.ONOScli1.startOnosCli( ONOS1Ip )
+ utilities.assert_equals( expect=main.TRUE, actual=cliResults,
+ onpass="ONOS cli startup successful",
+ onfail="ONOS cli startup failed" )
- caseResults = main.TRUE and onos1Isup and cliResult
- utilities.assert_equals( expect=main.TRUE, actual=caseResults,
- onpass="ONOS restart successful",
- onfail="ONOS restart NOT successful" )
if elapsed:
main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
str( elapsed ) )
+ main.restartTime = elapsed
+ else:
+ main.restartTime = -1
time.sleep( 5 )
+ # rerun on election apps
+ main.ONOScli1.electionTestRun()
def CASE7( self, main ):
"""
@@ -996,7 +1170,7 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
main.case( "Running ONOS Constant State Tests" )
-
+ main.step( "Check that each switch has a master" )
# Assert that each device has a master
rolesNotNull = main.ONOScli1.rolesNotNull()
utilities.assert_equals(
@@ -1009,14 +1183,12 @@
ONOS1Mastership = main.ONOScli1.roles()
# FIXME: Refactor this whole case for single instance
if "Error" in ONOS1Mastership or not ONOS1Mastership:
- main.log.report( "Error in getting ONOS mastership" )
+ main.log.error( "Error in getting ONOS mastership" )
main.log.warn( "ONOS1 mastership response: " +
repr( ONOS1Mastership ) )
consistentMastership = main.FALSE
else:
consistentMastership = main.TRUE
- main.log.report(
- "Switch roles are consistent across all ONOS nodes" )
utilities.assert_equals(
expect=main.TRUE,
actual=consistentMastership,
@@ -1042,8 +1214,6 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -1055,11 +1225,11 @@
ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
intentCheck = main.FALSE
if "Error" in ONOS1Intents or not ONOS1Intents:
- main.log.report( "Error in getting ONOS intents" )
+ main.log.error( "Error in getting ONOS intents" )
main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
else:
intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
+ main.log.error( "Intents are consistent across all ONOS nodes" )
utilities.assert_equals(
expect=main.TRUE,
actual=intentCheck,
@@ -1084,21 +1254,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
- if intentState and intentState == ONOS1Intents:
+ sameIntents = main.FALSE
+ if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
- # what the aceptable states are
- else:
+ # what the acceptable states are
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) )
- except Exception:
- pass
- sameIntents = main.FALSE
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1121,14 +1311,13 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
+ main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leader = ONOS1Ip
@@ -1144,36 +1333,23 @@
# all is well
pass
elif leaderN == main.FALSE:
- # error in response
- main.log.report( "Something is wrong with " +
+ # error in response
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
+ main.log.error( "ONOS" + str( controller ) + " sees " +
str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
onpass="Leadership election passed",
onfail="Something went wrong with Leadership election" )
- result = ( mastershipCheck and intentCheck and FlowTables and
- rolesNotNull and leaderResult )
- result = int( result )
- if result == main.TRUE:
- main.log.report( "Constant State Tests Passed" )
- utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Constant State Tests Passed",
- onfail="Constant state tests failed" )
-
def CASE8( self, main ):
"""
Compare topo
@@ -1189,15 +1365,24 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- node = main.ONOS1
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ node = main.ONOS1
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1224,7 +1409,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -1325,8 +1510,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
def CASE9( self, main ):
"""
@@ -1342,7 +1525,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -1369,7 +1551,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -1395,14 +1576,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -1439,11 +1619,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -1476,9 +1654,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -1521,20 +1697,28 @@
time.sleep( 10 )
main.step( "Stopping Mininet" )
- main.Mininet1.stopNet()
+ mnResult = main.Mininet1.stopNet()
+ utilities.assert_equals( expect=main.TRUE, actual=mnResult,
+ onpass="Mininet stopped",
+ onfail="MN cleanup NOT successful" )
main.step( "Checking ONOS Logs for errors" )
print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
colors[ 'end' ]
- print main.ONOSbench.checkLogs( ONOS1Ip )
+ print main.ONOSbench.checkLogs( ONOS1Ip, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
- # TODO: actually check something here
- utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
- onpass="Test cleanup successful",
- onfail="Test cleanup NOT successful" )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
def CASE14( self, main ):
"""
@@ -1544,9 +1728,17 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- leaderResult = main.TRUE
- main.log.info( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ main.case("Start Leadership Election app")
+ main.step( "Install leadership election app" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.ONOScli1.electionTestRun()
# check for leader
leader = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
@@ -1555,32 +1747,27 @@
pass
elif leader is None:
# No leader elected
- main.log.report( "No leader was elected" )
+ main.log.error( "No leader was elected" )
leaderResult = main.FALSE
elif leader == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader" +
+ main.log.error( "Something is wrong with electionTestLeader" +
" function, check the error logs" )
leaderResult = main.FALSE
else:
# error in response
- main.log.report(
+ main.log.error(
"Unexpected response from electionTestLeader function:'" +
str( leader ) +
"'" )
leaderResult = main.FALSE
-
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
def CASE15( self, main ):
"""
@@ -1591,7 +1778,6 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
@@ -1600,40 +1786,40 @@
if leader == ONOS1Ip:
oldLeader = getattr( main, "ONOScli1" )
elif leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
oldLeader = None
else:
main.log.error( "Leader election --- why am I HERE?!?")
+ leaderResult = main.FALSE
+ oldLeader = None
if oldLeader:
withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
leaderN = main.ONOScli1.electionTestLeader()
if leaderN == leader:
- main.log.report( "ONOS still sees " + str( leaderN ) +
+ main.log.error( "ONOS still sees " + str( leaderN ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader " +
+ main.log.error( "Something is wrong with electionTestLeader " +
"function, check the error logs" )
leaderResult = main.FALSE
elif leaderN is None:
main.log.info(
"There is no leader after the app withdrew from election" )
- if leaderResult:
- main.log.report( "Leadership election tests passed( There is no " +
- "leader after the old leader resigned )" )
+ leaderResult = main.TRUE
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1651,17 +1837,1109 @@
actual=runResult,
onpass="App re-ran for election",
onfail="App failed to run for election" )
- leader = main.ONOScli1.electionTestLeader()
+
+ main.step( "Node became leader when it ran for election" )
+ afterRun = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
- if leader == ONOS1Ip:
- leaderResult = main.TRUE
+ if afterRun == ONOS1Ip:
+ afterResult = main.TRUE
else:
- leaderResult = main.FALSE
- # TODO: assert on run and withdraw results?
+ afterResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
- onfail="ONOS1's election app was not leader after it re-ran " +
- "for election" )
+ actual=afterResult,
+ onpass="Old leader successfully re-ran for election",
+ onfail="Something went wrong with Leadership election after " +
+ "the old leader re-ran for election" )
+
+ def CASE16( self, main ):
+ """
+ Install Distributed Primitives app
+ """
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
+ # Variables for the distributed primitives tests
+ global pCounterName
+ global iCounterName
+ global pCounterValue
+ global iCounterValue
+ global onosSet
+ global onosSetName
+ pCounterName = "TestON-Partitions"
+ iCounterName = "TestON-inMemory"
+ pCounterValue = 0
+ iCounterValue = 0
+ onosSet = set([])
+ onosSetName = "TestON-set"
+
+ description = "Install Primitives app"
+ main.case( description )
+ main.step( "Install Primitives app" )
+ appName = "org.onosproject.distributedprimitives"
+ appResults = CLIs[0].activateApp( appName )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=appResults,
+ onpass="Primitives app activated",
+ onfail="Primitives app not activated" )
+
+ def CASE17( self, main ):
+ """
+ Check for basic functionality with distributed primitives
+ """
+ import json
+ # Make sure variables are defined/set
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert pCounterName, "pCounterName not defined"
+ assert iCounterName, "iCounterName not defined"
+ assert onosSetName, "onosSetName not defined"
+ # NOTE: assert fails if value is 0/None/Empty/False
+ try:
+ pCounterValue
+ except NameError:
+ main.log.error( "pCounterValue not defined, setting to 0" )
+ pCounterValue = 0
+ try:
+ iCounterValue
+ except NameError:
+ main.log.error( "iCounterValue not defined, setting to 0" )
+ iCounterValue = 0
+ try:
+ onosSet
+ except NameError:
+ main.log.error( "onosSet not defined, setting to empty Set" )
+ onosSet = set([])
+ # Variables for the distributed primitives tests. These are local only
+ addValue = "a"
+ addAllValue = "a b c d e f"
+ retainValue = "c d e f"
+
+ description = "Check for basic functionality with distributed " +\
+ "primitives"
+ main.case( description )
+ main.caseExplaination = "Test the methods of the distributed primitives (counters and sets) throught the cli"
+ # DISTRIBUTED ATOMIC COUNTERS
+ main.step( "Increment and get a default counter on each node" )
+ pCounters = []
+ threads = []
+ addedPValues = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="counterIncrement-" + str( i ),
+ args=[ pCounterName ] )
+ pCounterValue += 1
+ addedPValues.append( pCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ pCounters.append( t.result )
+ # Check that counter incremented numController times
+ pCounterResults = True
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=pCounterResults,
+ onpass="Default counter incremented",
+ onfail="Error incrementing default" +
+ " counter" )
+
+ main.step( "Increment and get an in memory counter on each node" )
+ iCounters = []
+ addedIValues = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counterTestIncrement,
+ name="icounterIncrement-" + str( i ),
+ args=[ iCounterName ],
+ kwargs={ "inMemory": True } )
+ iCounterValue += 1
+ addedIValues.append( iCounterValue )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ iCounters.append( t.result )
+ # Check that counter incremented numController times
+ iCounterResults = True
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
+ utilities.assert_equals( expect=True,
+ actual=iCounterResults,
+ onpass="In memory counter incremented",
+ onfail="Error incrementing in memory" +
+ " counter" )
+
+ main.step( "Check counters are consistant across nodes" )
+ onosCounters = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCounters.append( t.result )
+ tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
+ if all( tmp ):
+ main.log.info( "Counters are consistent across all nodes" )
+ consistentCounterResults = main.TRUE
+ else:
+ main.log.error( "Counters are not consistent across all nodes" )
+ consistentCounterResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=consistentCounterResults,
+ onpass="ONOS counters are consistent " +
+ "across nodes",
+ onfail="ONOS Counters are inconsistent " +
+ "across nodes" )
+
+ main.step( "Counters we added have the correct values" )
+ correctResults = main.TRUE
+ for i in range( numControllers ):
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
+ try:
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
+ except AttributeError, e:
+ main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
+ "is not as expected" )
+ correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=correctResults,
+ onpass="Added counters are correct",
+ onfail="Added counters are incorrect" )
+ # DISTRIBUTED SETS
+ main.step( "Distributed Set get" )
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=getResults,
+ onpass="Set elements are correct",
+ onfail="Set elements are incorrect" )
+
+ main.step( "Distributed Set size" )
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=sizeResults,
+ onpass="Set sizes are correct",
+ onfail="Set sizes are incorrect" )
+
+ main.step( "Distributed Set add()" )
+ onosSet.add( addValue )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAdd-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addResults = main.FALSE
+ else:
+ # unexpected result
+ addResults = main.FALSE
+ if addResults != main.TRUE:
+ main.log.error( "Error executing set add" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addResults = addResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addResults,
+ onpass="Set add correct",
+ onfail="Set add was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set contains()" )
+ containsResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContains-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsResponses.append( t.result )
+
+ containsResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsResults,
+ onpass="Set contains is functional",
+ onfail="Set contains failed" )
+
+ main.step( "Distributed Set containsAll()" )
+ containsAllResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setContainsAll-" + str( i ),
+ args=[ onosSetName ],
+ kwargs={ "values": addAllValue } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ # NOTE: This is the tuple
+ containsAllResponses.append( t.result )
+
+ containsAllResults = main.TRUE
+ for i in range( numControllers ):
+ if containsResponses[ i ] == main.ERROR:
+ containsResults = main.FALSE
+ else:
+ containsResults = containsResults and\
+ containsResponses[ i ][ 1 ]
+ utilities.assert_equals( expect=main.TRUE,
+ actual=containsAllResults,
+ onpass="Set containsAll is functional",
+ onfail="Set containsAll failed" )
+
+ main.step( "Distributed Set remove()" )
+ onosSet.remove( addValue )
+ removeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemove-" + str( i ),
+ args=[ onosSetName, addValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeResults = main.TRUE
+ for i in range( numControllers ):
+ if removeResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeResults = main.FALSE
+ else:
+ # unexpected result
+ removeResults = main.FALSE
+ if removeResults != main.TRUE:
+ main.log.error( "Error executing set remove" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeResults = removeResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeResults,
+ onpass="Set remove correct",
+ onfail="Set remove was incorrect" )
+
+ main.step( "Distributed Set removeAll()" )
+ onosSet.difference_update( addAllValue.split() )
+ removeAllResponses = []
+ threads = []
+ try:
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRemoveAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ removeAllResponses.append( t.result )
+ except Exception, e:
+ main.log.exception(e)
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ removeAllResults = main.TRUE
+ for i in range( numControllers ):
+ if removeAllResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif removeAllResponses[ i ] == main.FALSE:
+ # not in set, probably fine
+ pass
+ elif removeAllResponses[ i ] == main.ERROR:
+ # Error in execution
+ removeAllResults = main.FALSE
+ else:
+ # unexpected result
+ removeAllResults = main.FALSE
+ if removeAllResults != main.TRUE:
+ main.log.error( "Error executing set removeAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ removeAllResults = removeAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=removeAllResults,
+ onpass="Set removeAll correct",
+ onfail="Set removeAll was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set clear()" )
+ onosSet.clear()
+ clearResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestClear-" + str( i ),
+ args=[ onosSetName, " "], # Values doesn't matter
+ kwargs={ "clear": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ clearResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ clearResults = main.TRUE
+ for i in range( numControllers ):
+ if clearResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif clearResponses[ i ] == main.FALSE:
+ # Nothing set, probably fine
+ pass
+ elif clearResponses[ i ] == main.ERROR:
+ # Error in execution
+ clearResults = main.FALSE
+ else:
+ # unexpected result
+ clearResults = main.FALSE
+ if clearResults != main.TRUE:
+ main.log.error( "Error executing set clear" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ clearResults = clearResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=clearResults,
+ onpass="Set clear correct",
+ onfail="Set clear was incorrect" )
+
+ main.step( "Distributed Set addAll()" )
+ onosSet.update( addAllValue.split() )
+ addResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestAdd,
+ name="setTestAddAll-" + str( i ),
+ args=[ onosSetName, addAllValue ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ addResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ addAllResults = main.TRUE
+ for i in range( numControllers ):
+ if addResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif addResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif addResponses[ i ] == main.ERROR:
+ # Error in execution
+ addAllResults = main.FALSE
+ else:
+ # unexpected result
+ addAllResults = main.FALSE
+ if addAllResults != main.TRUE:
+ main.log.error( "Error executing set addAll" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " + str( size ) +
+ " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ addAllResults = addAllResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=addAllResults,
+ onpass="Set addAll correct",
+ onfail="Set addAll was incorrect" )
+
+ main.step( "Distributed Set retain()" )
+ onosSet.intersection_update( retainValue.split() )
+ retainResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestRemove,
+ name="setTestRetain-" + str( i ),
+ args=[ onosSetName, retainValue ],
+ kwargs={ "retain": True } )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ retainResponses.append( t.result )
+
+ # main.TRUE = successfully changed the set
+ # main.FALSE = action resulted in no change in set
+ # main.ERROR - Some error in executing the function
+ retainResults = main.TRUE
+ for i in range( numControllers ):
+ if retainResponses[ i ] == main.TRUE:
+ # All is well
+ pass
+ elif retainResponses[ i ] == main.FALSE:
+ # Already in set, probably fine
+ pass
+ elif retainResponses[ i ] == main.ERROR:
+ # Error in execution
+ retainResults = main.FALSE
+ else:
+ # unexpected result
+ retainResults = main.FALSE
+ if retainResults != main.TRUE:
+ main.log.error( "Error executing set retain" )
+
+ # Check if set is still correct
+ size = len( onosSet )
+ getResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestGet,
+ name="setTestGet-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ getResponses.append( t.result )
+ getResults = main.TRUE
+ for i in range( numControllers ):
+ if isinstance( getResponses[ i ], list):
+ current = set( getResponses[ i ] )
+ if len( current ) == len( getResponses[ i ] ):
+ # no repeats
+ if onosSet != current:
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has incorrect view" +
+ " of set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ main.log.debug( "Expected: " + str( onosSet ) )
+ main.log.debug( "Actual: " + str( current ) )
+ getResults = main.FALSE
+ else:
+ # error, set is not a set
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " has repeat elements in" +
+ " set " + onosSetName + ":\n" +
+ str( getResponses[ i ] ) )
+ getResults = main.FALSE
+ elif getResponses[ i ] == main.ERROR:
+ getResults = main.FALSE
+ sizeResponses = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].setTestSize,
+ name="setTestSize-" + str( i ),
+ args=[ onosSetName ] )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ sizeResponses.append( t.result )
+ sizeResults = main.TRUE
+ for i in range( numControllers ):
+ if size != sizeResponses[ i ]:
+ sizeResults = main.FALSE
+ main.log.error( "ONOS" + str( i + 1 ) +
+ " expected a size of " +
+ str( size ) + " for set " + onosSetName +
+ " but got " + str( sizeResponses[ i ] ) )
+ retainResults = retainResults and getResults and sizeResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=retainResults,
+ onpass="Set retain correct",
+ onfail="Set retain was incorrect" )
+
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.params b/TestON/tests/IntentEventTP/IntentEventTP.params
index fb2e625..ddda23e 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.params
+++ b/TestON/tests/IntentEventTP/IntentEventTP.params
@@ -1,19 +1,20 @@
<PARAMS>
- <testcases>1,2,1,2</testcases>
+ <testcases>1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
+ <isOnBaremetal>True</isOnBaremetal>
<debugMode></debugMode> #nothing means false
<ENV>
<cellName>intentTP</cellName>
- <cellApps>null,intentperf</cellApps>
+ <cellApps>drivers,null,intentperf</cellApps>
</ENV>
- <SCALE>1,3,5,7</SCALE>
+ <SCALE>1,3,3,5,5,7,7</SCALE>
<availableNodes>7</availableNodes>
<GIT>
- <autopull>off</autopull>
+ <autopull>on</autopull>
<checkout>master</checkout>
</GIT>
@@ -32,32 +33,33 @@
<ip4>10.128.5.54</ip4>
<port4>6633</port4>
- <ip5>10.128.5.65</ip5>
+ <ip5>10.128.5.55</ip5>
<port5>6633</port5>
- <ip6>10.128.5.66</ip6>
+ <ip6>10.128.5.56</ip6>
<port6>6633</port6>
- <ip7>10.128.5.67</ip7>
+ <ip7>10.128.5.57</ip7>
<port7>6633</port7>
</CTRL>
- <MN><ip1>10.128.5.55</ip1></MN>
+ <MN><ip1>10.128.5.50</ip1></MN>
<BENCH>
<user>admin</user>
- <ip1>10.128.5.55</ip1>
+ <ip1>10.128.5.50</ip1>
</BENCH>
<TEST>
<loadFrom>1,1,1,1,1,1,1</loadFrom> #generate load on server, 1 = generator on
<numSwitches>10,10,10,10,10,10,10</numSwitches>
<skipCleanInstall>yes</skipCleanInstall>
- <duration>30</duration>
+ <duration>400</duration>
<log_interval>20</log_interval>
- <numKeys>5000</numKeys>
+ <numKeys>40000</numKeys>
<cyclePeriod>1000</cyclePeriod>
<neighbors>0,a</neighbors> #a == all nodes (-1)
+ <flowRuleBUEnabled>false</flowRuleBUEnabled>
</TEST>
<METRICS>
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.py b/TestON/tests/IntentEventTP/IntentEventTP.py
index 0e8af7d..99332f7 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.py
+++ b/TestON/tests/IntentEventTP/IntentEventTP.py
@@ -16,6 +16,8 @@
def CASE1( self, main ):
import time
+ import os.path
+
global init
try:
if type(init) is not bool:
@@ -35,7 +37,13 @@
skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cellName = main.params[ 'ENV' ][ 'cellName' ]
numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
+ flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
+ onBaremetal = main.params['isOnBaremetal']
+ homeDir = os.path.expanduser('~')
+ main.exceptions = [0]*11
+ main.warnings = [0]*11
+ main.errors = [0]*11
# -- INIT SECTION, ONLY RUNS ONCE -- #
if init == False:
@@ -44,7 +52,7 @@
global ONOSIp #list of ONOS IP addresses
global scale
global commit
-
+
clusterCount = 0
ONOSIp = [ 0 ]
scale = (main.params[ 'SCALE' ]).split(",")
@@ -109,7 +117,24 @@
myDistribution.append(numSwitches[node-1])
#main.ONOSbench.createLinkGraphFile( BENCHIp,cellIp,myDistribution)
-
+
+ if onBaremetal == "True":
+ filename = "/onos/tools/package/bin/onos-service"
+ serviceConfig = open(homeDir + filename, 'w+')
+ serviceConfig.write("#!/bin/bash\n ")
+ serviceConfig.write("#------------------------------------- \n ")
+ serviceConfig.write("# Starts ONOS Apache Karaf container\n ")
+ serviceConfig.write("#------------------------------------- \n ")
+ serviceConfig.write("#export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}\n ")
+ serviceConfig.write("""export JAVA_OPTS="${JAVA_OPTS:--Xms8G -Xmx8G}" \n """)
+ serviceConfig.write("")
+ serviceConfig.write("ONOS_HOME=/opt/onos \n ")
+ serviceConfig.write("")
+ serviceConfig.write("[ -d $ONOS_HOME ] && cd $ONOS_HOME || ONOS_HOME=$(dirname $0)/..\n")
+ serviceConfig.write("""${ONOS_HOME}/apache-karaf-$KARAF_VERSION/bin/karaf "$@" \n """)
+ serviceConfig.close()
+ main.log.info("Set /onos/tools/package/bin/onos-service with 8G Xms/Xmx Options.")
+
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
@@ -133,17 +158,39 @@
time.sleep(20)
- #main.ONOSbench.handle.sendline("""onos $OC1 "cfg setorg.onosproject.provider.nil.NullProviders enabled true" """)
- #main.ONOSbench.handle.expect(":~")
- #print main.ONOSbench.handle.before
+
+ while True:
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
+ main.ONOSbench.handle.expect(":~")
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
+ main.ONOSbench.handle.expect(":~")
+ if ("value=" + str(clusterCount*10)) in main.ONOSbench.handle.before:
+ main.log.info("Device count set")
+ main.log.info("before" + main.ONOSbench.handle.before)
+ break
+ time.sleep(10)
+ main.log.info("cfg set failure, retrying")
+ main.log.info("before" + main.ONOSbench.handle.before)
- main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
+ while True:
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
+ main.ONOSbench.handle.expect(":~")
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
+ main.ONOSbench.handle.expect(":~")
+ if ("value=linear") in main.ONOSbench.handle.before:
+ main.log.info("Device count set")
+ main.log.info("before" + main.ONOSbench.handle.before)
+ break
+ time.sleep(10)
+ main.log.info("cfg set failure, retrying")
+ main.log.info("before" + main.ONOSbench.handle.before)
+
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBU + """" """)
main.ONOSbench.handle.expect(":~")
- print main.ONOSbench.handle.before
- time.sleep(10)
- main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
main.ONOSbench.handle.expect(":~")
- print main.ONOSbench.handle.before
+ main.log.info(main.ONOSbench.handle.before)
+
time.sleep(10)
main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
main.ONOSbench.handle.expect(":~")
@@ -159,6 +206,7 @@
while True:
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
+ main.log.info("before" + main.ONOSbench.handle.before)
clusterCheck = ((main.ONOSbench.handle.before).splitlines())[3]
print("\nBefore: " + str(clusterCheck))
if ("SCC(s)=1,") in clusterCheck:
@@ -174,7 +222,7 @@
break
lastOutput = clusterCheck
time.sleep(5)
-
+ main.ONOSbench.onosErrorLog(ONOSIp[1])
def CASE2( self, main ):
import time
@@ -182,6 +230,26 @@
import string
import csv
import numpy
+ import os.path
+
+ global currentNeighbors
+ neighbors = []
+
+ try:
+ currentNeighbors
+ except:
+ currentNeighbors = "0"
+ neighbors = ['0']
+ else:
+ if currentNeighbors == "r": #reset
+ currentNeighbors = "a"
+ neighbors = ['0']
+ else:
+ currentNeighbors = "r"
+ neighbors = ['a']
+
+ if clusterCount == 1:
+ currentNeighbors = "r"
main.log.info("Cluster Count = " + str(clusterCount))
@@ -193,7 +261,7 @@
debug = main.params[ 'debugMode' ]
numKeys = main.params[ 'TEST' ][ 'numKeys' ]
cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
- neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",")
+ #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",")
metricList = [intentsRate, intentsWithdrawn, intentsFailed]
for n in range(0, len(neighbors)):
@@ -265,7 +333,7 @@
clusterTotal = str(numpy.sum(groupResult))
main.log.report("Results from this round of polling: " + str(groupResult))
main.log.report("Cluster Total: " + clusterTotal + "\n")
-
+
cmd = "onos $OC1 intent-perf-stop"
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
@@ -284,6 +352,7 @@
resultsDB.write(resultString)
resultsDB.close()
-
+
+ main.ONOSbench.onosErrorLog(ONOSIp[1])
diff --git a/TestON/tests/IntentEventTP/IntentEventTP.topo b/TestON/tests/IntentEventTP/IntentEventTP.topo
index 30a1467..763a4d6 100644
--- a/TestON/tests/IntentEventTP/IntentEventTP.topo
+++ b/TestON/tests/IntentEventTP/IntentEventTP.topo
@@ -3,7 +3,7 @@
<COMPONENT>
<ONOSbench>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosDriver</type>
@@ -12,7 +12,7 @@
</ONOSbench>
<ONOS1cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
</ONOS1cli>
<ONOS2cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
</ONOS2cli>
<ONOS3cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
</ONOS3cli>
<ONOS4cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
</ONOS4cli>
<ONOS5cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
</ONOS5cli>
<ONOS6cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
</ONOS6cli>
<ONOS7cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -112,7 +112,7 @@
<ONOS5>
- <host>10.128.5.65</host>
+ <host>10.128.5.55</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -121,7 +121,7 @@
</ONOS5>
<ONOS6>
- <host>10.128.5.66</host>
+ <host>10.128.5.56</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -130,7 +130,7 @@
</ONOS6>
<ONOS7>
- <host>10.128.5.67</host>
+ <host>10.128.5.57</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -141,6 +141,4 @@
</COMPONENT>
</TOPOLOGY>
-
-
-
+
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
index 3185cc2..2b76662 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.params
@@ -1,22 +1,22 @@
<PARAMS>
- <testcases>1,2</testcases>
+ <testcases>1,2,1,2,1,2,1,2</testcases>
- <SCALE>3,5,7</SCALE>
+ <SCALE>1,3,5,7</SCALE>
<availableNodes>7</availableNodes>
<ENV>
<cellName>IntentInstallWithdrawCell</cellName>
- <cellApps>null</cellApps>
+ <cellApps>drivers,null</cellApps>
</ENV>
<TEST>
<skipCleanInstall>yes</skipCleanInstall>
<switchCount>7</switchCount>
- <warmUp>4</warmUp>
- <sampleSize>10</sampleSize>
+ <warmUp>10</warmUp>
+ <sampleSize>20</sampleSize>
<wait></wait>
- <intents>1000</intents> #list format, will be split on ','
+ <intents>1,100,1000</intents> #list format, will be split on ','
<debug>True</debug> #"True" for true
</TEST>
@@ -28,36 +28,36 @@
<CTRL>
<USER>admin</USER>
- <ip1>10.128.5.51</ip1>
+ <ip1>10.254.1.201</ip1>
<port1>6633</port1>
- <ip2>10.128.5.52</ip2>
+ <ip2>10.254.1.202</ip2>
<port2>6633</port2>
- <ip3>10.128.5.53</ip3>
+ <ip3>10.254.1.203</ip3>
<port3>6633</port3>
- <ip4>10.128.5.54</ip4>
+ <ip4>10.254.1.204</ip4>
<port4>6633</port4>
- <ip5>10.128.5.65</ip5>
+ <ip5>10.254.1.205</ip5>
<port5>6633</port5>
- <ip6>10.128.5.66</ip6>
+ <ip6>10.254.1.206</ip6>
<port6>6633</port6>
- <ip7>10.128.5.67</ip7>
+ <ip7>10.254.1.207</ip7>
<port7>6633</port7>
</CTRL>
<MN>
- <ip1>10.128.5.55</ip1>
+ <ip1>10.254.1.200</ip1>
</MN>
<BENCH>
<user>admin</user>
- <ip1>10.128.5.55</ip1>
+ <ip1>10.254.1.200</ip1>
</BENCH>
<JSON>
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
index be8f38c..83dac2a 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.py
@@ -252,6 +252,6 @@
resultString += str(numpy.std(installed)) + ","
resultString += str(numpy.mean(withdrawn)) + ","
resultString += str(numpy.std(withdrawn)) + "\n"
- resultsDB = open("IntentInstallWithdrawLatDB", "w+")
+ resultsDB = open("IntentInstallWithdrawLatDB", "a")
resultsDB.write(resultString)
resultsDB.close()
diff --git a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
index 30a1467..0e45e0f 100644
--- a/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
+++ b/TestON/tests/IntentInstallWithdrawLat/IntentInstallWithdrawLat.topo
@@ -3,7 +3,7 @@
<COMPONENT>
<ONOSbench>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosDriver</type>
@@ -12,7 +12,7 @@
</ONOSbench>
<ONOS1cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
</ONOS1cli>
<ONOS2cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
</ONOS2cli>
<ONOS3cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
</ONOS3cli>
<ONOS4cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
</ONOS4cli>
<ONOS5cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
</ONOS5cli>
<ONOS6cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
</ONOS6cli>
<ONOS7cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -75,7 +75,7 @@
</ONOS7cli>
<ONOS1>
- <host>10.128.5.51</host>
+ <host>10.254.1.201</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -84,7 +84,7 @@
</ONOS1>
<ONOS2>
- <host>10.128.5.52</host>
+ <host>10.254.1.202</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -93,7 +93,7 @@
</ONOS2>
<ONOS3>
- <host>10.128.5.53</host>
+ <host>10.254.1.203</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -102,7 +102,7 @@
</ONOS3>
<ONOS4>
- <host>10.128.5.54</host>
+ <host>10.254.1.204</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -112,7 +112,7 @@
<ONOS5>
- <host>10.128.5.65</host>
+ <host>10.254.1.205</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -121,7 +121,7 @@
</ONOS5>
<ONOS6>
- <host>10.128.5.66</host>
+ <host>10.254.1.206</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -130,7 +130,7 @@
</ONOS6>
<ONOS7>
- <host>10.128.5.67</host>
+ <host>10.254.1.207</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.params b/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
index 27af1f2..8e7a6a4 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.params
@@ -1,27 +1,28 @@
<PARAMS>
- <testcases>1,2</testcases>
+ <testcases>1,2,1,2,1,2,1,2</testcases>
- <SCALE>1</SCALE>
+ <SCALE>1,3,5,7</SCALE>
<availableNodes>7</availableNodes>
<ENV>
<cellName>intentRerouteCell</cellName>
- <cellApps>null,intentperf,metrics</cellApps>
- <scale1switches>8</scale1switches>
- <scale3switches>2,2,3</scale3switches>
- <scale5switches>2,2,1,1,1</scale5switches>
- <scale7switches>1,1,1,1,1,1,1</scale7switches>
-
+ <cellApps>drivers,null,intentperf,metrics</cellApps>
</ENV>
<TEST>
<skipCleanInstall>yes</skipCleanInstall>
- <warmUp>1</warmUp>
- <sampleSize>2</sampleSize>
+ <warmUp>5</warmUp>
+ <sampleSize>10</sampleSize>
<wait></wait>
- <intents>1,10</intents> #list format, will be split on ','
- <debug>True</debug>
+ <intents>1,100,1000</intents> #list format, will be split on ','
+ <debug>True</debug>
+
+ <s1>1,1,1,1,1,1,1,1</s1>
+ <s3>2,2,1,1,3,3,3,1</s3>
+ <s5>2,2,1,1,3,4,5,3</s5>
+ <s7>2,3,1,1,5,6,7,4</s7>
+
</TEST>
<METRICS>
@@ -40,36 +41,36 @@
<CTRL>
<USER>admin</USER>
- <ip1>10.128.5.51</ip1>
+ <ip1>10.254.1.201</ip1>
<port1>6633</port1>
- <ip2>10.128.5.52</ip2>
+ <ip2>10.254.1.202</ip2>
<port2>6633</port2>
- <ip3>10.128.5.53</ip3>
+ <ip3>10.254.1.203</ip3>
<port3>6633</port3>
- <ip4>10.128.5.54</ip4>
+ <ip4>10.254.1.204</ip4>
<port4>6633</port4>
- <ip5>10.128.5.65</ip5>
+ <ip5>10.254.1.205</ip5>
<port5>6633</port5>
- <ip6>10.128.5.66</ip6>
+ <ip6>10.254.1.206</ip6>
<port6>6633</port6>
- <ip7>10.128.5.67</ip7>
+ <ip7>10.254.1.207</ip7>
<port7>6633</port7>
</CTRL>
<MN>
- <ip1>10.128.5.55</ip1>
+ <ip1>10.254.1.200</ip1>
</MN>
<BENCH>
<user>admin</user>
- <ip1>10.128.5.55</ip1>
+ <ip1>10.254.1.200</ip1>
</BENCH>
<JSON>
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.py b/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
index 84d5c03..f824a5c 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.py
@@ -79,9 +79,6 @@
clusterCount = int(scale[0])
scale.remove(scale[0])
- switchParams = ("scale" + str(clusterCount) + "switches")
- switchCount = (main.params[ 'ENV' ][ switchParams ]).split(",")
-
#kill off all onos processes
main.log.step("Safety check, killing all ONOS processes")
main.log.step("before initiating enviornment setup")
@@ -105,15 +102,6 @@
main.step( "Set Cell" )
main.ONOSbench.setCell(cellName)
- if clusterCount == 1:
- temp = "one"
- if clusterCount == 3:
- temp = "three"
- if clusterCount == 5:
- temp = "five"
- if clusterCount == 7:
- temp = "seven"
-
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
@@ -134,7 +122,10 @@
if not isup:
main.log.report( "ONOS " + str(node) + " didn't start!" )
main.log.info("Startup sequence complete")
-
+
+ deviceMastership = (main.params[ 'TEST' ][ "s" + str(clusterCount) ]).split(",")
+ print("Device mastership list: " + str(deviceMastership))
+
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 8 " """)
main.ONOSbench.handle.expect(":~")
print repr(main.ONOSbench.handle.before)
@@ -143,53 +134,49 @@
main.ONOSbench.handle.expect(":~")
print repr(main.ONOSbench.handle.before)
time.sleep(3)
- main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
- main.ONOSbench.handle.expect(":~")
- print repr(main.ONOSbench.handle.before)
- while True:
+ time.sleep(10)
+
+ once = True
+ for attempt in range(0,10):
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
x = main.ONOSbench.handle.before
- if "devices=8" in x:
+ main.log.info("Initial setup. ONOS summary response: \n" + x + "\n")
+
+ if "devices=8" in x and "links=16," in x:
break
- else:
+ else:
+ if once == False:
+ main.log.error("Null provider start failed, retrying..")
+ once = False
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled false" """)
main.ONOSbench.handle.expect(":~")
time.sleep(3)
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders enabled true" """)
main.ONOSbench.handle.expect(":~")
- main.log.error("Null provider start failed, retrying..")
- time.sleep(8)
+ time.sleep(8)
- main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
- main.ONOSbench.handle.expect(":~")
- print main.ONOSbench.handle.before
- time.sleep(10)
- main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
- main.ONOSbench.handle.expect(":~")
-
- temp = 1
- for node in range(1, clusterCount + 1):
- for switch in range (0, int(switchCount[node-1])):
- cmd = ("""onos $OC1 "device-role null:000000000000000""" + str(temp) + " " + ONOSIp[node] + """ master" """)
+ index = 1
+ for node in deviceMastership:
+ for attempt in range(0,10):
+ cmd = ( "onos $OC" + node + """ "device-role null:000000000000000""" + str(index) + " " + ONOSIp[int(node)] + """ master" """)
+ main.log.info("assigning mastership of device " + str(index) + " to node " + node + ": \n " + cmd + "\n")
main.ONOSbench.handle.sendline(cmd)
- main.log.info( cmd )
main.ONOSbench.handle.expect(":~")
- temp += 1
+ time.sleep(4)
+
+ cmd = ( "onos $OC" + node + " roles|grep 00000" + str(index))
+ main.log.info(cmd)
+ main.ONOSbench.handle.sendline(cmd)
+ main.ONOSbench.handle.expect(":~")
+ check = main.ONOSbench.handle.before
+ main.log.info("CHECK:\n" + check)
+ if ("master=" + ONOSIp[int(node)]) in check:
+ break
+ index += 1
- # cmd = ( """onos $OC1 "device-role null:0000000000000008 10.128.5.52 master" """)
- #if clusterCount == 7:
- # cmd = ( """onos $OC1 "device-role null:0000000000000008 10.128.5.53 master" """)
-
- #main.ONOSbench.handle.sendline(cmd)
- #main.log.info( cmd )
- #main.ONOSbench.handle.expect(":~")
-
- #print "sleeping"
- #time.sleep(120)
-
def CASE2( self, main ):
import time
@@ -226,7 +213,7 @@
debug = False
ingress = "null:0000000000000001"
- egress = "null:0000000000000008"
+ egress = "null:0000000000000007"
for intents in intentsList:
main.log.report("Intent Batch size: " + str(intents) + "\n ")
@@ -236,7 +223,7 @@
if run > warmUp:
main.log.info("Starting test iteration " + str(run-warmUp))
- cmd = """onos $OC1 push-test-intents -i" """
+ cmd = """onos $OC1 "push-test-intents -i """
cmd += ingress + "/0 "
cmd += egress + "/0 "
cmd += str(intents) +""" 1" """
@@ -253,10 +240,10 @@
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
linkCheck = main.ONOSbench.handle.before
- if ("flows=16,") in linkCheck:
+ if ("links=16,") in linkCheck and ("flows=" + str(intents*7) + ","):
break
if i == 39:
- main.log.error("Flow count incorrect, data invalid."+ linkCheck)
+ main.log.error("Flow/link count incorrect, data invalid."+ linkCheck)
#collect timestamp from link cut
@@ -270,7 +257,8 @@
time.sleep(2)
main.ONOSbench.handle.expect(":~")
raw = main.ONOSbench.handle.before
- if "NullLinkProvider" in raw and "links=14" in raw:
+ #if "NullLinkProvider" in raw and "links=14" in raw:
+ if "links=14" in raw:
break
if i >= 9:
main.log.error("Expected output not being recieved... continuing")
@@ -292,22 +280,27 @@
main.ONOSbench.handle.sendline("onos $OC1 summary")
main.ONOSbench.handle.expect(":~")
linkCheck = main.ONOSbench.handle.before
- if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
+ #if "links=" + str(7*intents)+ "," in linkCheck and ("flows=" + str(7*intents) + ",") in linkCheck:
+ if "links=14," in linkCheck and ("flows=" + str(8*intents) + ",") in linkCheck:
break
if i == 39:
main.log.error("Link or flow count incorrect, data invalid." + linkCheck)
+
+ time.sleep(5) #trying to avoid negative values
#intents events metrics installed timestamp
IEMtimestamps = [0]*(clusterCount + 1)
installedTemp = [0]*(clusterCount + 1)
for node in range(1, clusterCount +1):
- cmd = "onos $OC" + str(node) + " intents-events-metrics|grep Timestamp"
+ cmd = "onos $OC" + str(node) + """ "intents-events-metrics"|grep Timestamp """
raw = ""
- while "Timestamp" not in raw:
+ while "epoch)" not in raw:
main.ONOSbench.handle.sendline(cmd)
main.ONOSbench.handle.expect(":~")
raw = main.ONOSbench.handle.before
+ print(raw)
+
intentsTimestamps = {}
rawTimestamps = raw.splitlines()
for line in rawTimestamps:
@@ -398,13 +391,16 @@
average = numpy.average(latTemp)
stdDev = numpy.std(latTemp)
+ average = numpy.multiply(average, 1000)
+ stdDev = numpy.multiply(stdDev, 1000)
+
main.log.report("Scale: " + str(clusterCount) + " \tIntent batch: " + str(intents))
main.log.report("Latency average:................" + str(average))
main.log.report("Latency standard deviation:....." + str(stdDev))
main.log.report("Mode of last node to respond:..." + str(nodeMode))
main.log.report("________________________________________________________")
- resultsDB = open("IntentRerouteLatDB", "w+")
+ resultsDB = open("IntentRerouteLatDB", "a")
resultsDB.write("'" + commit + "',")
resultsDB.write(str(clusterCount) + ",")
resultsDB.write(str(intents) + ",")
diff --git a/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo b/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
index 30a1467..0e45e0f 100644
--- a/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
+++ b/TestON/tests/IntentRerouteLat/IntentRerouteLat.topo
@@ -3,7 +3,7 @@
<COMPONENT>
<ONOSbench>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosDriver</type>
@@ -12,7 +12,7 @@
</ONOSbench>
<ONOS1cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
</ONOS1cli>
<ONOS2cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
</ONOS2cli>
<ONOS3cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
</ONOS3cli>
<ONOS4cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
</ONOS4cli>
<ONOS5cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
</ONOS5cli>
<ONOS6cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
</ONOS6cli>
<ONOS7cli>
- <host>10.128.5.55</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -75,7 +75,7 @@
</ONOS7cli>
<ONOS1>
- <host>10.128.5.51</host>
+ <host>10.254.1.201</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -84,7 +84,7 @@
</ONOS1>
<ONOS2>
- <host>10.128.5.52</host>
+ <host>10.254.1.202</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -93,7 +93,7 @@
</ONOS2>
<ONOS3>
- <host>10.128.5.53</host>
+ <host>10.254.1.203</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -102,7 +102,7 @@
</ONOS3>
<ONOS4>
- <host>10.128.5.54</host>
+ <host>10.254.1.204</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -112,7 +112,7 @@
<ONOS5>
- <host>10.128.5.65</host>
+ <host>10.254.1.205</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -121,7 +121,7 @@
</ONOS5>
<ONOS6>
- <host>10.128.5.66</host>
+ <host>10.254.1.206</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -130,7 +130,7 @@
</ONOS6>
<ONOS7>
- <host>10.128.5.67</host>
+ <host>10.254.1.207</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
diff --git a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py
deleted file mode 100644
index 4400262..0000000
--- a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# ScaleOutTemplate
-#
-# CASE1 starts number of nodes specified in param file
-#
-# cameron@onlab.us
-
-import sys
-import os.path
-
-
-class ScaleOutTemplate:
-
- def __init__( self ):
- self.default = ''
-
- def CASE1( self, main ):
-
- import time
- global init
- try:
- if type(init) is not bool:
- init = False
- except NameError:
- init = False
-
- #Load values from params file
- checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
- gitPull = main.params[ 'GIT' ][ 'autopull' ]
- cellName = main.params[ 'ENV' ][ 'cellName' ]
- Apps = main.params[ 'ENV' ][ 'cellApps' ]
- BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
- BENCHUser = main.params[ 'BENCH' ][ 'user' ]
- MN1Ip = main.params[ 'MN' ][ 'ip1' ]
- maxNodes = int(main.params[ 'availableNodes' ])
- skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
- cellName = main.params[ 'ENV' ][ 'cellName' ]
-
- # -- INIT SECTION, ONLY RUNS ONCE -- #
- if init == False:
- init = True
- global clusterCount #number of nodes running
- global ONOSIp #list of ONOS IP addresses
- global scale
-
- clusterCount = 0
- ONOSIp = [ 0 ]
- scale = (main.params[ 'SCALE' ]).split(",")
- clusterCount = int(scale[0])
-
- #Populate ONOSIp with ips from params
- for i in range(1, maxNodes + 1):
- ipString = 'ip' + str(i)
- ONOSIp.append(main.params[ 'CTRL' ][ ipString ])
-
- #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
- if skipMvn != "yes":
- mvnResult = main.ONOSbench.cleanInstall()
-
- #git
- main.step( "Git checkout and pull " + checkoutBranch )
- if gitPull == 'on':
- checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
- pullResult = main.ONOSbench.gitPull()
-
- else:
- checkoutResult = main.TRUE
- pullResult = main.TRUE
- main.log.info( "Skipped git checkout and pull" )
-
- # -- END OF INIT SECTION --#
-
- clusterCount = int(scale[0])
- scale.remove(scale[0])
-
- #kill off all onos processes
- main.log.step("Safety check, killing all ONOS processes")
- main.log.step("before initiating enviornment setup")
- for node in range(1, maxNodes + 1):
- main.ONOSbench.onosDie(ONOSIp[node])
-
- #Uninstall everywhere
- main.log.step( "Cleaning Enviornment..." )
- for i in range(1, maxNodes + 1):
- main.log.info(" Uninstalling ONOS " + str(i) )
- main.ONOSbench.onosUninstall( ONOSIp[i] )
-
- #construct the cell file
- main.log.info("Creating cell file")
- cellIp = []
- for node in range (1, clusterCount + 1):
- cellIp.append(ONOSIp[node])
-
- main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)
-
- main.step( "Set Cell" )
- main.ONOSbench.setCell(cellName)
-
- main.step( "Creating ONOS package" )
- packageResult = main.ONOSbench.onosPackage()
-
- main.step( "verify cells" )
- verifyCellResult = main.ONOSbench.verifyCell()
-
- main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
- for node in range(1, clusterCount + 1):
- main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
- main.ONOSbench.onosInstall( ONOSIp[node])
-
- for node in range(1, clusterCount + 1):
- for i in range( 2 ):
- isup = main.ONOSbench.isup( ONOSIp[node] )
- if isup:
- main.log.info("ONOS " + str(node) + " is up\n")
- break
- if not isup:
- main.log.report( "ONOS " + str(node) + " didn't start!" )
- main.log.info("Startup sequence complete")
-
- def CASE2( self, main ):
-
- print ("clusterCount: " + str(clusterCount))
- print ("scale: " + str(scale))
- print ("ONOSIp: " + str(ONOSIp))
- print ("INIT: " + str(init))
-
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.params b/TestON/tests/SdnIpTest/SdnIpTest.params
index 7f7ef2d..c84af17 100755
--- a/TestON/tests/SdnIpTest/SdnIpTest.params
+++ b/TestON/tests/SdnIpTest/SdnIpTest.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>4</testcases>
+ <testcases>100, [4]*5</testcases>
#Environment variables
<ENV>
@@ -8,13 +8,15 @@
</ENV>
<CTRL>
+ <numCtrl>1</numCtrl>
<ip1>10.128.4.52</ip1>
<port1>6633</port1>
</CTRL>
<GIT>
- <autoPull>off</autoPull>
- <checkout>master</checkout>
+ <autoPull>on</autoPull>
+ <branch1>master</branch1>
+ <branch2>onos-1.2</branch2>
</GIT>
<JSON>
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.py b/TestON/tests/SdnIpTest/SdnIpTest.py
index 85ac21e..43bf580 100644
--- a/TestON/tests/SdnIpTest/SdnIpTest.py
+++ b/TestON/tests/SdnIpTest/SdnIpTest.py
@@ -1,22 +1,86 @@
-# from cupshelpers.config import prefix
-# Testing the basic functionality of SDN-IP
-
-
+# Testing the functionality of SDN-IP with single ONOS instance
class SdnIpTest:
def __init__( self ):
self.default = ''
+ global branchName
-# from cupshelpers.config import prefix
+ # This case is to setup ONOS
+ def CASE100( self, main ):
+ """
+ CASE100 is to compile ONOS and push it to the test machines
+ Startup sequence:
+ git pull
+ mvn clean install
+ onos-package
+ cell <name>
+ onos-verify-cell
+ onos-install -f
+ onos-wait-for-start
+ """
+ import time
+ main.case( "Setting up test environment" )
-# Testing the basic functionality of SDN-IP
+ cellName = main.params[ 'ENV' ][ 'cellName' ]
+ ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
+ main.step( "Applying cell variable to environment" )
+ cellResult = main.ONOSbench.setCell( cellName )
+ verifyResult = main.ONOSbench.verifyCell()
-class SdnIpTest:
+ branchName = main.ONOSbench.getBranchName()
+ main.log.info( "ONOS is on branch: " + branchName )
- def __init__( self ):
- self.default = ''
+ main.log.report( "Uninstalling ONOS" )
+ main.ONOSbench.onosUninstall( ONOS1Ip )
+
+ cleanInstallResult = main.TRUE
+ gitPullResult = main.TRUE
+
+ main.step( "Git pull" )
+ gitPullResult = main.ONOSbench.gitPull()
+
+ main.step( "Using mvn clean & install" )
+ cleanInstallResult = main.TRUE
+# if gitPullResult == main.TRUE:
+# cleanInstallResult = main.ONOSbench.cleanInstall()
+# else:
+# main.log.warn( "Did not pull new code so skipping mvn " +
+# "clean install" )
+ cleanInstallResult = main.ONOSbench.cleanInstall( mciTimeout= 1000 )
+ main.ONOSbench.getVersion( report=True )
+
+ main.step( "Creating ONOS package" )
+ packageResult = main.ONOSbench.onosPackage( opTimeout=500 )
+
+ main.step( "Installing ONOS package" )
+ onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
+ node=ONOS1Ip )
+
+ main.step( "Checking if ONOS is up yet" )
+ for i in range( 2 ):
+ onos1Isup = main.ONOSbench.isup( ONOS1Ip, timeout=420 )
+ if onos1Isup:
+ break
+ if not onos1Isup:
+ main.log.report( "ONOS1 didn't start!" )
+
+ cliResult = main.ONOScli.startOnosCli( ONOS1Ip,
+ commandlineTimeout=100, onosStartTimeout=600)
+
+ case1Result = ( cleanInstallResult and packageResult and
+ cellResult and verifyResult and
+ onos1InstallResult and
+ onos1Isup and cliResult )
+
+ utilities.assert_equals( expect=main.TRUE, actual=case1Result,
+ onpass="ONOS startup successful",
+ onfail="ONOS startup NOT successful" )
+
+ if case1Result == main.FALSE:
+ main.cleanup()
+ main.exit()
def CASE4( self, main ):
"""
@@ -33,14 +97,12 @@
import time
import json
from operator import eq
- # from datetime import datetime
from time import localtime, strftime
- main.case("The test case is to help to setup the TestON environment \
- and test new drivers" )
- # SDNIPJSONFILEPATH = "../tests/SdnIpTest/sdnip.json"
+ main.case("This case is to testing the functionality of SDN-IP with \
+ single ONOS instance" )
SDNIPJSONFILEPATH = \
- "/home/admin/workspace/onos/tools/package/config/sdnip.json"
+ "/home/admin/ONOS/tools/package/config/sdnip.json"
# all expected routes for all BGP peers
allRoutesExpected = []
main.step( "Start to generate routes for all BGP peers" )
@@ -79,37 +141,12 @@
routeIntentsExpected = routeIntentsExpectedHost3 + \
routeIntentsExpectedHost4 + routeIntentsExpectedHost5
- cellName = main.params[ 'ENV' ][ 'cellName' ]
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- main.step( "Set cell for ONOS-cli environment" )
- main.ONOScli.setCell( cellName )
- verifyResult = main.ONOSbench.verifyCell()
-
- main.log.report( "Removing raft logs" )
- main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
- main.ONOSbench.onosUninstall( ONOS1Ip )
-
- main.step( "Installing ONOS package" )
- onos1InstallResult = main.ONOSbench.onosInstall(
- options="-f", node=ONOS1Ip )
-
- main.step( "Checking if ONOS is up yet" )
- time.sleep( 150 )
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
-
- main.step( "Start ONOS-cli" )
-
- main.ONOScli.startOnosCli( ONOS1Ip )
-
main.step( "Get devices in the network" )
listResult = main.ONOScli.devices( jsonFormat=False )
main.log.info( listResult )
time.sleep( 10 )
- main.log.info( "Installing sdn-ip feature" )
- main.ONOScli.featureInstall( "onos-app-sdnip" )
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
time.sleep( 10 )
main.step( "Login all BGP peers and add routes into peers" )
@@ -138,8 +175,8 @@
prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
main.log.info( prefixesHostX )
for prefix in prefixesHostX:
- allRoutesExpected.append(
- prefix + "/" + "192.168.40." + str( i - 100 ) )
+ allRoutesExpected.append( prefix + "/" + "192.168.40."
+ + str( i - 100 ) )
routeIntentsExpectedHostX = \
main.QuaggaCliHost.generateExpectedOnePeerRouteIntents(
@@ -158,13 +195,11 @@
QuaggaCliHostX.addRoutes( prefixesHostX, 1 )
time.sleep( 60 )
-
# get routes inside SDN-IP
getRoutesResult = main.ONOScli.routes( jsonFormat=True )
- # parse routes from ONOS CLI
allRoutesActual = \
- main.QuaggaCliHost3.extractActualRoutes( getRoutesResult )
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
allRoutesStrExpected = str( sorted( allRoutesExpected ) )
allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
@@ -189,7 +224,281 @@
main.step( "Check MultiPointToSinglePointIntent intents installed" )
# routeIntentsExpected are generated when generating routes
- # get rpoute intents from ONOS CLI
+ # get route intents from ONOS CLI
+ routeIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualRouteIntentNum( getIntentsResult )
+ routeIntentsExpectedNum = 1030
+ main.log.info( "MultiPointToSinglePoint Intent Num expected is:" )
+ main.log.info( routeIntentsExpectedNum )
+ main.log.info( "MultiPointToSinglePoint Intent NUM Actual is:" )
+ main.log.info( routeIntentsActualNum )
+ utilities.assertEquals(
+ expect=True,
+ actual=eq( routeIntentsExpectedNum, routeIntentsActualNum ),
+ onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+ correct!***",
+ onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+ wrong!***" )
+
+ if( eq( routeIntentsExpectedNum, routeIntentsActualNum ) ):
+ main.log.report( "***MultiPointToSinglePoint Intents before \
+ deleting routes correct!***" )
+ else:
+ main.log.report( "***MultiPointToSinglePoint Intents before \
+ deleting routes wrong!***" )
+
+ main.step( "Check BGP PointToPointIntent intents installed" )
+
+ bgpIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualBgpIntentNum( getIntentsResult )
+ bgpIntentsExpectedNum = 624
+ main.log.info( "bgpIntentsExpected num is:" )
+ main.log.info( bgpIntentsExpectedNum )
+ main.log.info( "bgpIntentsActual num is:" )
+ main.log.info( bgpIntentsActualNum)
+ utilities.assertEquals(
+ expect=True,
+ actual=eq( bgpIntentsExpectedNum, bgpIntentsActualNum ),
+ onpass="***PointToPointIntent Intent Num in SDN-IP are correct!***",
+ onfail="***PointToPointIntent Intent Num in SDN-IP are wrong!***" )
+ if ( eq( bgpIntentsExpectedNum, bgpIntentsActualNum ) ):
+ main.log.report(
+ "***PointToPointIntent Intents in SDN-IP are correct!***" )
+ else:
+ main.log.report(
+ "***PointToPointIntent Intents in SDN-IP are wrong!***" )
+
+ #============================= Ping Test ========================
+ # Wait until all MultiPointToSinglePoint intents are in system
+ time.sleep( 20 )
+ pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+ pingTestResultsFile = \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-before-delete-routes-"\
+ + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+ pingTestResults = main.QuaggaCliHost.pingTest(
+ "1.168.30.100", pingTestScript, pingTestResultsFile )
+ main.log.info( pingTestResults )
+ time.sleep( 20 )
+
+ #============================= Deleting Routes ==================
+ main.step( "Check deleting routes installed" )
+ main.QuaggaCliHost3.deleteRoutes( prefixesHost3, 1 )
+ main.QuaggaCliHost4.deleteRoutes( prefixesHost4, 1 )
+ main.QuaggaCliHost5.deleteRoutes( prefixesHost5, 1 )
+
+ for i in range( 101, 201 ):
+ prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+ main.log.info( prefixesHostX )
+ QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+ QuaggaCliHostX.deleteRoutes( prefixesHostX, 1 )
+
+ getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+ main.log.info( "allRoutes_actual = " )
+ main.log.info( allRoutesActual )
+
+ utilities.assertEquals(
+ expect="[]", actual=str( allRoutesActual ),
+ onpass="***Route number in SDN-IP is 0, correct!***",
+ onfail="***Routes number in SDN-IP is not 0, wrong!***" )
+
+ if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+ main.log.report( "***Routes in SDN-IP after deleting correct!***" )
+ else:
+ main.log.report( "***Routes in SDN-IP after deleting wrong!***" )
+
+ main.step( "Check intents after deleting routes" )
+ getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+ routeIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualRouteIntentNum(
+ getIntentsResult )
+ main.log.info( "route Intents Actual Num is: " )
+ main.log.info( routeIntentsActualNum )
+ utilities.assertEquals(
+ expect=0, actual=routeIntentsActualNum,
+ onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is 0, \
+ correct!***",
+ onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is not 0, \
+ wrong!***" )
+
+ if( eq( 0, routeIntentsActualNum ) ):
+ main.log.report( "***MultiPointToSinglePoint Intents after \
+ deleting routes correct!***" )
+ else:
+ main.log.report( "***MultiPointToSinglePoint Intents after \
+ deleting routes wrong!***" )
+
+ time.sleep( 20 )
+ pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+ pingTestResultsFile = \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-"\
+ + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+ pingTestResults = main.QuaggaCliHost.pingTest(
+ "1.168.30.100", pingTestScript, pingTestResultsFile )
+ main.log.info( pingTestResults )
+ time.sleep( 100 )
+
+ def CASE_4_origin( self, main ):
+ """
+ Test the SDN-IP functionality
+ allRoutesExpected: all expected routes for all BGP peers
+ routeIntentsExpected: all expected MultiPointToSinglePointIntent \
+ intents
+ bgpIntentsExpected: expected PointToPointIntent intents
+ allRoutesActual: all routes from ONOS LCI
+ routeIntentsActual: actual MultiPointToSinglePointIntent intents from \
+ ONOS CLI
+ bgpIntentsActual: actual PointToPointIntent intents from ONOS CLI
+ """
+ import time
+ import json
+ from operator import eq
+ from time import localtime, strftime
+
+ main.case("This case is to testing the functionality of SDN-IP with \
+ single ONOS instance" )
+ SDNIPJSONFILEPATH = \
+ "/home/admin/ONOS/tools/package/config/sdnip.json"
+ # all expected routes for all BGP peers
+ allRoutesExpected = []
+ main.step( "Start to generate routes for all BGP peers" )
+ main.log.info( "Generate prefixes for host3" )
+ prefixesHost3 = main.QuaggaCliHost3.generatePrefixes( 3, 10 )
+ main.log.info( prefixesHost3 )
+ # generate route with next hop
+ for prefix in prefixesHost3:
+ allRoutesExpected.append( prefix + "/" + "192.168.20.1" )
+ routeIntentsExpectedHost3 = \
+ main.QuaggaCliHost3.generateExpectedOnePeerRouteIntents(
+ prefixesHost3, "192.168.20.1", "00:00:00:00:02:02",
+ SDNIPJSONFILEPATH )
+
+ main.log.info( "Generate prefixes for host4" )
+ prefixesHost4 = main.QuaggaCliHost4.generatePrefixes( 4, 10 )
+ main.log.info( prefixesHost4 )
+ # generate route with next hop
+ for prefix in prefixesHost4:
+ allRoutesExpected.append( prefix + "/" + "192.168.30.1" )
+ routeIntentsExpectedHost4 = \
+ main.QuaggaCliHost4.generateExpectedOnePeerRouteIntents(
+ prefixesHost4, "192.168.30.1", "00:00:00:00:03:01",
+ SDNIPJSONFILEPATH )
+
+ main.log.info( "Generate prefixes for host5" )
+ prefixesHost5 = main.QuaggaCliHost5.generatePrefixes( 5, 10 )
+ main.log.info( prefixesHost5 )
+ for prefix in prefixesHost5:
+ allRoutesExpected.append( prefix + "/" + "192.168.60.2" )
+ routeIntentsExpectedHost5 = \
+ main.QuaggaCliHost5.generateExpectedOnePeerRouteIntents(
+ prefixesHost5, "192.168.60.1", "00:00:00:00:06:02",
+ SDNIPJSONFILEPATH )
+
+ routeIntentsExpected = routeIntentsExpectedHost3 + \
+ routeIntentsExpectedHost4 + routeIntentsExpectedHost5
+
+ main.step( "Get devices in the network" )
+ listResult = main.ONOScli.devices( jsonFormat=False )
+ main.log.info( listResult )
+ time.sleep( 10 )
+ if branchName == "onos-1.1":
+ main.log.info( "Installing sdn-ip feature" )
+ main.ONOScli.featureInstall( "onos-app-sdnip" )
+ else:
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
+ #main.log.info( "Installing sdn-ip feature" )
+ #main.ONOScli.featureInstall( "onos-app-sdnip" )
+ time.sleep( 10 )
+ main.step( "Login all BGP peers and add routes into peers" )
+
+ main.log.info( "Login Quagga CLI on host3" )
+ main.QuaggaCliHost3.loginQuagga( "1.168.30.2" )
+ main.log.info( "Enter configuration model of Quagga CLI on host3" )
+ main.QuaggaCliHost3.enterConfig( 64514 )
+ main.log.info( "Add routes to Quagga on host3" )
+ main.QuaggaCliHost3.addRoutes( prefixesHost3, 1 )
+
+ main.log.info( "Login Quagga CLI on host4" )
+ main.QuaggaCliHost4.loginQuagga( "1.168.30.3" )
+ main.log.info( "Enter configuration model of Quagga CLI on host4" )
+ main.QuaggaCliHost4.enterConfig( 64516 )
+ main.log.info( "Add routes to Quagga on host4" )
+ main.QuaggaCliHost4.addRoutes( prefixesHost4, 1 )
+
+ main.log.info( "Login Quagga CLI on host5" )
+ main.QuaggaCliHost5.loginQuagga( "1.168.30.5" )
+ main.log.info( "Enter configuration model of Quagga CLI on host5" )
+ main.QuaggaCliHost5.enterConfig( 64521 )
+ main.log.info( "Add routes to Quagga on host5" )
+ main.QuaggaCliHost5.addRoutes( prefixesHost5, 1 )
+
+ for i in range( 101, 201 ):
+ prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+ main.log.info( prefixesHostX )
+ for prefix in prefixesHostX:
+ allRoutesExpected.append( prefix + "/" + "192.168.40."
+ + str( i - 100 ) )
+
+ routeIntentsExpectedHostX = \
+ main.QuaggaCliHost.generateExpectedOnePeerRouteIntents(
+ prefixesHostX, "192.168.40." + str( i - 100 ),
+ "00:00:%02d:00:00:90" % ( i - 101 ), SDNIPJSONFILEPATH )
+ routeIntentsExpected = routeIntentsExpected + \
+ routeIntentsExpectedHostX
+
+ main.log.info( "Login Quagga CLI on host" + str( i ) )
+ QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+ QuaggaCliHostX.loginQuagga( "1.168.30." + str( i ) )
+ main.log.info(
+ "Enter configuration model of Quagga CLI on host" + str( i ) )
+ QuaggaCliHostX.enterConfig( 65000 + i - 100 )
+ main.log.info( "Add routes to Quagga on host" + str( i ) )
+ QuaggaCliHostX.addRoutes( prefixesHostX, 1 )
+
+ time.sleep( 60 )
+ # get routes inside SDN-IP
+ getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+
+ # parse routes from ONOS CLI
+ #if branchName == "master":
+ # allRoutesActual = \
+ # main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+ #elif branchName == "onos-1.0":
+ # allRoutesActual = \
+ # main.QuaggaCliHost3.extractActualRoutesOneDotZero( getRoutesResult )
+ #else:
+ # main.log("ONOS is on wrong branch")
+ # exit
+
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+
+ allRoutesStrExpected = str( sorted( allRoutesExpected ) )
+ allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
+ main.step( "Check routes installed" )
+ main.log.info( "Routes expected:" )
+ main.log.info( allRoutesStrExpected )
+ main.log.info( "Routes get from ONOS CLI:" )
+ main.log.info( allRoutesStrActual )
+ utilities.assertEquals(
+ expect=allRoutesStrExpected, actual=allRoutesStrActual,
+ onpass="***Routes in SDN-IP are correct!***",
+ onfail="***Routes in SDN-IP are wrong!***" )
+ if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+ main.log.report(
+ "***Routes in SDN-IP after adding routes are correct!***" )
+ else:
+ main.log.report(
+ "***Routes in SDN-IP after adding routes are wrong!***" )
+
+ time.sleep( 20 )
+ getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+
+ main.step( "Check MultiPointToSinglePointIntent intents installed" )
+ # routeIntentsExpected are generated when generating routes
+ # get route intents from ONOS CLI
routeIntentsActual = \
main.QuaggaCliHost3.extractActualRouteIntents(
getIntentsResult )
@@ -243,7 +552,7 @@
"***PointToPointIntent Intents in SDN-IP are wrong!***" )
#============================= Ping Test ========================
- # wait until all MultiPointToSinglePoint
+ # Wait until all MultiPointToSinglePoint intents are in system
time.sleep( 20 )
pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
pingTestResultsFile = \
@@ -306,22 +615,19 @@
time.sleep( 20 )
pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
pingTestResultsFile = \
- "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-" \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-"\
+ strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
pingTestResults = main.QuaggaCliHost.pingTest(
"1.168.30.100", pingTestScript, pingTestResultsFile )
main.log.info( pingTestResults )
time.sleep( 100 )
- # main.step( "Test whether Mininet is started" )
- # main.Mininet2.handle.sendline( "xterm host1" )
- # main.Mininet2.handle.expect( "mininet>" )
-
def CASE3( self, main ):
"""
Test the SDN-IP functionality
allRoutesExpected: all expected routes for all BGP peers
- routeIntentsExpected: all expected MultiPointToSinglePointIntent intents
+ routeIntentsExpected: all expected MultiPointToSinglePointIntent \
+ intents
bgpIntentsExpected: expected PointToPointIntent intents
allRoutesActual: all routes from ONOS LCI
routeIntentsActual: actual MultiPointToSinglePointIntent intents from \
@@ -338,7 +644,7 @@
environment and test new drivers" )
# SDNIPJSONFILEPATH = "../tests/SdnIpTest/sdnip.json"
SDNIPJSONFILEPATH = \
- "/home/admin/workspace/onos/tools/package/config/sdnip.json"
+ "/home/admin/ONOS/tools/package/config/sdnip.json"
# all expected routes for all BGP peers
allRoutesExpected = []
main.step( "Start to generate routes for all BGP peers" )
@@ -396,8 +702,12 @@
listResult = main.ONOScli.devices( jsonFormat=False )
main.log.info( listResult )
time.sleep( 10 )
- main.log.info( "Installing sdn-ip feature" )
- main.ONOScli.featureInstall( "onos-app-sdnip" )
+ if branchName == "onos-1.1":
+ main.log.info( "Installing sdn-ip feature" )
+ main.ONOScli.featureInstall( "onos-app-sdnip" )
+ else:
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
time.sleep( 10 )
main.step( "Login all BGP peers and add routes into peers" )
diff --git a/TestON/tests/TopoPerfNext/Backup/TopoPerfNext.py b/TestON/tests/TopoPerfNext/Backup/TopoPerfNext.py
deleted file mode 100644
index cc40b94..0000000
--- a/TestON/tests/TopoPerfNext/Backup/TopoPerfNext.py
+++ /dev/null
@@ -1,1710 +0,0 @@
-#TopoPerfNext
-#
-#Topology Performance test for ONOS-next
-#
-#andrew@onlab.us
-#
-#If your machine does not come with numpy
-#run the following command:
-#sudo apt-get install python-numpy python-scipy
-
-import time
-import sys
-import os
-import re
-
-class TopoPerfNext:
- def __init__(self):
- self.default = ''
-
- def CASE1(self, main):
- '''
- ONOS startup sequence
- '''
- import time
-
- ## Global cluster count for scale-out purposes
- global cluster_count
- #Set initial cluster count
- cluster_count = 1
- ##
-
- cell_name = main.params['ENV']['cellName']
-
- git_pull = main.params['GIT']['autoPull']
- checkout_branch = main.params['GIT']['checkout']
-
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS3_ip = main.params['CTRL']['ip3']
-
- #### Hardcoded ONOS nodes particular to my env ####
- ONOS4_ip = "10.128.174.4"
- ONOS5_ip = "10.128.174.5"
- ONOS6_ip = "10.128.174.6"
- ONOS7_ip = "10.128.174.7"
- #### ####
-
- MN1_ip = main.params['MN']['ip1']
- BENCH_ip = main.params['BENCH']['ip']
-
- topo_cfg_file = main.params['TEST']['topo_config_file']
- topo_cfg_name = main.params['TEST']['topo_config_name']
-
- main.case("Setting up test environment")
- main.log.info("Copying topology event accumulator config"+\
- " to ONOS /package/etc")
- main.ONOSbench.handle.sendline("cp ~/"+\
- topo_cfg_file+\
- " ~/ONOS/tools/package/etc/"+\
- topo_cfg_name)
- main.ONOSbench.handle.expect("\$")
-
- main.log.report("Setting up test environment")
-
- main.step("Cleaning previously installed ONOS if any")
- main.ONOSbench.onos_uninstall(node_ip=ONOS2_ip)
- main.ONOSbench.onos_uninstall(node_ip=ONOS3_ip)
- main.ONOSbench.onos_uninstall(node_ip=ONOS4_ip)
- main.ONOSbench.onos_uninstall(node_ip=ONOS5_ip)
- main.ONOSbench.onos_uninstall(node_ip=ONOS6_ip)
- main.ONOSbench.onos_uninstall(node_ip=ONOS7_ip)
-
- main.step("Creating cell file")
- cell_file_result = main.ONOSbench.create_cell_file(
- BENCH_ip, cell_name, MN1_ip, "onos-core,onos-app-metrics",
- ONOS1_ip)
-
- main.step("Applying cell file to environment")
- cell_apply_result = main.ONOSbench.set_cell(cell_name)
- verify_cell_result = main.ONOSbench.verify_cell()
-
- #NOTE: This step may be removed after proper
- # copy cat log functionality
- main.step("Removing raft/copy-cat logs from ONOS nodes")
- main.ONOSbench.onos_remove_raft_logs()
- time.sleep(30)
-
- main.step("Git checkout and pull "+checkout_branch)
- if git_pull == 'on':
- checkout_result = \
- main.ONOSbench.git_checkout(checkout_branch)
- pull_result = main.ONOSbench.git_pull()
- else:
- checkout_result = main.TRUE
- pull_result = main.TRUE
- main.log.info("Skipped git checkout and pull")
-
- main.log.report("Commit information - ")
- main.ONOSbench.get_version(report=True)
-
- main.step("Using mvn clean & install")
- mvn_result = main.ONOSbench.clean_install()
- mvn_result = main.TRUE
-
- main.step("Set cell for ONOS cli env")
- main.ONOS1cli.set_cell(cell_name)
- #main.ONOS2cli.set_cell(cell_name)
- #main.ONOS3cli.set_cell(cell_name)
-
- main.step("Creating ONOS package")
- package_result = main.ONOSbench.onos_package()
-
- main.step("Installing ONOS package")
- install1_result = main.ONOSbench.onos_install(node=ONOS1_ip)
- #install2_result = main.ONOSbench.onos_install(node=ONOS2_ip)
- #install3_result = main.ONOSbench.onos_install(node=ONOS3_ip)
-
- time.sleep(10)
-
- main.step("Start onos cli")
- cli1 = main.ONOS1cli.start_onos_cli(ONOS1_ip)
- #cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
- #cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
-
- utilities.assert_equals(expect=main.TRUE,
- actual= cell_file_result and cell_apply_result and\
- verify_cell_result and checkout_result and\
- pull_result and mvn_result and\
- install1_result, #and install2_result and\
- #install3_result,
- onpass="Test Environment setup successful",
- onfail="Failed to setup test environment")
-
- def CASE2(self, main):
- '''
- Assign s1 to ONOS1 and measure latency
-
- There are 4 levels of latency measurements to this test:
- 1) End-to-end measurement: Complete end-to-end measurement
- from TCP (SYN/ACK) handshake to Graph change
- 2) OFP-to-graph measurement: 'ONOS processing' snippet of
- measurement from OFP Vendor message to Graph change
- 3) OFP-to-device measurement: 'ONOS processing without
- graph change' snippet of measurement from OFP vendor
- message to Device change timestamp
- 4) T0-to-device measurement: Measurement that includes
- the switch handshake to devices timestamp without
- the graph view change. (TCP handshake -> Device
- change)
- '''
- import time
- import subprocess
- import json
- import requests
- import os
- import numpy
- global cluster_count
-
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS7_ip = main.params['CTRL']['ip7']
-
- ONOS_user = main.params['CTRL']['user']
-
- default_sw_port = main.params['CTRL']['port1']
-
- #Number of iterations of case
- num_iter = main.params['TEST']['numIter']
- #Number of first 'x' iterations to ignore:
- iter_ignore = int(main.params['TEST']['iterIgnore'])
-
- #Timestamp 'keys' for json metrics output.
- #These are subject to change, hence moved into params
- deviceTimestamp = main.params['JSON']['deviceTimestamp']
- graphTimestamp = main.params['JSON']['graphTimestamp']
-
- debug_mode = main.params['TEST']['debugMode']
- onos_log = main.params['TEST']['onosLogFile']
-
- #Threshold for the test
- threshold_str = main.params['TEST']['singleSwThreshold']
- threshold_obj = threshold_str.split(",")
- threshold_min = int(threshold_obj[0])
- threshold_max = int(threshold_obj[1])
-
- #List of switch add latency collected from
- #all iterations
- latency_end_to_end_list = []
- latency_ofp_to_graph_list = []
- latency_ofp_to_device_list = []
- latency_t0_to_device_list = []
- latency_tcp_to_ofp_list = []
-
- #Directory/file to store tshark results
- tshark_of_output = "/tmp/tshark_of_topo.txt"
- tshark_tcp_output = "/tmp/tshark_tcp_topo.txt"
-
- #String to grep in tshark output
- tshark_tcp_string = "TCP 74 "+default_sw_port
- tshark_of_string = "OFP 86 Vendor"
-
- #Initialize assertion to TRUE
- assertion = main.TRUE
-
- local_time = time.strftime('%x %X')
- local_time = local_time.replace("/","")
- local_time = local_time.replace(" ","_")
- local_time = local_time.replace(":","")
- if debug_mode == 'on':
- main.ONOS1.tshark_pcap("eth0",
- "/tmp/single_sw_lat_pcap_"+local_time)
-
- main.log.info("Debug mode is on")
-
- main.log.report("Latency of adding one switch to controller")
- main.log.report("First "+str(iter_ignore)+" iterations ignored"+
- " for jvm warmup time")
- main.log.report("Total iterations of test: "+str(num_iter))
-
- for i in range(0, int(num_iter)):
- main.log.info("Starting tshark capture")
-
- #* TCP [ACK, SYN] is used as t0_a, the
- # very first "exchange" between ONOS and
- # the switch for end-to-end measurement
- #* OFP [Stats Reply] is used for t0_b
- # the very last OFP message between ONOS
- # and the switch for ONOS measurement
- main.ONOS1.tshark_grep(tshark_tcp_string,
- tshark_tcp_output)
- main.ONOS1.tshark_grep(tshark_of_string,
- tshark_of_output)
-
- #Wait and ensure tshark is started and
- #capturing
- time.sleep(10)
-
- main.log.info("Assigning s1 to controller")
-
- main.Mininet1.assign_sw_controller(sw="1",
- ip1=ONOS1_ip, port1=default_sw_port)
-
- #Wait and ensure switch is assigned
- #before stopping tshark
- time.sleep(30)
-
- main.log.info("Stopping all Tshark processes")
- main.ONOS1.stop_tshark()
-
- #tshark output is saved in ONOS. Use subprocess
- #to copy over files to TestON for parsing
- main.log.info("Copying over tshark files")
-
- #TCP CAPTURE ****
- #Copy the tshark output from ONOS machine to
- #TestON machine in tshark_tcp_output directory>file
- os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
- tshark_tcp_output+" /tmp/")
- tcp_file = open(tshark_tcp_output, 'r')
- temp_text = tcp_file.readline()
- temp_text = temp_text.split(" ")
-
- main.log.info("Object read in from TCP capture: "+
- str(temp_text))
- if len(temp_text) > 1:
- t0_tcp = float(temp_text[1])*1000.0
- else:
- main.log.error("Tshark output file for TCP"+
- " returned unexpected results")
- t0_tcp = 0
- assertion = main.FALSE
-
- tcp_file.close()
- #****************
-
- #OF CAPTURE ****
- os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
- tshark_of_output+" /tmp/")
- of_file = open(tshark_of_output, 'r')
-
- line_ofp = ""
- #Read until last line of file
- while True:
- temp_text = of_file.readline()
- if temp_text !='':
- line_ofp = temp_text
- else:
- break
- obj = line_ofp.split(" ")
-
- main.log.info("Object read in from OFP capture: "+
- str(line_ofp))
-
- if len(line_ofp) > 1:
- t0_ofp = float(obj[1])*1000.0
- else:
- main.log.error("Tshark output file for OFP"+
- " returned unexpected results")
- t0_ofp = 0
- assertion = main.FALSE
-
- of_file.close()
- #****************
-
- json_str_1 = main.ONOS1cli.topology_events_metrics()
- #Initialize scale-out variables
- json_str_2 = ""
- json_str_3 = ""
- json_str_4 = ""
- json_str_5 = ""
- json_str_6 = ""
- json_str_7 = ""
-
- json_obj_1 = json.loads(json_str_1)
- json_obj_2 = json.loads(json_str_2)
- json_obj_3 = json.loads(json_str_3)
- #Initialize scale-out variables
- json_obj_4 = ""
- json_obj_5 = ""
- json_obj_6 = ""
- json_obj_7 = ""
-
- #Include scale-out measurements when applicable
- if cluster_count == 5:
- json_str_4 = main.ONOS4cli.topology_events_metrics()
- json_str_5 = main.ONOS5cli.topology_events_metrics()
-
- json_obj_4 = json.loads(json_str_4)
- json_obj_5 = json.loads(json_str_5)
- elif cluster_count == 6:
- main.log.info("TODO: create even number cluster events")
- elif cluster_count == 7:
- json_str_6 = main.ONOS6cli.topology_events_metrics()
- json_str_7 = main.ONOS7cli.topology_events_metrics()
-
- json_obj_6 = json.loads(json_str_6)
- json_obj_7 = json.loads(json_str_7)
-
- #Obtain graph timestamp. This timestsamp captures
- #the epoch time at which the topology graph was updated.
- graph_timestamp_1 = \
- json_obj_1[graphTimestamp]['value']
- graph_timestamp_2 = \
- json_obj_2[graphTimestamp]['value']
- graph_timestamp_3 = \
- json_obj_3[graphTimestamp]['value']
-
- #Obtain device timestamp. This timestamp captures
- #the epoch time at which the device event happened
- device_timestamp_1 = \
- json_obj_1[deviceTimestamp]['value']
- device_timestamp_2 = \
- json_obj_2[deviceTimestamp]['value']
- device_timestamp_3 = \
- json_obj_3[deviceTimestamp]['value']
-
- #t0 to device processing latency
- delta_device_1 = int(device_timestamp_1) - int(t0_tcp)
- delta_device_2 = int(device_timestamp_2) - int(t0_tcp)
- delta_device_3 = int(device_timestamp_3) - int(t0_tcp)
-
- #Get average of delta from all instances
- avg_delta_device = \
- (int(delta_device_1)+\
- int(delta_device_2)+\
- int(delta_device_3)) / 3
-
- #Ensure avg delta meets the threshold before appending
- if avg_delta_device > 0.0 and avg_delta_device < 10000\
- and int(i) > iter_ignore:
- latency_t0_to_device_list.append(avg_delta_device)
- else:
- main.log.info("Results for t0-to-device ignored"+\
- "due to excess in threshold / warmup iteration.")
-
- #t0 to graph processing latency (end-to-end)
- delta_graph_1 = int(graph_timestamp_1) - int(t0_tcp)
- delta_graph_2 = int(graph_timestamp_2) - int(t0_tcp)
- delta_graph_3 = int(graph_timestamp_3) - int(t0_tcp)
-
- #Get average of delta from all instances
- #TODO: use max delta graph
- #max_delta_graph = max(three)
- avg_delta_graph = \
- (int(delta_graph_1)+\
- int(delta_graph_2)+\
- int(delta_graph_3)) / 3
-
- #Ensure avg delta meets the threshold before appending
- if avg_delta_graph > 0.0 and avg_delta_graph < 10000\
- and int(i) > iter_ignore:
- latency_end_to_end_list.append(avg_delta_graph)
- else:
- main.log.info("Results for end-to-end ignored"+\
- "due to excess in threshold")
-
- #ofp to graph processing latency (ONOS processing)
- delta_ofp_graph_1 = int(graph_timestamp_1) - int(t0_ofp)
- delta_ofp_graph_2 = int(graph_timestamp_2) - int(t0_ofp)
- delta_ofp_graph_3 = int(graph_timestamp_3) - int(t0_ofp)
-
- avg_delta_ofp_graph = \
- (int(delta_ofp_graph_1)+\
- int(delta_ofp_graph_2)+\
- int(delta_ofp_graph_3)) / 3
-
- if avg_delta_ofp_graph > threshold_min \
- and avg_delta_ofp_graph < threshold_max\
- and int(i) > iter_ignore:
- latency_ofp_to_graph_list.append(avg_delta_ofp_graph)
- elif avg_delta_ofp_graph > (-10) and \
- avg_delta_ofp_graph < 0.0 and\
- int(i) > iter_ignore:
- main.log.info("Sub-millisecond result likely; "+
- "negative result was rounded to 0")
- #NOTE: Current metrics framework does not
- #support sub-millisecond accuracy. Therefore,
- #if the result is negative, we can reasonably
- #conclude sub-millisecond results and just
- #append the best rounded effort - 0 ms.
- latency_ofp_to_graph_list.append(0)
- else:
- main.log.info("Results for ofp-to-graph "+\
- "ignored due to excess in threshold")
-
- #ofp to device processing latency (ONOS processing)
- delta_ofp_device_1 = float(device_timestamp_1) - float(t0_ofp)
- delta_ofp_device_2 = float(device_timestamp_2) - float(t0_ofp)
- delta_ofp_device_3 = float(device_timestamp_3) - float(t0_ofp)
-
- avg_delta_ofp_device = \
- (float(delta_ofp_device_1)+\
- float(delta_ofp_device_2)+\
- float(delta_ofp_device_3)) / 3
-
- #NOTE: ofp - delta measurements are occasionally negative
- # due to system time misalignment.
- latency_ofp_to_device_list.append(avg_delta_ofp_device)
-
- delta_ofp_tcp = int(t0_ofp) - int(t0_tcp)
- if delta_ofp_tcp > threshold_min \
- and delta_ofp_tcp < threshold_max and\
- int(i) > iter_ignore:
- latency_tcp_to_ofp_list.append(delta_ofp_tcp)
- else:
- main.log.info("Results fo tcp-to-ofp "+\
- "ignored due to excess in threshold")
-
- #TODO:
- #Fetch logs upon threshold excess
-
- main.log.info("ONOS1 delta end-to-end: "+
- str(delta_graph_1) + " ms")
- main.log.info("ONOS2 delta end-to-end: "+
- str(delta_graph_2) + " ms")
- main.log.info("ONOS3 delta end-to-end: "+
- str(delta_graph_3) + " ms")
-
- main.log.info("ONOS1 delta OFP - graph: "+
- str(delta_ofp_graph_1) + " ms")
- main.log.info("ONOS2 delta OFP - graph: "+
- str(delta_ofp_graph_2) + " ms")
- main.log.info("ONOS3 delta OFP - graph: "+
- str(delta_ofp_graph_3) + " ms")
-
- main.log.info("ONOS1 delta device - t0: "+
- str(delta_device_1) + " ms")
- main.log.info("ONOS2 delta device - t0: "+
- str(delta_device_2) + " ms")
- main.log.info("ONOS3 delta device - t0: "+
- str(delta_device_3) + " ms")
-
- main.log.info("TCP to OFP delta: "+
- str(delta_ofp_tcp) + " ms")
- #main.log.info("ONOS1 delta OFP - device: "+
- # str(delta_ofp_device_1) + " ms")
- #main.log.info("ONOS2 delta OFP - device: "+
- # str(delta_ofp_device_2) + " ms")
- #main.log.info("ONOS3 delta OFP - device: "+
- # str(delta_ofp_device_3) + " ms")
-
- main.step("Remove switch from controller")
- main.Mininet1.delete_sw_controller("s1")
-
- time.sleep(5)
-
- #END of for loop iteration
-
- #If there is at least 1 element in each list,
- #pass the test case
- if len(latency_end_to_end_list) > 0 and\
- len(latency_ofp_to_graph_list) > 0 and\
- len(latency_ofp_to_device_list) > 0 and\
- len(latency_t0_to_device_list) > 0 and\
- len(latency_tcp_to_ofp_list) > 0:
- assertion = main.TRUE
- elif len(latency_end_to_end_list) == 0:
- #The appending of 0 here is to prevent
- #the min,max,sum functions from failing
- #below
- latency_end_to_end_list.append(0)
- assertion = main.FALSE
- elif len(latency_ofp_to_graph_list) == 0:
- latency_ofp_to_graph_list.append(0)
- assertion = main.FALSE
- elif len(latency_ofp_to_device_list) == 0:
- latency_ofp_to_device_list.append(0)
- assertion = main.FALSE
- elif len(latency_t0_to_device_list) == 0:
- latency_t0_to_device_list.append(0)
- assertion = main.FALSE
- elif len(latency_tcp_to_ofp_list) == 0:
- latency_tcp_to_ofp_list.append(0)
- assertion = main.FALSE
-
- #Calculate min, max, avg of latency lists
- latency_end_to_end_max = \
- int(max(latency_end_to_end_list))
- latency_end_to_end_min = \
- int(min(latency_end_to_end_list))
- latency_end_to_end_avg = \
- (int(sum(latency_end_to_end_list)) / \
- len(latency_end_to_end_list))
- latency_end_to_end_std_dev = \
- str(round(numpy.std(latency_end_to_end_list),1))
-
- latency_ofp_to_graph_max = \
- int(max(latency_ofp_to_graph_list))
- latency_ofp_to_graph_min = \
- int(min(latency_ofp_to_graph_list))
- latency_ofp_to_graph_avg = \
- (int(sum(latency_ofp_to_graph_list)) / \
- len(latency_ofp_to_graph_list))
- latency_ofp_to_graph_std_dev = \
- str(round(numpy.std(latency_ofp_to_graph_list),1))
-
- latency_ofp_to_device_max = \
- int(max(latency_ofp_to_device_list))
- latency_ofp_to_device_min = \
- int(min(latency_ofp_to_device_list))
- latency_ofp_to_device_avg = \
- (int(sum(latency_ofp_to_device_list)) / \
- len(latency_ofp_to_device_list))
- latency_ofp_to_device_std_dev = \
- str(round(numpy.std(latency_ofp_to_device_list),1))
-
- latency_t0_to_device_max = \
- int(max(latency_t0_to_device_list))
- latency_t0_to_device_min = \
- int(min(latency_t0_to_device_list))
- latency_t0_to_device_avg = \
- (int(sum(latency_t0_to_device_list)) / \
- len(latency_t0_to_device_list))
- latency_ofp_to_device_std_dev = \
- str(round(numpy.std(latency_t0_to_device_list),1))
-
- latency_tcp_to_ofp_max = \
- int(max(latency_tcp_to_ofp_list))
- latency_tcp_to_ofp_min = \
- int(min(latency_tcp_to_ofp_list))
- latency_tcp_to_ofp_avg = \
- (int(sum(latency_tcp_to_ofp_list)) / \
- len(latency_tcp_to_ofp_list))
- latency_tcp_to_ofp_std_dev = \
- str(round(numpy.std(latency_tcp_to_ofp_list),1))
-
- main.log.report("Switch add - End-to-end latency: "+\
- "Avg: "+str(latency_end_to_end_avg)+" ms "+
- "Std Deviation: "+latency_end_to_end_std_dev+" ms")
- main.log.report("Switch add - OFP-to-Graph latency: "+\
- "Note: results are not accurate to sub-millisecond. "+
- "Any sub-millisecond results are rounded to 0 ms. ")
- main.log.report("Avg: "+str(latency_ofp_to_graph_avg)+" ms "+
- "Std Deviation: "+latency_ofp_to_graph_std_dev+" ms")
- main.log.report("Switch add - TCP-to-OFP latency: "+\
- "Avg: "+str(latency_tcp_to_ofp_avg)+" ms "+
- "Std Deviation: "+latency_tcp_to_ofp_std_dev+" ms")
-
- if debug_mode == 'on':
- main.ONOS1.cp_logs_to_dir("/opt/onos/log/karaf.log",
- "/tmp/", copy_file_name="sw_lat_karaf")
-
- utilities.assert_equals(expect=main.TRUE, actual=assertion,
- onpass="Switch latency test successful",
- onfail="Switch latency test failed")
-
- def CASE3(self, main):
- '''
- Bring port up / down and measure latency.
- Port enable / disable is simulated by ifconfig up / down
-
- In ONOS-next, we must ensure that the port we are
- manipulating is connected to another switch with a valid
- connection. Otherwise, graph view will not be updated.
- '''
- import time
- import subprocess
- import os
- import requests
- import json
- import numpy
-
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS_user = main.params['CTRL']['user']
-
- default_sw_port = main.params['CTRL']['port1']
-
- assertion = main.TRUE
- #Number of iterations of case
- num_iter = main.params['TEST']['numIter']
-
- #Timestamp 'keys' for json metrics output.
- #These are subject to change, hence moved into params
- deviceTimestamp = main.params['JSON']['deviceTimestamp']
- graphTimestamp = main.params['JSON']['graphTimestamp']
-
- debug_mode = main.params['TEST']['debugMode']
-
- local_time = time.strftime('%x %X')
- local_time = local_time.replace("/","")
- local_time = local_time.replace(" ","_")
- local_time = local_time.replace(":","")
- if debug_mode == 'on':
- main.ONOS1.tshark_pcap("eth0",
- "/tmp/port_lat_pcap_"+local_time)
-
- #Threshold for this test case
- up_threshold_str = main.params['TEST']['portUpThreshold']
- down_threshold_str = main.params['TEST']['portDownThreshold']
-
- up_threshold_obj = up_threshold_str.split(",")
- down_threshold_obj = down_threshold_str.split(",")
-
- up_threshold_min = int(up_threshold_obj[0])
- up_threshold_max = int(up_threshold_obj[1])
-
- down_threshold_min = int(down_threshold_obj[0])
- down_threshold_max = int(down_threshold_obj[1])
-
- #NOTE: Some hardcoded variables you may need to configure
- # besides the params
-
- tshark_port_status = "OFP 130 Port Status"
-
- tshark_port_up = "/tmp/tshark_port_up.txt"
- tshark_port_down = "/tmp/tshark_port_down.txt"
- interface_config = "s1-eth1"
-
- main.log.report("Port enable / disable latency")
- main.log.report("Simulated by ifconfig up / down")
- main.log.report("Total iterations of test: "+str(num_iter))
-
- main.step("Assign switches s1 and s2 to controller 1")
- main.Mininet1.assign_sw_controller(sw="1",ip1=ONOS1_ip,
- port1=default_sw_port)
- main.Mininet1.assign_sw_controller(sw="2",ip1=ONOS1_ip,
- port1=default_sw_port)
-
- #Give enough time for metrics to propagate the
- #assign controller event. Otherwise, these events may
- #carry over to our measurements
- time.sleep(15)
-
- port_up_device_to_ofp_list = []
- port_up_graph_to_ofp_list = []
- port_down_device_to_ofp_list = []
- port_down_graph_to_ofp_list = []
-
- for i in range(0, int(num_iter)):
- main.step("Starting wireshark capture for port status down")
- main.ONOS1.tshark_grep(tshark_port_status,
- tshark_port_down)
-
- time.sleep(5)
-
- #Disable interface that is connected to switch 2
- main.step("Disable port: "+interface_config)
- main.Mininet1.handle.sendline("sh ifconfig "+
- interface_config+" down")
- main.Mininet1.handle.expect("mininet>")
-
- time.sleep(3)
- main.ONOS1.tshark_stop()
-
- main.step("Obtain t1 by metrics call")
- json_str_up_1 = main.ONOS1cli.topology_events_metrics()
- json_str_up_2 = main.ONOS2cli.topology_events_metrics()
- json_str_up_3 = main.ONOS3cli.topology_events_metrics()
-
- json_obj_1 = json.loads(json_str_up_1)
- json_obj_2 = json.loads(json_str_up_2)
- json_obj_3 = json.loads(json_str_up_3)
-
- #Copy tshark output file from ONOS to TestON instance
- #/tmp directory
- os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
- tshark_port_down+" /tmp/")
-
- f_port_down = open(tshark_port_down, 'r')
- #Get first line of port down event from tshark
- f_line = f_port_down.readline()
- obj_down = f_line.split(" ")
- if len(f_line) > 0:
- timestamp_begin_pt_down = int(float(obj_down[1])*1000)
- main.log.info("Port down begin timestamp: "+
- str(timestamp_begin_pt_down))
- else:
- main.log.info("Tshark output file returned unexpected"+
- " results: "+str(obj_down))
- timestamp_begin_pt_down = 0
-
- f_port_down.close()
-
- main.log.info("TEST tshark obj: "+str(obj_down))
-
- time.sleep(3)
-
- #Obtain graph timestamp. This timestsamp captures
- #the epoch time at which the topology graph was updated.
- graph_timestamp_1 = \
- json_obj_1[graphTimestamp]['value']
- graph_timestamp_2 = \
- json_obj_2[graphTimestamp]['value']
- graph_timestamp_3 = \
- json_obj_3[graphTimestamp]['value']
-
- main.log.info("TEST graph timestamp ONOS1: "+
- str(graph_timestamp_1))
-
- #Obtain device timestamp. This timestamp captures
- #the epoch time at which the device event happened
- device_timestamp_1 = \
- json_obj_1[deviceTimestamp]['value']
- device_timestamp_2 = \
- json_obj_2[deviceTimestamp]['value']
- device_timestamp_3 = \
- json_obj_3[deviceTimestamp]['value']
-
- #Get delta between graph event and OFP
- pt_down_graph_to_ofp_1 = int(graph_timestamp_1) -\
- int(timestamp_begin_pt_down)
- pt_down_graph_to_ofp_2 = int(graph_timestamp_2) -\
- int(timestamp_begin_pt_down)
- pt_down_graph_to_ofp_3 = int(graph_timestamp_3) -\
- int(timestamp_begin_pt_down)
-
- #Get delta between device event and OFP
- pt_down_device_to_ofp_1 = int(device_timestamp_1) -\
- int(timestamp_begin_pt_down)
- pt_down_device_to_ofp_2 = int(device_timestamp_2) -\
- int(timestamp_begin_pt_down)
- pt_down_device_to_ofp_3 = int(device_timestamp_3) -\
- int(timestamp_begin_pt_down)
-
- #Caluclate average across clusters
- pt_down_graph_to_ofp_avg =\
- (int(pt_down_graph_to_ofp_1) +
- int(pt_down_graph_to_ofp_2) +
- int(pt_down_graph_to_ofp_3)) / 3
- pt_down_device_to_ofp_avg = \
- (int(pt_down_device_to_ofp_1) +
- int(pt_down_device_to_ofp_2) +
- int(pt_down_device_to_ofp_3)) / 3
-
- if pt_down_graph_to_ofp_avg > down_threshold_min and \
- pt_down_graph_to_ofp_avg < down_threshold_max:
- port_down_graph_to_ofp_list.append(
- pt_down_graph_to_ofp_avg)
- main.log.info("Port down: graph to ofp avg: "+
- str(pt_down_graph_to_ofp_avg) + " ms")
- else:
- main.log.info("Average port down graph-to-ofp result" +
- " exceeded the threshold: "+
- str(pt_down_graph_to_ofp_avg))
-
- if pt_down_device_to_ofp_avg > 0 and \
- pt_down_device_to_ofp_avg < 1000:
- port_down_device_to_ofp_list.append(
- pt_down_device_to_ofp_avg)
- main.log.info("Port down: device to ofp avg: "+
- str(pt_down_device_to_ofp_avg) + " ms")
- else:
- main.log.info("Average port down device-to-ofp result" +
- " exceeded the threshold: "+
- str(pt_down_device_to_ofp_avg))
-
- #Port up events
- main.step("Enable port and obtain timestamp")
- main.step("Starting wireshark capture for port status up")
- main.ONOS1.tshark_grep(tshark_port_status, tshark_port_up)
- time.sleep(5)
-
- main.Mininet1.handle.sendline("sh ifconfig "+
- interface_config+" up")
- main.Mininet1.handle.expect("mininet>")
-
- #Allow time for tshark to capture event
- time.sleep(3)
- main.ONOS1.tshark_stop()
-
- #Obtain metrics shortly afterwards
- #This timestsamp captures
- #the epoch time at which the topology graph was updated.
- main.step("Obtain t1 by REST call")
- json_str_up_1 = main.ONOS1cli.topology_events_metrics()
- json_str_up_2 = main.ONOS2cli.topology_events_metrics()
- json_str_up_3 = main.ONOS3cli.topology_events_metrics()
-
- json_obj_1 = json.loads(json_str_up_1)
- json_obj_2 = json.loads(json_str_up_2)
- json_obj_3 = json.loads(json_str_up_3)
-
- os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
- tshark_port_up+" /tmp/")
-
- f_port_up = open(tshark_port_up, 'r')
- f_line = f_port_up.readline()
- obj_up = f_line.split(" ")
- if len(f_line) > 0:
- timestamp_begin_pt_up = int(float(obj_up[1])*1000)
- main.log.info("Port up begin timestamp: "+
- str(timestamp_begin_pt_up))
- else:
- main.log.info("Tshark output file returned unexpected"+
- " results.")
- timestamp_begin_pt_up = 0
-
- f_port_up.close()
-
- graph_timestamp_1 = \
- json_obj_1[graphTimestamp]['value']
- graph_timestamp_2 = \
- json_obj_2[graphTimestamp]['value']
- graph_timestamp_3 = \
- json_obj_3[graphTimestamp]['value']
-
- #Obtain device timestamp. This timestamp captures
- #the epoch time at which the device event happened
- device_timestamp_1 = \
- json_obj_1[deviceTimestamp]['value']
- device_timestamp_2 = \
- json_obj_2[deviceTimestamp]['value']
- device_timestamp_3 = \
- json_obj_3[deviceTimestamp]['value']
-
- #Get delta between graph event and OFP
- pt_up_graph_to_ofp_1 = int(graph_timestamp_1) -\
- int(timestamp_begin_pt_up)
- pt_up_graph_to_ofp_2 = int(graph_timestamp_2) -\
- int(timestamp_begin_pt_up)
- pt_up_graph_to_ofp_3 = int(graph_timestamp_3) -\
- int(timestamp_begin_pt_up)
-
- #Get delta between device event and OFP
- pt_up_device_to_ofp_1 = int(device_timestamp_1) -\
- int(timestamp_begin_pt_up)
- pt_up_device_to_ofp_2 = int(device_timestamp_2) -\
- int(timestamp_begin_pt_up)
- pt_up_device_to_ofp_3 = int(device_timestamp_3) -\
- int(timestamp_begin_pt_up)
-
- main.log.info("ONOS1 delta G2O: "+str(pt_up_graph_to_ofp_1))
- main.log.info("ONOS2 delta G2O: "+str(pt_up_graph_to_ofp_2))
- main.log.info("ONOS3 delta G2O: "+str(pt_up_graph_to_ofp_3))
-
- main.log.info("ONOS1 delta D2O: "+str(pt_up_device_to_ofp_1))
- main.log.info("ONOS2 delta D2O: "+str(pt_up_device_to_ofp_2))
- main.log.info("ONOS3 delta D2O: "+str(pt_up_device_to_ofp_3))
-
- pt_up_graph_to_ofp_avg = \
- (int(pt_up_graph_to_ofp_1) +
- int(pt_up_graph_to_ofp_2) +
- int(pt_up_graph_to_ofp_3)) / 3
-
- pt_up_device_to_ofp_avg = \
- (int(pt_up_device_to_ofp_1) +
- int(pt_up_device_to_ofp_2) +
- int(pt_up_device_to_ofp_3)) / 3
-
- if pt_up_graph_to_ofp_avg > up_threshold_min and \
- pt_up_graph_to_ofp_avg < up_threshold_max:
- port_up_graph_to_ofp_list.append(
- pt_up_graph_to_ofp_avg)
- main.log.info("Port down: graph to ofp avg: "+
- str(pt_up_graph_to_ofp_avg) + " ms")
- else:
- main.log.info("Average port up graph-to-ofp result"+
- " exceeded the threshold: "+
- str(pt_up_graph_to_ofp_avg))
-
- if pt_up_device_to_ofp_avg > up_threshold_min and \
- pt_up_device_to_ofp_avg < up_threshold_max:
- port_up_device_to_ofp_list.append(
- pt_up_device_to_ofp_avg)
- main.log.info("Port up: device to ofp avg: "+
- str(pt_up_device_to_ofp_avg) + " ms")
- else:
- main.log.info("Average port up device-to-ofp result"+
- " exceeded the threshold: "+
- str(pt_up_device_to_ofp_avg))
-
- #END ITERATION FOR LOOP
-
- #Check all list for latency existence and set assertion
- if (port_down_graph_to_ofp_list and port_down_device_to_ofp_list\
- and port_up_graph_to_ofp_list and port_up_device_to_ofp_list):
- assertion = main.TRUE
-
- #Calculate and report latency measurements
- port_down_graph_to_ofp_min = min(port_down_graph_to_ofp_list)
- port_down_graph_to_ofp_max = max(port_down_graph_to_ofp_list)
- port_down_graph_to_ofp_avg = \
- (sum(port_down_graph_to_ofp_list) /
- len(port_down_graph_to_ofp_list))
- port_down_graph_to_ofp_std_dev = \
- str(round(numpy.std(port_down_graph_to_ofp_list),1))
-
- main.log.report("Port down graph-to-ofp "+
- "Avg: "+str(port_down_graph_to_ofp_avg)+" ms "+
- "Std Deviation: "+port_down_graph_to_ofp_std_dev+" ms")
-
- port_down_device_to_ofp_min = min(port_down_device_to_ofp_list)
- port_down_device_to_ofp_max = max(port_down_device_to_ofp_list)
- port_down_device_to_ofp_avg = \
- (sum(port_down_device_to_ofp_list) /\
- len(port_down_device_to_ofp_list))
- port_down_device_to_ofp_std_dev = \
- str(round(numpy.std(port_down_device_to_ofp_list),1))
-
- main.log.report("Port down device-to-ofp "+
- "Avg: "+str(port_down_device_to_ofp_avg)+" ms "+
- "Std Deviation: "+port_down_device_to_ofp_std_dev+" ms")
-
- port_up_graph_to_ofp_min = min(port_up_graph_to_ofp_list)
- port_up_graph_to_ofp_max = max(port_up_graph_to_ofp_list)
- port_up_graph_to_ofp_avg = \
- (sum(port_up_graph_to_ofp_list) /\
- len(port_up_graph_to_ofp_list))
- port_up_graph_to_ofp_std_dev = \
- str(round(numpy.std(port_up_graph_to_ofp_list),1))
-
- main.log.report("Port up graph-to-ofp "+
- "Avg: "+str(port_up_graph_to_ofp_avg)+" ms "+
- "Std Deviation: "+port_up_graph_to_ofp_std_dev+" ms")
-
- port_up_device_to_ofp_min = min(port_up_device_to_ofp_list)
- port_up_device_to_ofp_max = max(port_up_device_to_ofp_list)
- port_up_device_to_ofp_avg = \
- (sum(port_up_device_to_ofp_list) /\
- len(port_up_device_to_ofp_list))
- port_up_device_to_ofp_std_dev = \
- str(round(numpy.std(port_up_device_to_ofp_list),1))
-
- main.log.report("Port up device-to-ofp "+
- "Avg: "+str(port_up_device_to_ofp_avg)+" ms "+
- "Std Deviation: "+port_up_device_to_ofp_std_dev+" ms")
-
- utilities.assert_equals(expect=main.TRUE, actual=assertion,
- onpass="Port discovery latency calculation successful",
- onfail="Port discovery test failed")
-
- def CASE4(self, main):
- '''
- Link down event using loss rate 100%
-
- Important:
- Use a simple 2 switch topology with 1 link between
- the two switches. Ensure that mac addresses of the
- switches are 1 / 2 respectively
- '''
- import time
- import subprocess
- import os
- import requests
- import json
- import numpy
-
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS_user = main.params['CTRL']['user']
-
- default_sw_port = main.params['CTRL']['port1']
-
- #Number of iterations of case
- num_iter = main.params['TEST']['numIter']
-
- #Timestamp 'keys' for json metrics output.
- #These are subject to change, hence moved into params
- deviceTimestamp = main.params['JSON']['deviceTimestamp']
- linkTimestamp = main.params['JSON']['linkTimestamp']
- graphTimestamp = main.params['JSON']['graphTimestamp']
-
- debug_mode = main.params['TEST']['debugMode']
-
- local_time = time.strftime('%x %X')
- local_time = local_time.replace("/","")
- local_time = local_time.replace(" ","_")
- local_time = local_time.replace(":","")
- if debug_mode == 'on':
- main.ONOS1.tshark_pcap("eth0",
- "/tmp/link_lat_pcap_"+local_time)
-
- #Threshold for this test case
- up_threshold_str = main.params['TEST']['linkUpThreshold']
- down_threshold_str = main.params['TEST']['linkDownThreshold']
-
- up_threshold_obj = up_threshold_str.split(",")
- down_threshold_obj = down_threshold_str.split(",")
-
- up_threshold_min = int(up_threshold_obj[0])
- up_threshold_max = int(up_threshold_obj[1])
-
- down_threshold_min = int(down_threshold_obj[0])
- down_threshold_max = int(down_threshold_obj[1])
-
- assertion = main.TRUE
- #Link event timestamp to system time list
- link_down_link_to_system_list = []
- link_up_link_to_system_list = []
- #Graph event timestamp to system time list
- link_down_graph_to_system_list = []
- link_up_graph_to_system_list = []
-
- main.log.report("Link up / down discovery latency between "+
- "two switches")
- main.log.report("Simulated by setting loss-rate 100%")
- main.log.report("'tc qdisc add dev <intfs> root netem loss 100%'")
- main.log.report("Total iterations of test: "+str(num_iter))
-
- main.step("Assign all switches")
- main.Mininet1.assign_sw_controller(sw="1",
- ip1=ONOS1_ip, port1=default_sw_port)
- main.Mininet1.assign_sw_controller(sw="2",
- ip1=ONOS1_ip, port1=default_sw_port)
-
- main.step("Verifying switch assignment")
- result_s1 = main.Mininet1.get_sw_controller(sw="s1")
- result_s2 = main.Mininet1.get_sw_controller(sw="s2")
-
- #Allow time for events to finish before taking measurements
- time.sleep(10)
-
- link_down1 = False
- link_down2 = False
- link_down3 = False
- #Start iteration of link event test
- for i in range(0, int(num_iter)):
- main.step("Getting initial system time as t0")
-
- #System time in epoch ms
- timestamp_link_down_t0 = time.time() * 1000
- #Link down is simulated by 100% loss rate using traffic
- #control command
- main.Mininet1.handle.sendline(
- "sh tc qdisc add dev s1-eth1 root netem loss 100%")
-
- #TODO: Iterate through 'links' command to verify that
- # link s1 -> s2 went down (loop timeout 30 seconds)
- # on all 3 ONOS instances
- main.log.info("Checking ONOS for link update")
- loop_count = 0
- while( not (link_down1 and link_down2 and link_down3)\
- and loop_count < 30 ):
- json_str1 = main.ONOS1cli.links()
- json_str2 = main.ONOS2cli.links()
- json_str3 = main.ONOS3cli.links()
-
- if not (json_str1 and json_str2 and json_str3):
- main.log.error("CLI command returned error ")
- break
- else:
- json_obj1 = json.loads(json_str1)
- json_obj2 = json.loads(json_str2)
- json_obj3 = json.loads(json_str3)
- for obj1 in json_obj1:
- if '01' not in obj1['src']['device']:
- link_down1 = True
- main.log.info("Link down from "+
- "s1 -> s2 on ONOS1 detected")
- for obj2 in json_obj2:
- if '01' not in obj2['src']['device']:
- link_down2 = True
- main.log.info("Link down from "+
- "s1 -> s2 on ONOS2 detected")
- for obj3 in json_obj3:
- if '01' not in obj3['src']['device']:
- link_down3 = True
- main.log.info("Link down from "+
- "s1 -> s2 on ONOS3 detected")
-
- loop_count += 1
- #If CLI doesn't like the continuous requests
- #and exits in this loop, increase the sleep here.
- #Consequently, while loop timeout will increase
- time.sleep(1)
-
- #Give time for metrics measurement to catch up
- #NOTE: May need to be configured more accurately
- time.sleep(10)
- #If we exited the while loop and link down 1,2,3 are still
- #false, then ONOS has failed to discover link down event
- if not (link_down1 and link_down2 and link_down3):
- main.log.info("Link down discovery failed")
-
- link_down_lat_graph1 = 0
- link_down_lat_graph2 = 0
- link_down_lat_graph3 = 0
- link_down_lat_device1 = 0
- link_down_lat_device2 = 0
- link_down_lat_device3 = 0
-
- assertion = main.FALSE
- else:
- json_topo_metrics_1 =\
- main.ONOS1cli.topology_events_metrics()
- json_topo_metrics_2 =\
- main.ONOS2cli.topology_events_metrics()
- json_topo_metrics_3 =\
- main.ONOS3cli.topology_events_metrics()
- json_topo_metrics_1 = json.loads(json_topo_metrics_1)
- json_topo_metrics_2 = json.loads(json_topo_metrics_2)
- json_topo_metrics_3 = json.loads(json_topo_metrics_3)
-
- main.log.info("Obtaining graph and device timestamp")
- graph_timestamp_1 = \
- json_topo_metrics_1[graphTimestamp]['value']
- graph_timestamp_2 = \
- json_topo_metrics_2[graphTimestamp]['value']
- graph_timestamp_3 = \
- json_topo_metrics_3[graphTimestamp]['value']
-
- link_timestamp_1 = \
- json_topo_metrics_1[linkTimestamp]['value']
- link_timestamp_2 = \
- json_topo_metrics_2[linkTimestamp]['value']
- link_timestamp_3 = \
- json_topo_metrics_3[linkTimestamp]['value']
-
- if graph_timestamp_1 and graph_timestamp_2 and\
- graph_timestamp_3 and link_timestamp_1 and\
- link_timestamp_2 and link_timestamp_3:
- link_down_lat_graph1 = int(graph_timestamp_1) -\
- int(timestamp_link_down_t0)
- link_down_lat_graph2 = int(graph_timestamp_2) -\
- int(timestamp_link_down_t0)
- link_down_lat_graph3 = int(graph_timestamp_3) -\
- int(timestamp_link_down_t0)
-
- link_down_lat_link1 = int(link_timestamp_1) -\
- int(timestamp_link_down_t0)
- link_down_lat_link2 = int(link_timestamp_2) -\
- int(timestamp_link_down_t0)
- link_down_lat_link3 = int(link_timestamp_3) -\
- int(timestamp_link_down_t0)
- else:
- main.log.error("There was an error calculating"+
- " the delta for link down event")
- link_down_lat_graph1 = 0
- link_down_lat_graph2 = 0
- link_down_lat_graph3 = 0
-
- link_down_lat_device1 = 0
- link_down_lat_device2 = 0
- link_down_lat_device3 = 0
-
- main.log.info("Link down latency ONOS1 iteration "+
- str(i)+" (end-to-end): "+
- str(link_down_lat_graph1)+" ms")
- main.log.info("Link down latency ONOS2 iteration "+
- str(i)+" (end-to-end): "+
- str(link_down_lat_graph2)+" ms")
- main.log.info("Link down latency ONOS3 iteration "+
- str(i)+" (end-to-end): "+
- str(link_down_lat_graph3)+" ms")
-
- main.log.info("Link down latency ONOS1 iteration "+
- str(i)+" (link-event-to-system-timestamp): "+
- str(link_down_lat_link1)+" ms")
- main.log.info("Link down latency ONOS2 iteration "+
- str(i)+" (link-event-to-system-timestamp): "+
- str(link_down_lat_link2)+" ms")
- main.log.info("Link down latency ONOS3 iteration "+
- str(i)+" (link-event-to-system-timestamp): "+
- str(link_down_lat_link3))
-
- #Calculate avg of node calculations
- link_down_lat_graph_avg =\
- (link_down_lat_graph1 +
- link_down_lat_graph2 +
- link_down_lat_graph3) / 3
- link_down_lat_link_avg =\
- (link_down_lat_link1 +
- link_down_lat_link2 +
- link_down_lat_link3) / 3
-
- #Set threshold and append latency to list
- if link_down_lat_graph_avg > down_threshold_min and\
- link_down_lat_graph_avg < down_threshold_max:
- link_down_graph_to_system_list.append(
- link_down_lat_graph_avg)
- else:
- main.log.info("Link down latency exceeded threshold")
- main.log.info("Results for iteration "+str(i)+
- "have been omitted")
- if link_down_lat_link_avg > down_threshold_min and\
- link_down_lat_link_avg < down_threshold_max:
- link_down_link_to_system_list.append(
- link_down_lat_link_avg)
- else:
- main.log.info("Link down latency exceeded threshold")
- main.log.info("Results for iteration "+str(i)+
- "have been omitted")
-
- #NOTE: To remove loss rate and measure latency:
- # 'sh tc qdisc del dev s1-eth1 root'
- timestamp_link_up_t0 = time.time() * 1000
- main.Mininet1.handle.sendline("sh tc qdisc del dev "+
- "s1-eth1 root")
- main.Mininet1.handle.expect("mininet>")
-
- main.log.info("Checking ONOS for link update")
-
- link_down1 = True
- link_down2 = True
- link_down3 = True
- loop_count = 0
- while( (link_down1 and link_down2 and link_down3)\
- and loop_count < 30 ):
- json_str1 = main.ONOS1cli.links()
- json_str2 = main.ONOS2cli.links()
- json_str3 = main.ONOS3cli.links()
- if not (json_str1 and json_str2 and json_str3):
- main.log.error("CLI command returned error ")
- break
- else:
- json_obj1 = json.loads(json_str1)
- json_obj2 = json.loads(json_str2)
- json_obj3 = json.loads(json_str3)
-
- for obj1 in json_obj1:
- if '01' in obj1['src']['device']:
- link_down1 = False
- main.log.info("Link up from "+
- "s1 -> s2 on ONOS1 detected")
- for obj2 in json_obj2:
- if '01' in obj2['src']['device']:
- link_down2 = False
- main.log.info("Link up from "+
- "s1 -> s2 on ONOS2 detected")
- for obj3 in json_obj3:
- if '01' in obj3['src']['device']:
- link_down3 = False
- main.log.info("Link up from "+
- "s1 -> s2 on ONOS3 detected")
-
- loop_count += 1
- time.sleep(1)
-
- if (link_down1 and link_down2 and link_down3):
- main.log.info("Link up discovery failed")
-
- link_up_lat_graph1 = 0
- link_up_lat_graph2 = 0
- link_up_lat_graph3 = 0
- link_up_lat_device1 = 0
- link_up_lat_device2 = 0
- link_up_lat_device3 = 0
-
- assertion = main.FALSE
- else:
- json_topo_metrics_1 =\
- main.ONOS1cli.topology_events_metrics()
- json_topo_metrics_2 =\
- main.ONOS2cli.topology_events_metrics()
- json_topo_metrics_3 =\
- main.ONOS3cli.topology_events_metrics()
- json_topo_metrics_1 = json.loads(json_topo_metrics_1)
- json_topo_metrics_2 = json.loads(json_topo_metrics_2)
- json_topo_metrics_3 = json.loads(json_topo_metrics_3)
-
- main.log.info("Obtaining graph and device timestamp")
- graph_timestamp_1 = \
- json_topo_metrics_1[graphTimestamp]['value']
- graph_timestamp_2 = \
- json_topo_metrics_2[graphTimestamp]['value']
- graph_timestamp_3 = \
- json_topo_metrics_3[graphTimestamp]['value']
-
- link_timestamp_1 = \
- json_topo_metrics_1[linkTimestamp]['value']
- link_timestamp_2 = \
- json_topo_metrics_2[linkTimestamp]['value']
- link_timestamp_3 = \
- json_topo_metrics_3[linkTimestamp]['value']
-
- if graph_timestamp_1 and graph_timestamp_2 and\
- graph_timestamp_3 and link_timestamp_1 and\
- link_timestamp_2 and link_timestamp_3:
- link_up_lat_graph1 = int(graph_timestamp_1) -\
- int(timestamp_link_up_t0)
- link_up_lat_graph2 = int(graph_timestamp_2) -\
- int(timestamp_link_up_t0)
- link_up_lat_graph3 = int(graph_timestamp_3) -\
- int(timestamp_link_up_t0)
-
- link_up_lat_link1 = int(link_timestamp_1) -\
- int(timestamp_link_up_t0)
- link_up_lat_link2 = int(link_timestamp_2) -\
- int(timestamp_link_up_t0)
- link_up_lat_link3 = int(link_timestamp_3) -\
- int(timestamp_link_up_t0)
- else:
- main.log.error("There was an error calculating"+
- " the delta for link down event")
- link_up_lat_graph1 = 0
- link_up_lat_graph2 = 0
- link_up_lat_graph3 = 0
-
- link_up_lat_device1 = 0
- link_up_lat_device2 = 0
- link_up_lat_device3 = 0
-
- if debug_mode == 'on':
- main.log.info("Link up latency ONOS1 iteration "+
- str(i)+" (end-to-end): "+
- str(link_up_lat_graph1)+" ms")
- main.log.info("Link up latency ONOS2 iteration "+
- str(i)+" (end-to-end): "+
- str(link_up_lat_graph2)+" ms")
- main.log.info("Link up latency ONOS3 iteration "+
- str(i)+" (end-to-end): "+
- str(link_up_lat_graph3)+" ms")
-
- main.log.info("Link up latency ONOS1 iteration "+
- str(i)+" (link-event-to-system-timestamp): "+
- str(link_up_lat_link1)+" ms")
- main.log.info("Link up latency ONOS2 iteration "+
- str(i)+" (link-event-to-system-timestamp): "+
- str(link_up_lat_link2)+" ms")
- main.log.info("Link up latency ONOS3 iteration "+
- str(i)+" (link-event-to-system-timestamp): "+
- str(link_up_lat_link3))
-
- #Calculate avg of node calculations
- link_up_lat_graph_avg =\
- (link_up_lat_graph1 +
- link_up_lat_graph2 +
- link_up_lat_graph3) / 3
- link_up_lat_link_avg =\
- (link_up_lat_link1 +
- link_up_lat_link2 +
- link_up_lat_link3) / 3
-
- #Set threshold and append latency to list
- if link_up_lat_graph_avg > up_threshold_min and\
- link_up_lat_graph_avg < up_threshold_max:
- link_up_graph_to_system_list.append(
- link_up_lat_graph_avg)
- else:
- main.log.info("Link up latency exceeded threshold")
- main.log.info("Results for iteration "+str(i)+
- "have been omitted")
- if link_up_lat_link_avg > up_threshold_min and\
- link_up_lat_link_avg < up_threshold_max:
- link_up_link_to_system_list.append(
- link_up_lat_link_avg)
- else:
- main.log.info("Link up latency exceeded threshold")
- main.log.info("Results for iteration "+str(i)+
- "have been omitted")
-
- #Calculate min, max, avg of list and report
- link_down_min = min(link_down_graph_to_system_list)
- link_down_max = max(link_down_graph_to_system_list)
- link_down_avg = sum(link_down_graph_to_system_list) / \
- len(link_down_graph_to_system_list)
- link_up_min = min(link_up_graph_to_system_list)
- link_up_max = max(link_up_graph_to_system_list)
- link_up_avg = sum(link_up_graph_to_system_list) / \
- len(link_up_graph_to_system_list)
- link_down_std_dev = \
- str(round(numpy.std(link_down_graph_to_system_list),1))
- link_up_std_dev = \
- str(round(numpy.std(link_up_graph_to_system_list),1))
-
- main.log.report("Link down latency " +
- "Avg: "+str(link_down_avg)+" ms "+
- "Std Deviation: "+link_down_std_dev+" ms")
- main.log.report("Link up latency "+
- "Avg: "+str(link_up_avg)+" ms "+
- "Std Deviation: "+link_up_std_dev+" ms")
-
- utilities.assert_equals(expect=main.TRUE, actual=assertion,
- onpass="Link discovery latency calculation successful",
- onfail="Link discovery latency case failed")
-
- def CASE5(self, main):
- '''
- 100 Switch discovery latency
-
- Important:
- This test case can be potentially dangerous if
- your machine has previously set iptables rules.
- One of the steps of the test case will flush
- all existing iptables rules.
- Note:
- You can specify the number of switches in the
- params file to adjust the switch discovery size
- (and specify the corresponding topology in Mininet1
- .topo file)
- '''
- import time
- import subprocess
- import os
- import requests
- import json
-
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS3_ip = main.params['CTRL']['ip3']
- MN1_ip = main.params['MN']['ip1']
- ONOS_user = main.params['CTRL']['user']
-
- default_sw_port = main.params['CTRL']['port1']
-
- #Number of iterations of case
- num_iter = main.params['TEST']['numIter']
- num_sw = main.params['TEST']['numSwitch']
-
- #Timestamp 'keys' for json metrics output.
- #These are subject to change, hence moved into params
- deviceTimestamp = main.params['JSON']['deviceTimestamp']
- graphTimestamp = main.params['JSON']['graphTimestamp']
-
- debug_mode = main.params['TEST']['debugMode']
-
- local_time = time.strftime('%X')
- local_time = local_time.replace("/","")
- local_time = local_time.replace(" ","_")
- local_time = local_time.replace(":","")
- if debug_mode == 'on':
- main.ONOS1.tshark_pcap("eth0",
- "/tmp/100_sw_lat_pcap_"+local_time)
-
- #Threshold for this test case
- sw_disc_threshold_str = main.params['TEST']['swDisc100Threshold']
- sw_disc_threshold_obj = sw_disc_threshold_str.split(",")
- sw_disc_threshold_min = int(sw_disc_threshold_obj[0])
- sw_disc_threshold_max = int(sw_disc_threshold_obj[1])
-
- tshark_ofp_output = "/tmp/tshark_ofp_"+num_sw+"sw.txt"
- tshark_tcp_output = "/tmp/tshark_tcp_"+num_sw+"sw.txt"
-
- tshark_ofp_result_list = []
- tshark_tcp_result_list = []
-
- sw_discovery_lat_list = []
-
- main.case(num_sw+" Switch discovery latency")
- main.step("Assigning all switches to ONOS1")
- for i in range(1, int(num_sw)+1):
- main.Mininet1.assign_sw_controller(
- sw=str(i),
- ip1=ONOS1_ip,
- port1=default_sw_port)
-
- #Ensure that nodes are configured with ptpd
- #Just a warning message
- main.log.info("Please check ptpd configuration to ensure"+\
- " All nodes' system times are in sync")
- time.sleep(5)
-
- for i in range(0, int(num_iter)):
-
- main.step("Set iptables rule to block incoming sw connections")
- #Set iptables rule to block incoming switch connections
- #The rule description is as follows:
- # Append to INPUT rule,
- # behavior DROP that matches following:
- # * packet type: tcp
- # * source IP: MN1_ip
- # * destination PORT: 6633
- main.ONOS1.handle.sendline(
- "sudo iptables -A INPUT -p tcp -s "+MN1_ip+
- " --dport "+default_sw_port+" -j DROP")
- main.ONOS1.handle.expect("\$")
- # Append to OUTPUT rule,
- # behavior DROP that matches following:
- # * packet type: tcp
- # * source IP: MN1_ip
- # * destination PORT: 6633
- main.ONOS1.handle.sendline(
- "sudo iptables -A OUTPUT -p tcp -s "+MN1_ip+
- " --dport "+default_sw_port+" -j DROP")
- main.ONOS1.handle.expect("\$")
- #Give time to allow rule to take effect
- #NOTE: Sleep period may need to be configured
- # based on the number of switches in the topology
- main.log.info("Please wait for switch connection to "+
- "time out")
- time.sleep(60)
-
- #Gather vendor OFP with tshark
- main.ONOS1.tshark_grep("OFP 86 Vendor",
- tshark_ofp_output)
- main.ONOS1.tshark_grep("TCP 74 ",
- tshark_tcp_output)
-
- #NOTE: Remove all iptables rule quickly (flush)
- # Before removal, obtain TestON timestamp at which
- # removal took place
- # (ensuring nodes are configured via ptp)
- # sudo iptables -F
-
- t0_system = time.time() * 1000
- main.ONOS1.handle.sendline(
- "sudo iptables -F")
-
- #Counter to track loop count
- counter_loop = 0
- counter_avail1 = 0
- counter_avail2 = 0
- counter_avail3 = 0
- onos1_dev = False
- onos2_dev = False
- onos3_dev = False
- while counter_loop < 60:
- #Continue to check devices for all device
- #availability. When all devices in all 3
- #ONOS instances indicate that devices are available
- #obtain graph event timestamp for t1.
- device_str_obj1 = main.ONOS1cli.devices()
- device_str_obj2 = main.ONOS2cli.devices()
- device_str_obj3 = main.ONOS3cli.devices()
-
- device_json1 = json.loads(device_str_obj1)
- device_json2 = json.loads(device_str_obj2)
- device_json3 = json.loads(device_str_obj3)
-
- for device1 in device_json1:
- if device1['available'] == True:
- counter_avail1 += 1
- if counter_avail1 == int(num_sw):
- onos1_dev = True
- main.log.info("All devices have been "+
- "discovered on ONOS1")
- else:
- counter_avail1 = 0
- for device2 in device_json2:
- if device2['available'] == True:
- counter_avail2 += 1
- if counter_avail2 == int(num_sw):
- onos2_dev = True
- main.log.info("All devices have been "+
- "discovered on ONOS2")
- else:
- counter_avail2 = 0
- for device3 in device_json3:
- if device3['available'] == True:
- counter_avail3 += 1
- if counter_avail3 == int(num_sw):
- onos3_dev = True
- main.log.info("All devices have been "+
- "discovered on ONOS3")
- else:
- counter_avail3 = 0
-
- if onos1_dev and onos2_dev and onos3_dev:
- main.log.info("All devices have been discovered "+
- "on all ONOS instances")
- json_str_topology_metrics_1 =\
- main.ONOS1cli.topology_events_metrics()
- json_str_topology_metrics_2 =\
- main.ONOS2cli.topology_events_metrics()
- json_str_topology_metrics_3 =\
- main.ONOS3cli.topology_events_metrics()
-
- #Exit while loop if all devices discovered
- break
-
- counter_loop += 1
- #Give some time in between CLI calls
- #(will not affect measurement)
- time.sleep(3)
-
- main.ONOS1.tshark_stop()
-
- os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
- tshark_ofp_output+" /tmp/")
- os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
- tshark_tcp_output+" /tmp/")
-
- #TODO: Automate OFP output analysis
- #Debug mode - print out packets captured at runtime
- if debug_mode == 'on':
- ofp_file = open(tshark_ofp_output, 'r')
- main.log.info("Tshark OFP Vendor output: ")
- for line in ofp_file:
- tshark_ofp_result_list.append(line)
- main.log.info(line)
- ofp_file.close()
-
- tcp_file = open(tshark_tcp_output, 'r')
- main.log.info("Tshark TCP 74 output: ")
- for line in tcp_file:
- tshark_tcp_result_list.append(line)
- main.log.info(line)
- tcp_file.close()
-
- json_obj_1 = json.loads(json_str_topology_metrics_1)
- json_obj_2 = json.loads(json_str_topology_metrics_2)
- json_obj_3 = json.loads(json_str_topology_metrics_3)
-
- graph_timestamp_1 = \
- json_obj_1[graphTimestamp]['value']
- graph_timestamp_2 = \
- json_obj_2[graphTimestamp]['value']
- graph_timestamp_3 = \
- json_obj_3[graphTimestamp]['value']
-
- graph_lat_1 = int(graph_timestamp_1) - int(t0_system)
- graph_lat_2 = int(graph_timestamp_2) - int(t0_system)
- graph_lat_3 = int(graph_timestamp_3) - int(t0_system)
-
- avg_graph_lat = \
- (int(graph_lat_1) +\
- int(graph_lat_2) +\
- int(graph_lat_3)) / 3
-
- if avg_graph_lat > sw_disc_threshold_min \
- and avg_graph_lat < sw_disc_threshold_max:
- sw_discovery_lat_list.append(
- avg_graph_lat)
- else:
- main.log.info("100 Switch discovery latency "+
- "exceeded the threshold.")
-
- #END ITERATION FOR LOOP
-
- sw_lat_min = min(sw_discovery_lat_list)
- sw_lat_max = max(sw_discovery_lat_list)
- sw_lat_avg = sum(sw_discovery_lat_list) /\
- len(sw_discovery_lat_list)
-
- main.log.report("100 Switch discovery lat "+\
- "Min: "+str(sw_lat_min)+" ms"+\
- "Max: "+str(sw_lat_max)+" ms"+\
- "Avg: "+str(sw_lat_avg)+" ms")
-
- def CASE6(self, main):
- '''
- Increase number of nodes and initiate CLI
- '''
- import time
-
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS7_ip = main.params['CTRL']['ip7']
-
- cell_name = main.params['ENV']['cellName']
-
- global cluster_count
-
- #Cluster size increased everytime the case is defined
- cluster_count += 2
-
- main.log.report("Increasing cluster size to "+
- str(cluster_count))
-
- install_result = main.FALSE
- if cluster_count == 5:
- main.log.info("Installing nodes 4 and 5")
- node4_result = \
- main.ONOSbench.onos_install(node=ONOS4_ip)
- node5_result = \
- main.ONOSbench.onos_install(node=ONOS5_ip)
- install_result = node4_result and node5_result
-
- time.sleep(5)
-
- main.ONOS4cli.start_onos_cli(ONOS4_ip)
- main.ONOS5cli.start_onos_cli(ONOS5_ip)
-
- elif cluster_count == 7:
- main.log.info("Installing nodes 4 and 5")
- node6_result = \
- main.ONOSbench.onos_install(node=ONOS6_ip)
- node7_result = \
- main.ONOSbench.onos_install(node=ONOS7_ip)
- install_result = node6_result and node7_result
-
- time.sleep(5)
-
- main.ONOS6cli.start_onos_cli(ONOS6_ip)
- main.ONOS7cli.start_onos_cli(ONOS7_ip)
-
-
-
-
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.py b/TestON/tests/TopoPerfNext/TopoPerfNext.py
deleted file mode 100644
index 0f37909..0000000
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.py
+++ /dev/null
@@ -1,2006 +0,0 @@
-# TopoPerfNext
-#
-# Topology Performance test for ONOS-next
-#
-# andrew@onlab.us
-#
-# If your machine does not come with numpy
-# run the following command:
-# sudo apt-get install python-numpy python-scipy
-
-import time
-import sys
-import os
-import re
-
-
-class TopoPerfNext:
-
- def __init__( self ):
- self.default = ''
-
- def CASE1( self, main ):
- """
- ONOS startup sequence
- """
- import time
-
- # Global cluster count for scale-out purposes
- global clusterCount
- #TODO: fix run number implementation
- global runNum
- global timeToPost
-
- #Test run time
- timeToPost = time.strftime("%Y-%m-%d %H:%M:%S")
- # Set initial cluster count
- clusterCount = 1
- ##
-
- runNum = time.strftime("%d%H%M%S")
-
- cellName = main.params[ 'ENV' ][ 'cellName' ]
-
- gitPull = main.params[ 'GIT' ][ 'autoPull' ]
- checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
- ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
- ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
- ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
-
- MN1Ip = main.params[ 'MN' ][ 'ip1' ]
- BENCHIp = main.params[ 'BENCH' ][ 'ip' ]
-
- topoCfgFile = main.params[ 'TEST' ][ 'topoConfigFile' ]
- topoCfgName = main.params[ 'TEST' ][ 'topoConfigName' ]
-
- portEventResultPath = main.params[ 'DB' ][ 'portEventResultPath' ]
- switchEventResultPath = main.params[ 'DB' ][ 'switchEventResultPath' ]
-
- mvnCleanInstall = main.params[ 'TEST' ][ 'mci' ]
-
- main.case( "Setting up test environment" )
- main.log.info( "Copying topology event accumulator config" +
- " to ONOS /package/etc" )
- main.ONOSbench.handle.sendline( "cp ~/" +
- topoCfgFile +
- " ~/ONOS/tools/package/etc/" +
- topoCfgName )
- main.ONOSbench.handle.expect( "\$" )
-
- main.log.report( "Setting up test environment" )
-
- main.step( "Starting mininet topology " )
- main.Mininet1.startNet()
-
- main.step( "Cleaning previously installed ONOS if any" )
- main.ONOSbench.onosUninstall( nodeIp=ONOS2Ip )
- main.ONOSbench.onosUninstall( nodeIp=ONOS3Ip )
- main.ONOSbench.onosUninstall( nodeIp=ONOS4Ip )
- main.ONOSbench.onosUninstall( nodeIp=ONOS5Ip )
- main.ONOSbench.onosUninstall( nodeIp=ONOS6Ip )
- main.ONOSbench.onosUninstall( nodeIp=ONOS7Ip )
-
- main.step( "Clearing previous DB log files" )
- fPortLog = open(portEventResultPath, 'w')
- fPortLog.write('')
- fPortLog.close()
- fSwitchLog = open(switchEventResultPath, 'w')
- fSwitchLog.write('')
- fSwitchLog.close()
-
- main.step( "Creating cell file" )
- cellFileResult = main.ONOSbench.createCellFile(
- BENCHIp, cellName, MN1Ip,
- ("onos-core,onos-api,webconsole,onos-app-metrics,onos-app-gui,"
- "onos-cli,onos-openflow"),
- ONOS1Ip )
-
- main.step( "Applying cell file to environment" )
- cellApplyResult = main.ONOSbench.setCell( cellName )
- verifyCellResult = main.ONOSbench.verifyCell()
-
- # NOTE: This step may be removed after proper
- # copy cat log functionality
- main.step( "Removing raft/copy-cat logs from ONOS nodes" )
- main.ONOSbench.onosRemoveRaftLogs()
- time.sleep( 30 )
-
- main.step( "Git checkout and pull " + checkoutBranch )
- if gitPull == 'on':
- # checkoutResult = \
- # main.ONOSbench.gitCheckout( checkoutBranch )
- checkoutResult = main.TRUE
- pullResult = main.ONOSbench.gitPull()
- else:
- checkoutResult = main.TRUE
- pullResult = main.TRUE
- main.log.info( "Skipped git checkout and pull" )
-
- main.log.report( "Commit information - " )
- main.ONOSbench.getVersion( report=True )
-
- main.step( "Using mvn clean & install" )
- if mvnCleanInstall == 'on':
- mvnResult = main.ONOSbench.cleanInstall()
- elif mvnCleanInstall == 'off':
- main.log.info("mci turned off by settings")
- mvnResult = main.TRUE
-
- main.step( "Set cell for ONOS cli env" )
- main.ONOS1cli.setCell( cellName )
-
- main.step( "Creating ONOS package" )
- packageResult = main.ONOSbench.onosPackage()
-
- main.step( "Installing ONOS package" )
- install1Result = main.ONOSbench.onosInstall( node=ONOS1Ip )
-
- time.sleep( 10 )
-
- main.step( "Start onos cli" )
- cli1 = main.ONOS1cli.startOnosCli( ONOS1Ip )
-
- utilities.assert_equals( expect=main.TRUE,
- actual=cellFileResult and cellApplyResult and
- verifyCellResult and checkoutResult and
- pullResult and mvnResult and
- install1Result, # and install2Result and
- # install3Result,
- onpass="Test Environment setup successful",
- onfail="Failed to setup test environment" )
-
- def CASE2( self, main ):
- """
- Assign s1 to ONOS1 and measure latency
-
- There are 4 levels of latency measurements to this test:
- 1 ) End-to-end measurement: Complete end-to-end measurement
- from TCP ( SYN/ACK ) handshake to Graph change
- 2 ) OFP-to-graph measurement: 'ONOS processing' snippet of
- measurement from OFP Vendor message to Graph change
- 3 ) OFP-to-device measurement: 'ONOS processing without
- graph change' snippet of measurement from OFP vendor
- message to Device change timestamp
- 4 ) T0-to-device measurement: Measurement that includes
- the switch handshake to devices timestamp without
- the graph view change. ( TCP handshake -> Device
- change )
- """
- import time
- import subprocess
- import json
- import requests
- import os
- import numpy
- global clusterCount
- global timeToPost
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
- ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
- ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
- ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
-
- ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
- defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
- # Number of iterations of case
- numIter = main.params[ 'TEST' ][ 'numIter' ]
- # Number of first 'x' iterations to ignore:
- iterIgnore = int( main.params[ 'TEST' ][ 'iterIgnore' ] )
-
- # Timestamp 'keys' for json metrics output.
- # These are subject to change, hence moved into params
- deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
- graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
- debugMode = main.params[ 'TEST' ][ 'debugMode' ]
- onosLog = main.params[ 'TEST' ][ 'onosLogFile' ]
- resultPath = main.params[ 'DB' ][ 'switchEventResultPath' ]
-
- # Threshold for the test
- thresholdStr = main.params[ 'TEST' ][ 'singleSwThreshold' ]
- thresholdObj = thresholdStr.split( "," )
- thresholdMin = int( thresholdObj[ 0 ] )
- thresholdMax = int( thresholdObj[ 1 ] )
-
- # List of switch add latency collected from
- # all iterations
- latencyEndToEndList = []
- latencyOfpToGraphList = []
- latencyOfpToDeviceList = []
- latencyT0ToDeviceList = []
- latencyTcpToOfpList = []
-
- # Initialize 2d array for [node][iteration] storage
- endToEndLatNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
- ofpToGraphLatNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
- # tcp-to-ofp measurements are same throughout each iteration
- tcpToOfpLatIter = []
-
- # Directory/file to store tshark results
- tsharkOfOutput = "/tmp/tshark_of_topo.txt"
- tsharkTcpOutput = "/tmp/tshark_tcp_topo.txt"
-
- # String to grep in tshark output
- tsharkTcpString = "TCP 74 " + defaultSwPort
- tsharkOfString = "OFP 86 Vendor"
-
- # Initialize assertion to TRUE
- assertion = main.TRUE
-
- localTime = time.strftime( '%x %X' )
- localTime = localTime.replace( "/", "" )
- localTime = localTime.replace( " ", "_" )
- localTime = localTime.replace( ":", "" )
- if debugMode == 'on':
- main.ONOS1.tsharkPcap( "eth0",
- "/tmp/single_sw_lat_pcap_" + localTime )
-
- main.log.info( "Debug mode is on" )
-
- main.log.report( "Latency of adding one switch to controller" )
- main.log.report( "First " + str( iterIgnore ) + " iterations ignored" +
- " for jvm warmup time" )
- main.log.report( "Total iterations of test: " + str( numIter ) )
-
- for i in range( 0, int( numIter ) ):
- main.log.info( "Starting tshark capture" )
-
- #* TCP [ ACK, SYN ] is used as t0A, the
- # very first "exchange" between ONOS and
- # the switch for end-to-end measurement
- #* OFP [ Stats Reply ] is used for t0B
- # the very last OFP message between ONOS
- # and the switch for ONOS measurement
- main.ONOS1.tsharkGrep( tsharkTcpString,
- tsharkTcpOutput )
- main.ONOS1.tsharkGrep( tsharkOfString,
- tsharkOfOutput )
-
- # Wait and ensure tshark is started and
- # capturing
- time.sleep( 10 )
-
- main.log.info( "Assigning s1 to controller" )
-
- main.Mininet1.assignSwController(
- sw="1",
- ip1=ONOS1Ip,
- port1=defaultSwPort )
-
- # Wait and ensure switch is assigned
- # before stopping tshark
- time.sleep( 30 )
-
- main.log.info( "Stopping all Tshark processes" )
- main.ONOS1.stopTshark()
-
- # tshark output is saved in ONOS. Use subprocess
- # to copy over files to TestON for parsing
- main.log.info( "Copying over tshark files" )
-
- # TCP CAPTURE ****
- # Copy the tshark output from ONOS machine to
- # TestON machine in tsharkTcpOutput directory>file
- os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
- tsharkTcpOutput + " /tmp/" )
- tcpFile = open( tsharkTcpOutput, 'r' )
- tempText = tcpFile.readline()
- tempText = tempText.split( " " )
-
- main.log.info( "Object read in from TCP capture: " +
- str( tempText ) )
- if len( tempText ) > 1:
- t0Tcp = float( tempText[ 1 ] ) * 1000.0
- else:
- main.log.error( "Tshark output file for TCP" +
- " returned unexpected results" )
- t0Tcp = 0
- assertion = main.FALSE
-
- tcpFile.close()
- #****************
-
- # OF CAPTURE ****
- os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
- tsharkOfOutput + " /tmp/" )
- ofFile = open( tsharkOfOutput, 'r' )
-
- lineOfp = ""
- # Read until last line of file
- while True:
- tempText = ofFile.readline()
- if tempText != '':
- lineOfp = tempText
- else:
- break
- obj = lineOfp.split( " " )
-
- main.log.info( "Object read in from OFP capture: " +
- str( lineOfp ) )
-
- if len( obj ) > 1:
- t0Ofp = float( obj[ 1 ] ) * 1000.0
- else:
- main.log.error( "Tshark output file for OFP" +
- " returned unexpected results" )
- t0Ofp = 0
- assertion = main.FALSE
-
- ofFile.close()
- #****************
-
- jsonStr1 = main.ONOS1cli.topologyEventsMetrics()
- # Initialize scale-out variables
- jsonStr2 = ""
- jsonStr3 = ""
- jsonStr4 = ""
- jsonStr5 = ""
- jsonStr6 = ""
- jsonStr7 = ""
-
- jsonObj1 = json.loads( jsonStr1 )
- # Initialize scale-out variables
- jsonObj2 = ""
- jsonObj3 = ""
- jsonObj4 = ""
- jsonObj5 = ""
- jsonObj6 = ""
- jsonObj7 = ""
-
- # Obtain graph timestamp. This timestsamp captures
- # the epoch time at which the topology graph was updated.
- graphTimestamp1 = \
- jsonObj1[ graphTimestamp ][ 'value' ]
- # Obtain device timestamp. This timestamp captures
- # the epoch time at which the device event happened
- deviceTimestamp1 = \
- jsonObj1[ deviceTimestamp ][ 'value' ]
-
- # t0 to device processing latency
- deltaDevice1 = int( deviceTimestamp1 ) - int( t0Tcp )
- # t0 to graph processing latency ( end-to-end )
- deltaGraph1 = int( graphTimestamp1 ) - int( t0Tcp )
- # ofp to graph processing latency ( ONOS processing )
- deltaOfpGraph1 = int( graphTimestamp1 ) - int( t0Ofp )
- # ofp to device processing latency ( ONOS processing )
- deltaOfpDevice1 = float( deviceTimestamp1 ) - float( t0Ofp )
- # tcp to ofp processing latency ( switch connection )
- deltaTcpOfp1 = int(t0Ofp) - int(t0Tcp)
-
- if deltaTcpOfp1 > thresholdMin and deltaTcpOfp1 < thresholdMax\
- and i >= iterIgnore:
- tcpToOfpLatIter.append(deltaTcpOfp1)
- main.log.info("iter"+str(i)+" tcp-to-ofp: "+
- str(deltaTcpOfp1)+" ms")
- else:
- tcpToOfpLatIter.append(0)
- main.log.info("iter"+str(i)+" tcp-to-ofp: "+
- str(deltaTcpOfp1)+" ms - ignored this iteration")
-
- # Store initial measurements in data array
- #This measurement is for node 1
-
- if deltaGraph1 > thresholdMin and deltaGraph1 < thresholdMax\
- and i >= iterIgnore:
- endToEndLatNodeIter[0][i] = deltaGraph1
- main.log.info("ONOS1 iter"+str(i)+" end-to-end: "+
- str(deltaGraph1)+" ms")
- else:
- main.log.info("ONOS1 iter"+str(i)+" end-to-end: "+
- str(deltaGraph1)+" ms - ignored this iteration")
-
-
- if deltaOfpGraph1 > thresholdMin and deltaOfpGraph1 < thresholdMax\
- and i >= iterIgnore:
- ofpToGraphLatNodeIter[0][i] = deltaOfpGraph1
-
- main.log.info("ONOS1 iter"+str(i)+" ofp-to-graph: "+
- str(deltaOfpGraph1)+" ms")
-
- # TODO: Create even cluster number events
-
- # Include scale-out measurements when applicable
- if clusterCount >= 3:
- jsonStr2 = main.ONOS2cli.topologyEventsMetrics()
- jsonStr3 = main.ONOS3cli.topologyEventsMetrics()
- jsonObj2 = json.loads( jsonStr2 )
- jsonObj3 = json.loads( jsonStr3 )
- graphTimestamp2 = \
- jsonObj2[ graphTimestamp ][ 'value' ]
- graphTimestamp3 = \
- jsonObj3[ graphTimestamp ][ 'value' ]
- deviceTimestamp2 = \
- jsonObj2[ deviceTimestamp ][ 'value' ]
- deviceTimestamp3 = \
- jsonObj3[ deviceTimestamp ][ 'value' ]
- deltaDevice2 = int( deviceTimestamp2 ) - int( t0Tcp )
- deltaDevice3 = int( deviceTimestamp3 ) - int( t0Tcp )
- deltaGraph2 = int( graphTimestamp2 ) - int( t0Tcp )
- deltaGraph3 = int( graphTimestamp3 ) - int( t0Tcp )
- deltaOfpGraph2 = int( graphTimestamp2 ) - int( t0Ofp )
- deltaOfpGraph3 = int( graphTimestamp3 ) - int( t0Ofp )
- deltaOfpDevice2 = float( deviceTimestamp2 ) -\
- float( t0Ofp )
- deltaOfpDevice3 = float( deviceTimestamp3 ) -\
- float( t0Ofp )
-
- if deltaGraph2 > thresholdMin and\
- deltaGraph2 < thresholdMax and i >= iterIgnore:
- endToEndLatNodeIter[1][i] = deltaGraph2
- main.log.info("ONOS2 iter"+str(i)+" end-to-end: "+
- str(deltaGraph2)+" ms")
-
- if deltaOfpGraph2 > thresholdMin and\
- deltaOfpGraph2 < thresholdMax and i >= iterIgnore:
- ofpToGraphLatNodeIter[1][i] = deltaOfpGraph2
- main.log.info("ONOS2 iter"+str(i)+" ofp-to-graph: "+
- str(deltaOfpGraph2)+" ms")
-
- if deltaGraph3 > thresholdMin and\
- deltaGraph3 < thresholdMax and i >= iterIgnore:
- endToEndLatNodeIter[2][i] = deltaGraph3
- main.log.info("ONOS3 iter"+str(i)+" end-to-end: "+
- str(deltaGraph3)+" ms")
-
- if deltaOfpGraph3 > thresholdMin and\
- deltaOfpGraph3 < thresholdMax and i >= iterIgnore:
- ofpToGraphLatNodeIter[2][i] = deltaOfpGraph3
- main.log.info("ONOS3 iter"+str(i)+" ofp-to-graph: "+
- str(deltaOfpGraph3)+" ms")
-
- if clusterCount >= 5:
- jsonStr4 = main.ONOS4cli.topologyEventsMetrics()
- jsonStr5 = main.ONOS5cli.topologyEventsMetrics()
- jsonObj4 = json.loads( jsonStr4 )
- jsonObj5 = json.loads( jsonStr5 )
- graphTimestamp4 = \
- jsonObj4[ graphTimestamp ][ 'value' ]
- graphTimestamp5 = \
- jsonObj5[ graphTimestamp ][ 'value' ]
- deviceTimestamp4 = \
- jsonObj4[ deviceTimestamp ][ 'value' ]
- deviceTimestamp5 = \
- jsonObj5[ deviceTimestamp ][ 'value' ]
- deltaDevice4 = int( deviceTimestamp4 ) - int( t0Tcp )
- deltaDevice5 = int( deviceTimestamp5 ) - int( t0Tcp )
- deltaGraph4 = int( graphTimestamp4 ) - int( t0Tcp )
- deltaGraph5 = int( graphTimestamp5 ) - int( t0Tcp )
- deltaOfpGraph4 = int( graphTimestamp4 ) - int( t0Ofp )
- deltaOfpGraph5 = int( graphTimestamp5 ) - int( t0Ofp )
- deltaOfpDevice4 = float( deviceTimestamp4 ) -\
- float( t0Ofp )
- deltaOfpDevice5 = float( deviceTimestamp5 ) -\
- float( t0Ofp )
-
- if deltaGraph4 > thresholdMin and\
- deltaGraph4 < thresholdMax and i >= iterIgnore:
- endToEndLatNodeIter[3][i] = deltaGraph4
- main.log.info("ONOS4 iter"+str(i)+" end-to-end: "+
- str(deltaGraph4)+" ms")
-
- #TODO:
- if deltaOfpGraph4 > thresholdMin and\
- deltaOfpGraph4 < thresholdMax and i >= iterIgnore:
- ofpToGraphLatNodeIter[3][i] = deltaOfpGraph4
- main.log.info("ONOS4 iter"+str(i)+" ofp-to-graph: "+
- str(deltaOfpGraph4)+" ms")
-
- if deltaGraph5 > thresholdMin and\
- deltaGraph5 < thresholdMax and i >= iterIgnore:
- endToEndLatNodeIter[4][i] = deltaGraph5
- main.log.info("ONOS5 iter"+str(i)+" end-to-end: "+
- str(deltaGraph5)+" ms")
-
- if deltaOfpGraph5 > thresholdMin and\
- deltaOfpGraph5 < thresholdMax and i >= iterIgnore:
- ofpToGraphLatNodeIter[4][i] = deltaOfpGraph5
- main.log.info("ONOS5 iter"+str(i)+" ofp-to-graph: "+
- str(deltaOfpGraph5)+" ms")
-
- if clusterCount >= 7:
- jsonStr6 = main.ONOS6cli.topologyEventsMetrics()
- jsonStr7 = main.ONOS7cli.topologyEventsMetrics()
- jsonObj6 = json.loads( jsonStr6 )
- jsonObj7 = json.loads( jsonStr7 )
- graphTimestamp6 = \
- jsonObj6[ graphTimestamp ][ 'value' ]
- graphTimestamp7 = \
- jsonObj7[ graphTimestamp ][ 'value' ]
- deviceTimestamp6 = \
- jsonObj6[ deviceTimestamp ][ 'value' ]
- deviceTimestamp7 = \
- jsonObj7[ deviceTimestamp ][ 'value' ]
- deltaDevice6 = int( deviceTimestamp6 ) - int( t0Tcp )
- deltaDevice7 = int( deviceTimestamp7 ) - int( t0Tcp )
- deltaGraph6 = int( graphTimestamp6 ) - int( t0Tcp )
- deltaGraph7 = int( graphTimestamp7 ) - int( t0Tcp )
- deltaOfpGraph6 = int( graphTimestamp6 ) - int( t0Ofp )
- deltaOfpGraph7 = int( graphTimestamp7 ) - int( t0Ofp )
- deltaOfpDevice6 = float( deviceTimestamp6 ) -\
- float( t0Ofp )
- deltaOfpDevice7 = float( deviceTimestamp7 ) -\
- float( t0Ofp )
-
- if deltaGraph6 > thresholdMin and\
- deltaGraph6 < thresholdMax and i >= iterIgnore:
- endToEndLatNodeIter[5][i] = deltaGraph6
- main.log.info("ONOS6 iter"+str(i)+" end-to-end: "+
- str(deltaGraph6)+" ms")
-
- #TODO:
- if deltaOfpGraph6 > thresholdMin and\
- deltaOfpGraph6 < thresholdMax and i >= iterIgnore:
- ofpToGraphLatNodeIter[5][i] = deltaOfpGraph6
- main.log.info("ONOS6 iter"+str(i)+" ofp-to-graph: "+
- str(deltaOfpGraph6)+" ms")
-
- if deltaGraph7 > thresholdMin and\
- deltaGraph7 < thresholdMax and i >= iterIgnore:
- endToEndLatNodeIter[6][i] = deltaGraph7
- main.log.info("ONOS7 iter"+str(i)+" end-to-end: "+
- str(deltaGraph7)+" ms")
-
- if deltaOfpGraph7 > thresholdMin and\
- deltaOfpGraph7 < thresholdMax and i >= iterIgnore:
- ofpToGraphLatNodeIter[6][i] = deltaOfpGraph7
- main.log.info("ONOS7 iter"+str(i)+" ofp-to-graph: "+
- str(deltaOfpGraph7)+" ms")
-
- main.log.info("Switch up discovery latency")
-
- main.log.info("Starting tshark capture")
-
- main.step( "Remove switch from controller" )
- main.Mininet1.deleteSwController( "s1" )
-
- #TODO: del controller does not have an OFP message.
- # However, we can capture TCP Fin,Ack as T0
-
- time.sleep( 5 )
-
- # END of for loop iteration
-
- #str( round( numpy.std( latencyT0ToDeviceList ), 1 ) )
-
- endToEndAvg = 0
- ofpToGraphAvg = 0
- endToEndList = []
- ofpToGraphList = []
- dbCmdList = []
-
- for node in range( 0, clusterCount ):
- # The latency 2d array was initialized to 0.
- # If an iteration was ignored, then we have some 0's in
- # our calculation. To avoid having this interfere with our
- # results, we must delete any index where 0 is found...
- # WARN: Potentially, we could have latency that hovers at
- # 0 ms once we have optimized code. FIXME for when this is
- # the case. Being able to obtain sub-millisecond accuracy
- # can prevent this from happening
- for item in endToEndLatNodeIter[node]:
- if item > 0.0:
- endToEndList.append(item)
- for item in ofpToGraphLatNodeIter[node]:
- if item > 0.0:
- ofpToGraphList.append(item)
-
- endToEndAvg = round(numpy.mean(endToEndList), 2)
- ofpToGraphAvg = numpy.mean(ofpToGraphList)
- endToEndStd = round(numpy.std(endToEndList), 2)
-
- main.log.report( " - Node "+str(node+1)+" Summary - " )
- main.log.report( " End-to-end Avg: "+
- str(round(endToEndAvg,2))+" ms"+
- " End-to-end Std dev: "+
- str(round(endToEndStd,2))+" ms")
-
- dbCmdList.append(
- "INSERT INTO switch_latency_tests VALUES("
- "'"+timeToPost+"','switch_latency_results',"
- ""+runNum+","+str(clusterCount)+",'baremetal"+str(node+1)+"',"
- ""+str(endToEndAvg)+","+str(endToEndStd)+",0,0);"
- )
- #main.log.report( " Ofp-to-graph Avg: "+
- # str(round(ofpToGraphAvg,2))+" ms"+
- # " Ofp-to-graph Std dev: "+
- # str(round(numpy.std(ofpToGraphList),2))+
- # " ms")
-
- if debugMode == 'on':
- main.ONOS1.cpLogsToDir( "/opt/onos/log/karaf.log",
- "/tmp/", copyFileName="sw_lat_karaf" )
-
- #Write to file for posting to DB
- fResult = open(resultPath, 'a')
- for line in dbCmdList:
- if line:
- fResult.write(line+"\n")
- fResult.close()
-
- #TODO: correct assert
- assertion = main.TRUE
-
- utilities.assert_equals( expect=main.TRUE, actual=assertion,
- onpass="Switch latency test successful",
- onfail="Switch latency test failed" )
-
- def CASE3( self, main ):
- """
- Bring port up / down and measure latency.
- Port enable / disable is simulated by ifconfig up / down
-
- In ONOS-next, we must ensure that the port we are
- manipulating is connected to another switch with a valid
- connection. Otherwise, graph view will not be updated.
- """
- import time
- import subprocess
- import os
- import requests
- import json
- import numpy
- global clusterCount
- global runNum
- global timeToPost
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
- defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
- assertion = main.TRUE
- # Number of iterations of case
- numIter = main.params[ 'TEST' ][ 'numIter' ]
- iterIgnore = int( main.params[ 'TEST' ][ 'iterIgnore' ] )
-
- # Timestamp 'keys' for json metrics output.
- # These are subject to change, hence moved into params
- deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
- graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
- debugMode = main.params[ 'TEST' ][ 'debugMode' ]
- postToDB = main.params[ 'DB' ][ 'postToDB' ]
- resultPath = main.params[ 'DB' ][ 'portEventResultPath' ]
-
- localTime = time.strftime( '%x %X' )
- localTime = localTime.replace( "/", "" )
- localTime = localTime.replace( " ", "_" )
- localTime = localTime.replace( ":", "" )
- if debugMode == 'on':
- main.ONOS1.tsharkPcap( "eth0",
- "/tmp/port_lat_pcap_" + localTime )
-
- # Threshold for this test case
- upThresholdStr = main.params[ 'TEST' ][ 'portUpThreshold' ]
- downThresholdStr = main.params[ 'TEST' ][ 'portDownThreshold' ]
-
- upThresholdObj = upThresholdStr.split( "," )
- downThresholdObj = downThresholdStr.split( "," )
-
- upThresholdMin = int( upThresholdObj[ 0 ] )
- upThresholdMax = int( upThresholdObj[ 1 ] )
-
- downThresholdMin = int( downThresholdObj[ 0 ] )
- downThresholdMax = int( downThresholdObj[ 1 ] )
-
- # NOTE: Some hardcoded variables you may need to configure
- # besides the params
-
- tsharkPortStatus = "OFP 130 Port Status"
-
- tsharkPortUp = "/tmp/tshark_port_up.txt"
- tsharkPortDown = "/tmp/tshark_port_down.txt"
- interfaceConfig = "s1-eth1"
-
- main.log.report( "Port enable / disable latency" )
- main.log.report( "Simulated by ifconfig up / down" )
- main.log.report( "Total iterations of test: " + str( numIter ) )
-
- main.step( "Assign switches s1 and s2 to controller 1" )
- main.Mininet1.assignSwController( sw="1", ip1=ONOS1Ip,
- port1=defaultSwPort )
- main.Mininet1.assignSwController( sw="2", ip1=ONOS1Ip,
- port1=defaultSwPort )
-
- # Give enough time for metrics to propagate the
- # assign controller event. Otherwise, these events may
- # carry over to our measurements
- time.sleep( 15 )
-
- portUpDeviceToOfpList = []
- portUpGraphToOfpList = []
- portDownDeviceToOfpList = []
- portDownGraphToOfpList = []
-
- # Initialize 2d array filled with 0's
- # arraySizeFormat[clusterCount][numIter]
- portUpDevNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
- portUpGraphNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
- portDownDevNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
- portDownGraphNodeIter = numpy.zeros(( clusterCount, int(numIter) ))
-
- for i in range( 0, int( numIter ) ):
- main.step( "Starting wireshark capture for port status down" )
- main.ONOS1.tsharkGrep( tsharkPortStatus,
- tsharkPortDown )
-
- time.sleep( 5 )
-
- # Disable interface that is connected to switch 2
- main.step( "Disable port: " + interfaceConfig )
- main.Mininet1.handle.sendline( "sh ifconfig " +
- interfaceConfig + " down" )
- main.Mininet1.handle.expect( "mininet>" )
-
- time.sleep( 3 )
- main.ONOS1.tsharkStop()
-
- # Copy tshark output file from ONOS to TestON instance
- #/tmp directory
- os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
- tsharkPortDown + " /tmp/" )
-
- fPortDown = open( tsharkPortDown, 'r' )
- # Get first line of port down event from tshark
- fLine = fPortDown.readline()
- objDown = fLine.split( " " )
- if len( fLine ) > 0:
- # NOTE: objDown[ 1 ] is a very unreliable
- # way to determine the timestamp. If
- # results seem way off, check the object
- # itself by printing it out
- timestampBeginPtDown = int( float( objDown[ 1 ] ) * 1000 )
- # For some reason, wireshark decides to record the
- # timestamp at the 3rd object position instead of
- # 2nd at unpredictable times. This statement is
- # used to capture that odd behavior and use the
- # correct epoch time
- if timestampBeginPtDown < 1400000000000:
- timestampBeginPtDown = \
- int( float( objDown[ 2 ] ) * 1000 )
-
- main.log.info( "Port down begin timestamp: " +
- str( timestampBeginPtDown ) )
- else:
- main.log.info( "Tshark output file returned unexpected" +
- " results: " + str( objDown ) )
- timestampBeginPtDown = 0
- fPortDown.close()
-
- main.step( "Obtain t1 by metrics call" )
- jsonStrUp1 = main.ONOS1cli.topologyEventsMetrics()
- jsonObj1 = json.loads( jsonStrUp1 )
- # Obtain graph timestamp. This timestsamp captures
- # the epoch time at which the topology graph was updated.
- graphTimestamp1 = \
- jsonObj1[ graphTimestamp ][ 'value' ]
- # Obtain device timestamp. This timestamp captures
- # the epoch time at which the device event happened
- deviceTimestamp1 = \
- jsonObj1[ deviceTimestamp ][ 'value' ]
- # Get delta between graph event and OFP
- ptDownGraphToOfp1 = int( graphTimestamp1 ) -\
- int( timestampBeginPtDown )
- # Get delta between device event and OFP
- ptDownDeviceToOfp1 = int( deviceTimestamp1 ) -\
- int( timestampBeginPtDown )
-
- if ptDownGraphToOfp1 > downThresholdMin and\
- ptDownGraphToOfp1 < downThresholdMax and i > iterIgnore:
- portDownGraphNodeIter[0][i] = ptDownGraphToOfp1
- main.log.info("iter"+str(i)+" port down graph-to-ofp: "+
- str(ptDownGraphToOfp1)+" ms")
- else:
- main.log.info("iter"+str(i)+" skipped. Result: "+
- str(ptDownGraphToOfp1)+" ms")
- if ptDownDeviceToOfp1 > downThresholdMin and\
- ptDownDeviceToOfp1 < downThresholdMax and i > iterIgnore:
- portDownDevNodeIter[0][i] = ptDownDeviceToOfp1
- main.log.info("iter"+str(i)+" port down device-to-ofp: "+
- str(ptDownDeviceToOfp1)+" ms")
- else:
- main.log.info("iter"+str(i)+" skipped. Result: "+
- str(ptDownDeviceToOfp1)+" ms")
-
- if clusterCount >= 3:
- jsonStrUp2 = main.ONOS2cli.topologyEventsMetrics()
- jsonStrUp3 = main.ONOS3cli.topologyEventsMetrics()
- jsonObj2 = json.loads( jsonStrUp2 )
- jsonObj3 = json.loads( jsonStrUp3 )
- graphTimestamp2 = \
- jsonObj2[ graphTimestamp ][ 'value' ]
- graphTimestamp3 = \
- jsonObj3[ graphTimestamp ][ 'value' ]
- deviceTimestamp2 = \
- jsonObj2[ deviceTimestamp ][ 'value' ]
- deviceTimestamp3 = \
- jsonObj3[ deviceTimestamp ][ 'value' ]
- ptDownGraphToOfp2 = int( graphTimestamp2 ) -\
- int( timestampBeginPtDown )
- ptDownGraphToOfp3 = int( graphTimestamp3 ) -\
- int( timestampBeginPtDown )
- ptDownDeviceToOfp2 = int( deviceTimestamp2 ) -\
- int( timestampBeginPtDown )
- ptDownDeviceToOfp3 = int( deviceTimestamp3 ) -\
- int( timestampBeginPtDown )
-
- if ptDownGraphToOfp2 > downThresholdMin and\
- ptDownGraphToOfp2 < downThresholdMax and i > iterIgnore:
- portDownGraphNodeIter[1][i] = ptDownGraphToOfp2
- main.log.info("ONOS2 iter"+str(i)+" graph-to-ofp: "+
- str(ptDownGraphToOfp2)+" ms")
-
- if ptDownDeviceToOfp2 > downThresholdMin and\
- ptDownDeviceToOfp2 < downThresholdMax and i > iterIgnore:
- portDownDevNodeIter[1][i] = ptDownDeviceToOfp2
- main.log.info("ONOS2 iter"+str(i)+" device-to-ofp: "+
- str(ptDownDeviceToOfp2)+" ms")
-
- if ptDownGraphToOfp3 > downThresholdMin and\
- ptDownGraphToOfp3 < downThresholdMax and i > iterIgnore:
- portDownGraphNodeIter[2][i] = ptDownGraphToOfp3
- main.log.info("ONOS3 iter"+str(i)+" graph-to-ofp: "+
- str(ptDownGraphToOfp3)+" ms")
-
- if ptDownDeviceToOfp3 > downThresholdMin and\
- ptDownDeviceToOfp3 < downThresholdMax and i > iterIgnore:
- portDownDevNodeIter[2][i] = ptDownDeviceToOfp3
- main.log.info("ONOS3 iter"+str(i)+" device-to-ofp: "+
- str(ptDownDeviceToOfp3)+" ms")
-
- if clusterCount >= 5:
- jsonStrUp4 = main.ONOS4cli.topologyEventsMetrics()
- jsonStrUp5 = main.ONOS5cli.topologyEventsMetrics()
- jsonObj4 = json.loads( jsonStrUp4 )
- jsonObj5 = json.loads( jsonStrUp5 )
- graphTimestamp4 = \
- jsonObj4[ graphTimestamp ][ 'value' ]
- graphTimestamp5 = \
- jsonObj5[ graphTimestamp ][ 'value' ]
- deviceTimestamp4 = \
- jsonObj4[ deviceTimestamp ][ 'value' ]
- deviceTimestamp5 = \
- jsonObj5[ deviceTimestamp ][ 'value' ]
- ptDownGraphToOfp4 = int( graphTimestamp4 ) -\
- int( timestampBeginPtDown )
- ptDownGraphToOfp5 = int( graphTimestamp5 ) -\
- int( timestampBeginPtDown )
- ptDownDeviceToOfp4 = int( deviceTimestamp4 ) -\
- int( timestampBeginPtDown )
- ptDownDeviceToOfp5 = int( deviceTimestamp5 ) -\
- int( timestampBeginPtDown )
-
- if ptDownGraphToOfp4 > downThresholdMin and\
- ptDownGraphToOfp4 < downThresholdMax and i > iterIgnore:
- portDownGraphNodeIter[3][i] = ptDownGraphToOfp4
- main.log.info("ONOS4 iter"+str(i)+" graph-to-ofp: "+
- str(ptDownGraphToOfp4)+" ms")
-
- if ptDownDeviceToOfp4 > downThresholdMin and\
- ptDownDeviceToOfp4 < downThresholdMax and i > iterIgnore:
- portDownDevNodeIter[3][i] = ptDownDeviceToOfp4
- main.log.info("ONOS4 iter"+str(i)+" device-to-ofp: "+
- str(ptDownDeviceToOfp4)+" ms")
-
- if ptDownGraphToOfp5 > downThresholdMin and\
- ptDownGraphToOfp5 < downThresholdMax and i > iterIgnore:
- portDownGraphNodeIter[4][i] = ptDownGraphToOfp5
- main.log.info("ONOS5 iter"+str(i)+" graph-to-ofp: "+
- str(ptDownGraphToOfp5)+" ms")
-
- if ptDownDeviceToOfp5 > downThresholdMin and\
- ptDownDeviceToOfp5 < downThresholdMax and i > iterIgnore:
- portDownDevNodeIter[4][i] = ptDownDeviceToOfp5
- main.log.info("ONOS5 iter"+str(i)+" device-to-ofp: "+
- str(ptDownDeviceToOfp5)+" ms")
-
- if clusterCount >= 7:
- jsonStrUp6 = main.ONOS6cli.topologyEventsMetrics()
- jsonStrUp7 = main.ONOS7cli.topologyEventsMetrics()
- jsonObj6 = json.loads( jsonStrUp6 )
- jsonObj7 = json.loads( jsonStrUp7 )
- graphTimestamp6 = \
- jsonObj6[ graphTimestamp ][ 'value' ]
- graphTimestamp7 = \
- jsonObj7[ graphTimestamp ][ 'value' ]
- deviceTimestamp6 = \
- jsonObj6[ deviceTimestamp ][ 'value' ]
- deviceTimestamp7 = \
- jsonObj7[ deviceTimestamp ][ 'value' ]
- ptDownGraphToOfp6 = int( graphTimestamp6 ) -\
- int( timestampBeginPtDown )
- ptDownGraphToOfp7 = int( graphTimestamp7 ) -\
- int( timestampBeginPtDown )
- ptDownDeviceToOfp6 = int( deviceTimestamp6 ) -\
- int( timestampBeginPtDown )
- ptDownDeviceToOfp7 = int( deviceTimestamp7 ) -\
- int( timestampBeginPtDown )
-
- if ptDownGraphToOfp6 > downThresholdMin and\
- ptDownGraphToOfp6 < downThresholdMax and i > iterIgnore:
- portDownGraphNodeIter[5][i] = ptDownGraphToOfp6
- main.log.info("ONOS6 iter"+str(i)+" graph-to-ofp: "+
- str(ptDownGraphToOfp6)+" ms")
-
- if ptDownDeviceToOfp6 > downThresholdMin and\
- ptDownDeviceToOfp6 < downThresholdMax and i > iterIgnore:
- portDownDevNodeIter[5][i] = ptDownDeviceToOfp6
- main.log.info("ONOS6 iter"+str(i)+" device-to-ofp: "+
- str(ptDownDeviceToOfp6)+" ms")
-
- if ptDownGraphToOfp7 > downThresholdMin and\
- ptDownGraphToOfp7 < downThresholdMax and i > iterIgnore:
- portDownGraphNodeIter[6][i] = ptDownGraphToOfp7
- main.log.info("ONOS7 iter"+str(i)+" graph-to-ofp: "+
- str(ptDownGraphToOfp7)+" ms")
-
- if ptDownDeviceToOfp7 > downThresholdMin and\
- ptDownDeviceToOfp7 < downThresholdMax and i > iterIgnore:
- portDownDevNodeIter[6][i] = ptDownDeviceToOfp7
- main.log.info("ONOS7 iter"+str(i)+" device-to-ofp: "+
- str(ptDownDeviceToOfp7)+" ms")
-
- time.sleep( 3 )
-
- # Port up events
- main.step( "Enable port and obtain timestamp" )
- main.step( "Starting wireshark capture for port status up" )
- main.ONOS1.tsharkGrep( tsharkPortStatus, tsharkPortUp )
- time.sleep( 5 )
-
- main.Mininet1.handle.sendline( "sh ifconfig " +
- interfaceConfig + " up" )
- main.Mininet1.handle.expect( "mininet>" )
-
- # Allow time for tshark to capture event
- time.sleep( 5 )
- main.ONOS1.tsharkStop()
-
- time.sleep( 3 )
- os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
- tsharkPortUp + " /tmp/" )
- fPortUp = open( tsharkPortUp, 'r' )
- fLine = fPortUp.readline()
- objUp = fLine.split( " " )
- if len( fLine ) > 0:
- timestampBeginPtUp = int( float( objUp[ 1 ] ) * 1000 )
- if timestampBeginPtUp < 1400000000000:
- timestampBeginPtUp = \
- int( float( objUp[ 2 ] ) * 1000 )
- main.log.info( "Port up begin timestamp: " +
- str( timestampBeginPtUp ) )
- else:
- main.log.info( "Tshark output file returned unexpected" +
- " results." )
- timestampBeginPtUp = 0
- fPortUp.close()
-
- # Obtain metrics shortly afterwards
- # This timestsamp captures
- # the epoch time at which the topology graph was updated.
- main.step( "Obtain t1 by REST call" )
- jsonStrUp1 = main.ONOS1cli.topologyEventsMetrics()
- jsonObj1 = json.loads( jsonStrUp1 )
- graphTimestamp1 = \
- jsonObj1[ graphTimestamp ][ 'value' ]
- # Obtain device timestamp. This timestamp captures
- # the epoch time at which the device event happened
- deviceTimestamp1 = \
- jsonObj1[ deviceTimestamp ][ 'value' ]
- # Get delta between graph event and OFP
- ptUpGraphToOfp1 = int( graphTimestamp1 ) -\
- int( timestampBeginPtUp )
- # Get delta between device event and OFP
- ptUpDeviceToOfp1 = int( deviceTimestamp1 ) -\
- int( timestampBeginPtUp )
-
- if ptUpGraphToOfp1 > upThresholdMin and\
- ptUpGraphToOfp1 < upThresholdMax and i > iterIgnore:
- portUpGraphNodeIter[0][i] = ptUpGraphToOfp1
- main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
- str(ptUpGraphToOfp1)+" ms")
- else:
- main.log.info("iter"+str(i)+" skipped. Result: "+
- str(ptUpGraphToOfp1)+" ms")
-
- if ptUpDeviceToOfp1 > upThresholdMin and\
- ptUpDeviceToOfp1 < upThresholdMax and i > iterIgnore:
- portUpDevNodeIter[0][i] = ptUpDeviceToOfp1
- main.log.info("iter"+str(i)+" port up device-to-ofp: "+
- str(ptUpDeviceToOfp1)+" ms")
- else:
- main.log.info("iter"+str(i)+" skipped. Result: "+
- str(ptUpDeviceToOfp1)+" ms")
-
- if clusterCount >= 3:
- jsonStrUp2 = main.ONOS2cli.topologyEventsMetrics()
- jsonStrUp3 = main.ONOS3cli.topologyEventsMetrics()
- jsonObj2 = json.loads( jsonStrUp2 )
- jsonObj3 = json.loads( jsonStrUp3 )
- graphTimestamp2 = \
- jsonObj2[ graphTimestamp ][ 'value' ]
- graphTimestamp3 = \
- jsonObj3[ graphTimestamp ][ 'value' ]
- deviceTimestamp2 = \
- jsonObj2[ deviceTimestamp ][ 'value' ]
- deviceTimestamp3 = \
- jsonObj3[ deviceTimestamp ][ 'value' ]
- ptUpGraphToOfp2 = int( graphTimestamp2 ) -\
- int( timestampBeginPtUp )
- ptUpGraphToOfp3 = int( graphTimestamp3 ) -\
- int( timestampBeginPtUp )
- ptUpDeviceToOfp2 = int( deviceTimestamp2 ) -\
- int( timestampBeginPtUp )
- ptUpDeviceToOfp3 = int( deviceTimestamp3 ) -\
- int( timestampBeginPtUp )
-
- if ptUpGraphToOfp2 > upThresholdMin and\
- ptUpGraphToOfp2 < upThresholdMax and i > iterIgnore:
- portUpGraphNodeIter[1][i] = ptUpGraphToOfp2
- main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
- str(ptUpGraphToOfp2)+" ms")
-
- if ptUpDeviceToOfp2 > upThresholdMin and\
- ptUpDeviceToOfp2 < upThresholdMax and i > iterIgnore:
- portUpDevNodeIter[1][i] = ptUpDeviceToOfp2
- main.log.info("iter"+str(i)+" port up device-to-ofp: "+
- str(ptUpDeviceToOfp2)+" ms")
-
- if ptUpGraphToOfp3 > upThresholdMin and\
- ptUpGraphToOfp3 < upThresholdMax and i > iterIgnore:
- portUpGraphNodeIter[2][i] = ptUpGraphToOfp3
- main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
- str(ptUpGraphToOfp3)+" ms")
-
- if ptUpDeviceToOfp3 > upThresholdMin and\
- ptUpDeviceToOfp3 < upThresholdMax and i > iterIgnore:
- portUpDevNodeIter[2][i] = ptUpDeviceToOfp3
- main.log.info("iter"+str(i)+" port up device-to-ofp: "+
- str(ptUpDeviceToOfp3)+" ms")
-
- if clusterCount >= 5:
- jsonStrUp4 = main.ONOS4cli.topologyEventsMetrics()
- jsonStrUp5 = main.ONOS5cli.topologyEventsMetrics()
- jsonObj4 = json.loads( jsonStrUp4 )
- jsonObj5 = json.loads( jsonStrUp5 )
- graphTimestamp4 = \
- jsonObj4[ graphTimestamp ][ 'value' ]
- graphTimestamp5 = \
- jsonObj5[ graphTimestamp ][ 'value' ]
- deviceTimestamp4 = \
- jsonObj4[ deviceTimestamp ][ 'value' ]
- deviceTimestamp5 = \
- jsonObj5[ deviceTimestamp ][ 'value' ]
- ptUpGraphToOfp4 = int( graphTimestamp4 ) -\
- int( timestampBeginPtUp )
- ptUpGraphToOfp5 = int( graphTimestamp5 ) -\
- int( timestampBeginPtUp )
- ptUpDeviceToOfp4 = int( deviceTimestamp4 ) -\
- int( timestampBeginPtUp )
- ptUpDeviceToOfp5 = int( deviceTimestamp5 ) -\
- int( timestampBeginPtUp )
-
- if ptUpGraphToOfp4 > upThresholdMin and\
- ptUpGraphToOfp4 < upThresholdMax and i > iterIgnore:
- portUpGraphNodeIter[3][i] = ptUpGraphToOfp4
- main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
- str(ptUpGraphToOfp4)+" ms")
-
- if ptUpDeviceToOfp4 > upThresholdMin and\
- ptUpDeviceToOfp4 < upThresholdMax and i > iterIgnore:
- portUpDevNodeIter[3][i] = ptUpDeviceToOfp4
- main.log.info("iter"+str(i)+" port up device-to-ofp: "+
- str(ptUpDeviceToOfp4)+" ms")
-
- if ptUpGraphToOfp5 > upThresholdMin and\
- ptUpGraphToOfp5 < upThresholdMax and i > iterIgnore:
- portUpGraphNodeIter[4][i] = ptUpGraphToOfp5
- main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
- str(ptUpGraphToOfp5)+" ms")
-
- if ptUpDeviceToOfp5 > upThresholdMin and\
- ptUpDeviceToOfp5 < upThresholdMax and i > iterIgnore:
- portUpDevNodeIter[4][i] = ptUpDeviceToOfp5
- main.log.info("iter"+str(i)+" port up device-to-ofp: "+
- str(ptUpDeviceToOfp5)+" ms")
-
- if clusterCount >= 7:
- jsonStrUp6 = main.ONOS6cli.topologyEventsMetrics()
- jsonStrUp7 = main.ONOS7cli.topologyEventsMetrics()
- jsonObj6 = json.loads( jsonStrUp6 )
- jsonObj7 = json.loads( jsonStrUp7 )
- graphTimestamp6 = \
- jsonObj6[ graphTimestamp ][ 'value' ]
- graphTimestamp7 = \
- jsonObj7[ graphTimestamp ][ 'value' ]
- deviceTimestamp6 = \
- jsonObj6[ deviceTimestamp ][ 'value' ]
- deviceTimestamp7 = \
- jsonObj7[ deviceTimestamp ][ 'value' ]
- ptUpGraphToOfp6 = int( graphTimestamp6 ) -\
- int( timestampBeginPtUp )
- ptUpGraphToOfp7 = int( graphTimestamp7 ) -\
- int( timestampBeginPtUp )
- ptUpDeviceToOfp6 = int( deviceTimestamp6 ) -\
- int( timestampBeginPtUp )
- ptUpDeviceToOfp7 = int( deviceTimestamp7 ) -\
- int( timestampBeginPtUp )
-
- if ptUpGraphToOfp6 > upThresholdMin and\
- ptUpGraphToOfp6 < upThresholdMax and i > iterIgnore:
- portUpGraphNodeIter[5][i] = ptUpGraphToOfp6
- main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
- str(ptUpGraphToOfp6)+" ms")
-
- if ptUpDeviceToOfp6 > upThresholdMin and\
- ptUpDeviceToOfp6 < upThresholdMax and i > iterIgnore:
- portUpDevNodeIter[5][i] = ptUpDeviceToOfp6
- main.log.info("iter"+str(i)+" port up device-to-ofp: "+
- str(ptUpDeviceToOfp6)+" ms")
-
- if ptUpGraphToOfp7 > upThresholdMin and\
- ptUpGraphToOfp7 < upThresholdMax and i > iterIgnore:
- portUpGraphNodeIter[6][i] = ptUpGraphToOfp7
- main.log.info("iter"+str(i)+" port up graph-to-ofp: "+
- str(ptUpGraphToOfp7)+" ms")
-
- if ptUpDeviceToOfp7 > upThresholdMin and\
- ptUpDeviceToOfp7 < upThresholdMax and i > iterIgnore:
- portUpDevNodeIter[6][i] = ptUpDeviceToOfp7
- main.log.info("iter"+str(i)+" port up device-to-ofp: "+
- str(ptUpDeviceToOfp7)+" ms")
-
- # END ITERATION FOR LOOP
-
- portUpDevList = []
- portUpGraphList = []
- portDownDevList = []
- portDownGraphList = []
-
- portUpDevAvg = 0
- portUpGraphAvg = 0
- portDownDevAvg = 0
- portDownGraphAvg = 0
-
- dbCmdList = []
-
- for node in range( 0, clusterCount ):
-
- # NOTE:
- # Currently the 2d array is initialized with 0's.
- # We want to avoid skewing our results if the array
- # was not modified with the correct latency.
- for item in portUpDevNodeIter[node]:
- if item > 0.0:
- portUpDevList.append(item)
- for item in portUpGraphNodeIter[node]:
- if item > 0.0:
- portUpGraphList.append(item)
- for item in portDownDevNodeIter[node]:
- if item > 0.0:
- portDownDevList.append(item)
- for item in portDownGraphNodeIter[node]:
- if item > 0.0:
- portDownGraphList.append(item)
-
- portUpDevAvg = round(numpy.mean(portUpDevList), 2)
- portUpGraphAvg = round(numpy.mean(portUpGraphList), 2)
- portDownDevAvg = round(numpy.mean(portDownDevList), 2)
- portDownGraphAvg = round(numpy.mean(portDownGraphList), 2)
-
- portUpStdDev = round(numpy.std(portUpGraphList),2)
- portDownStdDev = round(numpy.std(portDownGraphList),2)
-
- main.log.report( " - Node "+str(node+1)+" Summary - " )
- #main.log.report( " Port up ofp-to-device "+
- # str(round(portUpDevAvg, 2))+" ms")
- main.log.report( " Port up ofp-to-graph "+
- str(portUpGraphAvg)+" ms")
- #main.log.report( " Port down ofp-to-device "+
- # str(round(portDownDevAvg, 2))+" ms")
- main.log.report( " Port down ofp-to-graph "+
- str(portDownGraphAvg)+" ms")
-
- dbCmdList.append(
- "INSERT INTO port_latency_tests VALUES("
- "'"+timeToPost+"','port_latency_results',"
- ""+runNum+","+str(clusterCount)+",'baremetal"+str(node+1)+"',"
- ""+str(portUpGraphAvg)+","+str(portUpStdDev)+
- ","+str(portDownGraphAvg)+","+str(portDownStdDev)+");"
- )
-
- #Write to file for posting to DB
- fResult = open(resultPath, 'a')
- for line in dbCmdList:
- if line:
- fResult.write(line+"\n")
- fResult.close()
-
- print dbCmdList
-
- # Remove switches from controller for next test
- main.Mininet1.deleteSwController( "s1" )
- main.Mininet1.deleteSwController( "s2" )
-
- #TODO: correct assertion
-
- utilities.assert_equals(
- expect=main.TRUE,
- actual=assertion,
- onpass="Port discovery latency calculation successful",
- onfail="Port discovery test failed" )
-
- def CASE4( self, main ):
- """
- Link down event using loss rate 100%
-
- Important:
- Use a simple 2 switch topology with 1 link between
- the two switches. Ensure that mac addresses of the
- switches are 1 / 2 respectively
- """
- import time
- import subprocess
- import os
- import requests
- import json
- import numpy
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
- defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
- # Number of iterations of case
- numIter = main.params[ 'TEST' ][ 'numIter' ]
-
- # Timestamp 'keys' for json metrics output.
- # These are subject to change, hence moved into params
- deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
- linkTimestamp = main.params[ 'JSON' ][ 'linkTimestamp' ]
- graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
- debugMode = main.params[ 'TEST' ][ 'debugMode' ]
-
- localTime = time.strftime( '%x %X' )
- localTime = localTime.replace( "/", "" )
- localTime = localTime.replace( " ", "_" )
- localTime = localTime.replace( ":", "" )
- if debugMode == 'on':
- main.ONOS1.tsharkPcap( "eth0",
- "/tmp/link_lat_pcap_" + localTime )
-
- # Threshold for this test case
- upThresholdStr = main.params[ 'TEST' ][ 'linkUpThreshold' ]
- downThresholdStr = main.params[ 'TEST' ][ 'linkDownThreshold' ]
-
- upThresholdObj = upThresholdStr.split( "," )
- downThresholdObj = downThresholdStr.split( "," )
-
- upThresholdMin = int( upThresholdObj[ 0 ] )
- upThresholdMax = int( upThresholdObj[ 1 ] )
-
- downThresholdMin = int( downThresholdObj[ 0 ] )
- downThresholdMax = int( downThresholdObj[ 1 ] )
-
- assertion = main.TRUE
- # Link event timestamp to system time list
- linkDownLinkToSystemList = []
- linkUpLinkToSystemList = []
- # Graph event timestamp to system time list
- linkDownGraphToSystemList = []
- linkUpGraphToSystemList = []
-
- main.log.report( "Link up / down discovery latency between " +
- "two switches" )
- main.log.report( "Simulated by setting loss-rate 100%" )
- main.log.report( "'tc qdisc add dev <intfs> root netem loss 100%'" )
- main.log.report( "Total iterations of test: " + str( numIter ) )
-
- main.step( "Assign all switches" )
- main.Mininet1.assignSwController( sw="1",
- ip1=ONOS1Ip, port1=defaultSwPort )
- main.Mininet1.assignSwController( sw="2",
- ip1=ONOS1Ip, port1=defaultSwPort )
-
- main.step( "Verifying switch assignment" )
- resultS1 = main.Mininet1.getSwController( sw="s1" )
- resultS2 = main.Mininet1.getSwController( sw="s2" )
-
- # Allow time for events to finish before taking measurements
- time.sleep( 10 )
-
- linkDown1 = False
- linkDown2 = False
- linkDown3 = False
- # Start iteration of link event test
- for i in range( 0, int( numIter ) ):
- main.step( "Getting initial system time as t0" )
-
- # System time in epoch ms
- timestampLinkDownT0 = time.time() * 1000
- # Link down is simulated by 100% loss rate using traffic
- # control command
- main.Mininet1.handle.sendline(
- "sh tc qdisc add dev s1-eth1 root netem loss 100%" )
-
- # TODO: Iterate through 'links' command to verify that
- # link s1 -> s2 went down ( loop timeout 30 seconds )
- # on all 3 ONOS instances
- main.log.info( "Checking ONOS for link update" )
- loopCount = 0
- while( not ( linkDown1 and linkDown2 and linkDown3 )
- and loopCount < 30 ):
- jsonStr1 = main.ONOS1cli.links()
- jsonStr2 = main.ONOS2cli.links()
- jsonStr3 = main.ONOS3cli.links()
-
- if not ( jsonStr1 and jsonStr2 and jsonStr3 ):
- main.log.error( "CLI command returned error " )
- break
- else:
- jsonObj1 = json.loads( jsonStr1 )
- jsonObj2 = json.loads( jsonStr2 )
- jsonObj3 = json.loads( jsonStr3 )
- for obj1 in jsonObj1:
- if '01' not in obj1[ 'src' ][ 'device' ]:
- linkDown1 = True
- main.log.info( "Link down from " +
- "s1 -> s2 on ONOS1 detected" )
- for obj2 in jsonObj2:
- if '01' not in obj2[ 'src' ][ 'device' ]:
- linkDown2 = True
- main.log.info( "Link down from " +
- "s1 -> s2 on ONOS2 detected" )
- for obj3 in jsonObj3:
- if '01' not in obj3[ 'src' ][ 'device' ]:
- linkDown3 = True
- main.log.info( "Link down from " +
- "s1 -> s2 on ONOS3 detected" )
-
- loopCount += 1
- # If CLI doesn't like the continuous requests
- # and exits in this loop, increase the sleep here.
- # Consequently, while loop timeout will increase
- time.sleep( 1 )
-
- # Give time for metrics measurement to catch up
- # NOTE: May need to be configured more accurately
- time.sleep( 10 )
- # If we exited the while loop and link down 1,2,3 are still
- # false, then ONOS has failed to discover link down event
- if not ( linkDown1 and linkDown2 and linkDown3 ):
- main.log.info( "Link down discovery failed" )
-
- linkDownLatGraph1 = 0
- linkDownLatGraph2 = 0
- linkDownLatGraph3 = 0
- linkDownLatDevice1 = 0
- linkDownLatDevice2 = 0
- linkDownLatDevice3 = 0
-
- assertion = main.FALSE
- else:
- jsonTopoMetrics1 =\
- main.ONOS1cli.topologyEventsMetrics()
- jsonTopoMetrics2 =\
- main.ONOS2cli.topologyEventsMetrics()
- jsonTopoMetrics3 =\
- main.ONOS3cli.topologyEventsMetrics()
- jsonTopoMetrics1 = json.loads( jsonTopoMetrics1 )
- jsonTopoMetrics2 = json.loads( jsonTopoMetrics2 )
- jsonTopoMetrics3 = json.loads( jsonTopoMetrics3 )
-
- main.log.info( "Obtaining graph and device timestamp" )
- graphTimestamp1 = \
- jsonTopoMetrics1[ graphTimestamp ][ 'value' ]
- graphTimestamp2 = \
- jsonTopoMetrics2[ graphTimestamp ][ 'value' ]
- graphTimestamp3 = \
- jsonTopoMetrics3[ graphTimestamp ][ 'value' ]
-
- linkTimestamp1 = \
- jsonTopoMetrics1[ linkTimestamp ][ 'value' ]
- linkTimestamp2 = \
- jsonTopoMetrics2[ linkTimestamp ][ 'value' ]
- linkTimestamp3 = \
- jsonTopoMetrics3[ linkTimestamp ][ 'value' ]
-
- if graphTimestamp1 and graphTimestamp2 and\
- graphTimestamp3 and linkTimestamp1 and\
- linkTimestamp2 and linkTimestamp3:
- linkDownLatGraph1 = int( graphTimestamp1 ) -\
- int( timestampLinkDownT0 )
- linkDownLatGraph2 = int( graphTimestamp2 ) -\
- int( timestampLinkDownT0 )
- linkDownLatGraph3 = int( graphTimestamp3 ) -\
- int( timestampLinkDownT0 )
-
- linkDownLatLink1 = int( linkTimestamp1 ) -\
- int( timestampLinkDownT0 )
- linkDownLatLink2 = int( linkTimestamp2 ) -\
- int( timestampLinkDownT0 )
- linkDownLatLink3 = int( linkTimestamp3 ) -\
- int( timestampLinkDownT0 )
- else:
- main.log.error( "There was an error calculating" +
- " the delta for link down event" )
- linkDownLatGraph1 = 0
- linkDownLatGraph2 = 0
- linkDownLatGraph3 = 0
-
- linkDownLatDevice1 = 0
- linkDownLatDevice2 = 0
- linkDownLatDevice3 = 0
-
- main.log.info( "Link down latency ONOS1 iteration " +
- str( i ) + " (end-to-end): " +
- str( linkDownLatGraph1 ) + " ms" )
- main.log.info( "Link down latency ONOS2 iteration " +
- str( i ) + " (end-to-end): " +
- str( linkDownLatGraph2 ) + " ms" )
- main.log.info( "Link down latency ONOS3 iteration " +
- str( i ) + " (end-to-end): " +
- str( linkDownLatGraph3 ) + " ms" )
-
- main.log.info( "Link down latency ONOS1 iteration " +
- str( i ) + " (link-event-to-system-timestamp): " +
- str( linkDownLatLink1 ) + " ms" )
- main.log.info( "Link down latency ONOS2 iteration " +
- str( i ) + " (link-event-to-system-timestamp): " +
- str( linkDownLatLink2 ) + " ms" )
- main.log.info( "Link down latency ONOS3 iteration " +
- str( i ) + " (link-event-to-system-timestamp): " +
- str( linkDownLatLink3 ) )
-
- # Calculate avg of node calculations
- linkDownLatGraphAvg =\
- ( linkDownLatGraph1 +
- linkDownLatGraph2 +
- linkDownLatGraph3 ) / 3
- linkDownLatLinkAvg =\
- ( linkDownLatLink1 +
- linkDownLatLink2 +
- linkDownLatLink3 ) / 3
-
- # Set threshold and append latency to list
- if linkDownLatGraphAvg > downThresholdMin and\
- linkDownLatGraphAvg < downThresholdMax:
- linkDownGraphToSystemList.append(
- linkDownLatGraphAvg )
- else:
- main.log.info( "Link down latency exceeded threshold" )
- main.log.info( "Results for iteration " + str( i ) +
- "have been omitted" )
- if linkDownLatLinkAvg > downThresholdMin and\
- linkDownLatLinkAvg < downThresholdMax:
- linkDownLinkToSystemList.append(
- linkDownLatLinkAvg )
- else:
- main.log.info( "Link down latency exceeded threshold" )
- main.log.info( "Results for iteration " + str( i ) +
- "have been omitted" )
-
- # NOTE: To remove loss rate and measure latency:
- # 'sh tc qdisc del dev s1-eth1 root'
- timestampLinkUpT0 = time.time() * 1000
- main.Mininet1.handle.sendline( "sh tc qdisc del dev " +
- "s1-eth1 root" )
- main.Mininet1.handle.expect( "mininet>" )
-
- main.log.info( "Checking ONOS for link update" )
-
- linkDown1 = True
- linkDown2 = True
- linkDown3 = True
- loopCount = 0
- while( ( linkDown1 and linkDown2 and linkDown3 )
- and loopCount < 30 ):
- jsonStr1 = main.ONOS1cli.links()
- jsonStr2 = main.ONOS2cli.links()
- jsonStr3 = main.ONOS3cli.links()
- if not ( jsonStr1 and jsonStr2 and jsonStr3 ):
- main.log.error( "CLI command returned error " )
- break
- else:
- jsonObj1 = json.loads( jsonStr1 )
- jsonObj2 = json.loads( jsonStr2 )
- jsonObj3 = json.loads( jsonStr3 )
-
- for obj1 in jsonObj1:
- if '01' in obj1[ 'src' ][ 'device' ]:
- linkDown1 = False
- main.log.info( "Link up from " +
- "s1 -> s2 on ONOS1 detected" )
- for obj2 in jsonObj2:
- if '01' in obj2[ 'src' ][ 'device' ]:
- linkDown2 = False
- main.log.info( "Link up from " +
- "s1 -> s2 on ONOS2 detected" )
- for obj3 in jsonObj3:
- if '01' in obj3[ 'src' ][ 'device' ]:
- linkDown3 = False
- main.log.info( "Link up from " +
- "s1 -> s2 on ONOS3 detected" )
-
- loopCount += 1
- time.sleep( 1 )
-
- if ( linkDown1 and linkDown2 and linkDown3 ):
- main.log.info( "Link up discovery failed" )
-
- linkUpLatGraph1 = 0
- linkUpLatGraph2 = 0
- linkUpLatGraph3 = 0
- linkUpLatDevice1 = 0
- linkUpLatDevice2 = 0
- linkUpLatDevice3 = 0
-
- assertion = main.FALSE
- else:
- jsonTopoMetrics1 =\
- main.ONOS1cli.topologyEventsMetrics()
- jsonTopoMetrics2 =\
- main.ONOS2cli.topologyEventsMetrics()
- jsonTopoMetrics3 =\
- main.ONOS3cli.topologyEventsMetrics()
- jsonTopoMetrics1 = json.loads( jsonTopoMetrics1 )
- jsonTopoMetrics2 = json.loads( jsonTopoMetrics2 )
- jsonTopoMetrics3 = json.loads( jsonTopoMetrics3 )
-
- main.log.info( "Obtaining graph and device timestamp" )
- graphTimestamp1 = \
- jsonTopoMetrics1[ graphTimestamp ][ 'value' ]
- graphTimestamp2 = \
- jsonTopoMetrics2[ graphTimestamp ][ 'value' ]
- graphTimestamp3 = \
- jsonTopoMetrics3[ graphTimestamp ][ 'value' ]
-
- linkTimestamp1 = \
- jsonTopoMetrics1[ linkTimestamp ][ 'value' ]
- linkTimestamp2 = \
- jsonTopoMetrics2[ linkTimestamp ][ 'value' ]
- linkTimestamp3 = \
- jsonTopoMetrics3[ linkTimestamp ][ 'value' ]
-
- if graphTimestamp1 and graphTimestamp2 and\
- graphTimestamp3 and linkTimestamp1 and\
- linkTimestamp2 and linkTimestamp3:
- linkUpLatGraph1 = int( graphTimestamp1 ) -\
- int( timestampLinkUpT0 )
- linkUpLatGraph2 = int( graphTimestamp2 ) -\
- int( timestampLinkUpT0 )
- linkUpLatGraph3 = int( graphTimestamp3 ) -\
- int( timestampLinkUpT0 )
-
- linkUpLatLink1 = int( linkTimestamp1 ) -\
- int( timestampLinkUpT0 )
- linkUpLatLink2 = int( linkTimestamp2 ) -\
- int( timestampLinkUpT0 )
- linkUpLatLink3 = int( linkTimestamp3 ) -\
- int( timestampLinkUpT0 )
- else:
- main.log.error( "There was an error calculating" +
- " the delta for link down event" )
- linkUpLatGraph1 = 0
- linkUpLatGraph2 = 0
- linkUpLatGraph3 = 0
-
- linkUpLatDevice1 = 0
- linkUpLatDevice2 = 0
- linkUpLatDevice3 = 0
-
- if debugMode == 'on':
- main.log.info( "Link up latency ONOS1 iteration " +
- str( i ) + " (end-to-end): " +
- str( linkUpLatGraph1 ) + " ms" )
- main.log.info( "Link up latency ONOS2 iteration " +
- str( i ) + " (end-to-end): " +
- str( linkUpLatGraph2 ) + " ms" )
- main.log.info( "Link up latency ONOS3 iteration " +
- str( i ) + " (end-to-end): " +
- str( linkUpLatGraph3 ) + " ms" )
-
- main.log.info(
- "Link up latency ONOS1 iteration " +
- str( i ) +
- " (link-event-to-system-timestamp): " +
- str( linkUpLatLink1 ) +
- " ms" )
- main.log.info(
- "Link up latency ONOS2 iteration " +
- str( i ) +
- " (link-event-to-system-timestamp): " +
- str( linkUpLatLink2 ) +
- " ms" )
- main.log.info(
- "Link up latency ONOS3 iteration " +
- str( i ) +
- " (link-event-to-system-timestamp): " +
- str( linkUpLatLink3 ) )
-
- # Calculate avg of node calculations
- linkUpLatGraphAvg =\
- ( linkUpLatGraph1 +
- linkUpLatGraph2 +
- linkUpLatGraph3 ) / 3
- linkUpLatLinkAvg =\
- ( linkUpLatLink1 +
- linkUpLatLink2 +
- linkUpLatLink3 ) / 3
-
- # Set threshold and append latency to list
- if linkUpLatGraphAvg > upThresholdMin and\
- linkUpLatGraphAvg < upThresholdMax:
- linkUpGraphToSystemList.append(
- linkUpLatGraphAvg )
- else:
- main.log.info( "Link up latency exceeded threshold" )
- main.log.info( "Results for iteration " + str( i ) +
- "have been omitted" )
- if linkUpLatLinkAvg > upThresholdMin and\
- linkUpLatLinkAvg < upThresholdMax:
- linkUpLinkToSystemList.append(
- linkUpLatLinkAvg )
- else:
- main.log.info( "Link up latency exceeded threshold" )
- main.log.info( "Results for iteration " + str( i ) +
- "have been omitted" )
-
- # Calculate min, max, avg of list and report
- linkDownMin = min( linkDownGraphToSystemList )
- linkDownMax = max( linkDownGraphToSystemList )
- linkDownAvg = sum( linkDownGraphToSystemList ) / \
- len( linkDownGraphToSystemList )
- linkUpMin = min( linkUpGraphToSystemList )
- linkUpMax = max( linkUpGraphToSystemList )
- linkUpAvg = sum( linkUpGraphToSystemList ) / \
- len( linkUpGraphToSystemList )
- linkDownStdDev = \
- str( round( numpy.std( linkDownGraphToSystemList ), 1 ) )
- linkUpStdDev = \
- str( round( numpy.std( linkUpGraphToSystemList ), 1 ) )
-
- main.log.report( "Link down latency " +
- "Avg: " + str( linkDownAvg ) + " ms " +
- "Std Deviation: " + linkDownStdDev + " ms" )
- main.log.report( "Link up latency " +
- "Avg: " + str( linkUpAvg ) + " ms " +
- "Std Deviation: " + linkUpStdDev + " ms" )
-
- utilities.assert_equals(
- expect=main.TRUE,
- actual=assertion,
- onpass="Link discovery latency calculation successful",
- onfail="Link discovery latency case failed" )
-
- def CASE5( self, main ):
- """
- 100 Switch discovery latency
-
- Important:
- This test case can be potentially dangerous if
- your machine has previously set iptables rules.
- One of the steps of the test case will flush
- all existing iptables rules.
- Note:
- You can specify the number of switches in the
- params file to adjust the switch discovery size
- ( and specify the corresponding topology in Mininet1
- .topo file )
- """
- import time
- import subprocess
- import os
- import requests
- import json
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- MN1Ip = main.params[ 'MN' ][ 'ip1' ]
- ONOSUser = main.params[ 'CTRL' ][ 'user' ]
-
- defaultSwPort = main.params[ 'CTRL' ][ 'port1' ]
-
- # Number of iterations of case
- numIter = main.params[ 'TEST' ][ 'numIter' ]
- numSw = main.params[ 'TEST' ][ 'numSwitch' ]
-
- # Timestamp 'keys' for json metrics output.
- # These are subject to change, hence moved into params
- deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
- graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]
-
- debugMode = main.params[ 'TEST' ][ 'debugMode' ]
-
- localTime = time.strftime( '%X' )
- localTime = localTime.replace( "/", "" )
- localTime = localTime.replace( " ", "_" )
- localTime = localTime.replace( ":", "" )
- if debugMode == 'on':
- main.ONOS1.tsharkPcap( "eth0",
- "/tmp/100_sw_lat_pcap_" + localTime )
-
- # Threshold for this test case
- swDiscThresholdStr = main.params[ 'TEST' ][ 'swDisc100Threshold' ]
- swDiscThresholdObj = swDiscThresholdStr.split( "," )
- swDiscThresholdMin = int( swDiscThresholdObj[ 0 ] )
- swDiscThresholdMax = int( swDiscThresholdObj[ 1 ] )
-
- tsharkOfpOutput = "/tmp/tshark_ofp_" + numSw + "sw.txt"
- tsharkTcpOutput = "/tmp/tshark_tcp_" + numSw + "sw.txt"
-
- tsharkOfpResultList = []
- tsharkTcpResultList = []
-
- swDiscoveryLatList = []
-
- main.case( numSw + " Switch discovery latency" )
- main.step( "Assigning all switches to ONOS1" )
- for i in range( 1, int( numSw ) + 1 ):
- main.Mininet1.assignSwController(
- sw=str( i ),
- ip1=ONOS1Ip,
- port1=defaultSwPort )
-
- # Ensure that nodes are configured with ptpd
- # Just a warning message
- main.log.info( "Please check ptpd configuration to ensure" +
- " All nodes' system times are in sync" )
- time.sleep( 5 )
-
- for i in range( 0, int( numIter ) ):
-
- main.step( "Set iptables rule to block incoming sw connections" )
- # Set iptables rule to block incoming switch connections
- # The rule description is as follows:
- # Append to INPUT rule,
- # behavior DROP that matches following:
- # * packet type: tcp
- # * source IP: MN1Ip
- # * destination PORT: 6633
- main.ONOS1.handle.sendline(
- "sudo iptables -A INPUT -p tcp -s " + MN1Ip +
- " --dport " + defaultSwPort + " -j DROP" )
- main.ONOS1.handle.expect( "\$" )
- # Append to OUTPUT rule,
- # behavior DROP that matches following:
- # * packet type: tcp
- # * source IP: MN1Ip
- # * destination PORT: 6633
- main.ONOS1.handle.sendline(
- "sudo iptables -A OUTPUT -p tcp -s " + MN1Ip +
- " --dport " + defaultSwPort + " -j DROP" )
- main.ONOS1.handle.expect( "\$" )
- # Give time to allow rule to take effect
- # NOTE: Sleep period may need to be configured
- # based on the number of switches in the topology
- main.log.info( "Please wait for switch connection to " +
- "time out" )
- time.sleep( 60 )
-
- # Gather vendor OFP with tshark
- main.ONOS1.tsharkGrep( "OFP 86 Vendor",
- tsharkOfpOutput )
- main.ONOS1.tsharkGrep( "TCP 74 ",
- tsharkTcpOutput )
-
- # NOTE: Remove all iptables rule quickly ( flush )
- # Before removal, obtain TestON timestamp at which
- # removal took place
- # ( ensuring nodes are configured via ptp )
- # sudo iptables -F
-
- t0System = time.time() * 1000
- main.ONOS1.handle.sendline(
- "sudo iptables -F" )
-
- # Counter to track loop count
- counterLoop = 0
- counterAvail1 = 0
- counterAvail2 = 0
- counterAvail3 = 0
- onos1Dev = False
- onos2Dev = False
- onos3Dev = False
- while counterLoop < 60:
- # Continue to check devices for all device
- # availability. When all devices in all 3
- # ONOS instances indicate that devices are available
- # obtain graph event timestamp for t1.
- deviceStrObj1 = main.ONOS1cli.devices()
- deviceStrObj2 = main.ONOS2cli.devices()
- deviceStrObj3 = main.ONOS3cli.devices()
-
- deviceJson1 = json.loads( deviceStrObj1 )
- deviceJson2 = json.loads( deviceStrObj2 )
- deviceJson3 = json.loads( deviceStrObj3 )
-
- for device1 in deviceJson1:
- if device1[ 'available' ]:
- counterAvail1 += 1
- if counterAvail1 == int( numSw ):
- onos1Dev = True
- main.log.info( "All devices have been " +
- "discovered on ONOS1" )
- else:
- counterAvail1 = 0
- for device2 in deviceJson2:
- if device2[ 'available' ]:
- counterAvail2 += 1
- if counterAvail2 == int( numSw ):
- onos2Dev = True
- main.log.info( "All devices have been " +
- "discovered on ONOS2" )
- else:
- counterAvail2 = 0
- for device3 in deviceJson3:
- if device3[ 'available' ]:
- counterAvail3 += 1
- if counterAvail3 == int( numSw ):
- onos3Dev = True
- main.log.info( "All devices have been " +
- "discovered on ONOS3" )
- else:
- counterAvail3 = 0
-
- if onos1Dev and onos2Dev and onos3Dev:
- main.log.info( "All devices have been discovered " +
- "on all ONOS instances" )
- jsonStrTopologyMetrics1 =\
- main.ONOS1cli.topologyEventsMetrics()
- jsonStrTopologyMetrics2 =\
- main.ONOS2cli.topologyEventsMetrics()
- jsonStrTopologyMetrics3 =\
- main.ONOS3cli.topologyEventsMetrics()
-
- # Exit while loop if all devices discovered
- break
-
- counterLoop += 1
- # Give some time in between CLI calls
- #( will not affect measurement )
- time.sleep( 3 )
-
- main.ONOS1.tsharkStop()
-
- os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
- tsharkOfpOutput + " /tmp/" )
- os.system( "scp " + ONOSUser + "@" + ONOS1Ip + ":" +
- tsharkTcpOutput + " /tmp/" )
-
- # TODO: Automate OFP output analysis
- # Debug mode - print out packets captured at runtime
- if debugMode == 'on':
- ofpFile = open( tsharkOfpOutput, 'r' )
- main.log.info( "Tshark OFP Vendor output: " )
- for line in ofpFile:
- tsharkOfpResultList.append( line )
- main.log.info( line )
- ofpFile.close()
-
- tcpFile = open( tsharkTcpOutput, 'r' )
- main.log.info( "Tshark TCP 74 output: " )
- for line in tcpFile:
- tsharkTcpResultList.append( line )
- main.log.info( line )
- tcpFile.close()
-
- jsonObj1 = json.loads( jsonStrTopologyMetrics1 )
- jsonObj2 = json.loads( jsonStrTopologyMetrics2 )
- jsonObj3 = json.loads( jsonStrTopologyMetrics3 )
-
- graphTimestamp1 = \
- jsonObj1[ graphTimestamp ][ 'value' ]
- graphTimestamp2 = \
- jsonObj2[ graphTimestamp ][ 'value' ]
- graphTimestamp3 = \
- jsonObj3[ graphTimestamp ][ 'value' ]
-
- graphLat1 = int( graphTimestamp1 ) - int( t0System )
- graphLat2 = int( graphTimestamp2 ) - int( t0System )
- graphLat3 = int( graphTimestamp3 ) - int( t0System )
-
- avgGraphLat = \
- ( int( graphLat1 ) +
- int( graphLat2 ) +
- int( graphLat3 ) ) / 3
-
- if avgGraphLat > swDiscThresholdMin \
- and avgGraphLat < swDiscThresholdMax:
- swDiscoveryLatList.append(
- avgGraphLat )
- else:
- main.log.info( "100 Switch discovery latency " +
- "exceeded the threshold." )
-
- # END ITERATION FOR LOOP
-
- swLatMin = min( swDiscoveryLatList )
- swLatMax = max( swDiscoveryLatList )
- swLatAvg = sum( swDiscoveryLatList ) /\
- len( swDiscoveryLatList )
-
- main.log.report( "100 Switch discovery lat " +
- "Min: " + str( swLatMin ) + " ms" +
- "Max: " + str( swLatMax ) + " ms" +
- "Avg: " + str( swLatAvg ) + " ms" )
-
- def CASE6( self, main ):
- """
- Increase number of nodes and initiate CLI
- """
- import time
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
- ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
- ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
- ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
-
- cellName = main.params[ 'ENV' ][ 'cellName' ]
-
- global clusterCount
-
- # Cluster size increased everytime the case is defined
- clusterCount += 2
-
- main.log.report( "Increasing cluster size to " +
- str( clusterCount ) )
-
- installResult = main.FALSE
- if clusterCount == 3:
- main.log.info( "Installing nodes 2 and 3" )
- node2Result = \
- main.ONOSbench.onosInstall( node=ONOS2Ip )
- node3Result = \
- main.ONOSbench.onosInstall( node=ONOS3Ip )
- installResult = node2Result and node3Result
-
- time.sleep( 5 )
-
- main.ONOS2cli.startOnosCli( ONOS2Ip )
- main.ONOS3cli.startOnosCli( ONOS3Ip )
-
- elif clusterCount == 5:
- main.log.info( "Installing nodes 4 and 5" )
- node4Result = \
- main.ONOSbench.onosInstall( node=ONOS4Ip )
- node5Result = \
- main.ONOSbench.onosInstall( node=ONOS5Ip )
- installResult = node4Result and node5Result
-
- time.sleep( 5 )
-
- main.ONOS4cli.startOnosCli( ONOS4Ip )
- main.ONOS5cli.startOnosCli( ONOS5Ip )
-
- elif clusterCount == 7:
- main.log.info( "Installing nodes 6 and 7" )
- node6Result = \
- main.ONOSbench.onosInstall( node=ONOS6Ip )
- node7Result = \
- main.ONOSbench.onosInstall( node=ONOS7Ip )
- installResult = node6Result and node7Result
-
- time.sleep( 5 )
-
- main.ONOS6cli.startOnosCli( ONOS6Ip )
- main.ONOS7cli.startOnosCli( ONOS7Ip )
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.topo b/TestON/tests/TopoPerfNext/TopoPerfNext.topo
deleted file mode 100644
index f12d192..0000000
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.topo
+++ /dev/null
@@ -1,163 +0,0 @@
-<TOPOLOGY>
- <COMPONENT>
-
- <ONOSbench>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>1</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOSbench>
-
- <ONOS1cli>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosCliDriver</type>
- <connect_order>2</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS1cli>
-
- <ONOS2cli>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosCliDriver</type>
- <connect_order>3</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS2cli>
-
- <ONOS3cli>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosCliDriver</type>
- <connect_order>4</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS3cli>
-
- <ONOS4cli>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosCliDriver</type>
- <connect_order>5</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS4cli>
-
- <ONOS5cli>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosCliDriver</type>
- <connect_order>6</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS5cli>
-
- <ONOS6cli>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosCliDriver</type>
- <connect_order>7</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS6cli>
-
- <ONOS7cli>
- <host>10.128.174.10</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosCliDriver</type>
- <connect_order>8</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS7cli>
-
- <ONOS1>
- <host>10.128.174.1</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>9</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS1>
-
- <ONOS2>
- <host>10.128.174.2</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>10</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS2>
-
- <ONOS3>
- <host>10.128.174.3</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>11</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS3>
-
- <ONOS4>
- <host>10.128.174.4</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>12</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS4>
-
- <ONOS5>
- <host>10.128.174.5</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>13</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS5>
-
- <ONOS6>
- <host>10.128.174.6</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>14</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS6>
-
- <ONOS7>
- <host>10.128.174.7</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>OnosDriver</type>
- <connect_order>15</connect_order>
- <COMPONENTS> </COMPONENTS>
- </ONOS7>
-
- <Mininet1>
- <host>10.128.10.90</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>MininetCliDriver</type>
- <connect_order>16</connect_order>
- <COMPONENTS>
- <arg1> --custom topo-perf-2sw.py </arg1>
- <arg2> --arp --mac --topo mytopo</arg2>
- <arg3> </arg3>
- <controller> remote </controller>
- </COMPONENTS>
- </Mininet1>
-
- <Mininet2>
- <host>10.128.10.90</host>
- <user>admin</user>
- <password>onos_test</password>
- <type>RemoteMininetDriver</type>
- <connect_order>17</connect_order>
- <COMPONENTS> </COMPONENTS>
- </Mininet2>
-
- </COMPONENT>
-</TOPOLOGY>
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
index ab215e9..eba319c 100644
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
+++ b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.params
@@ -3,7 +3,7 @@
<ENV>
<cellName>topo_perf_test</cellName>
- <cellFeatures>"org.onosproject.metrics,org.onosproject.openflow"</cellFeatures>
+ <cellFeatures>"drivers,metrics,openflow"</cellFeatures>
</ENV>
<GIT>
@@ -13,7 +13,7 @@
</GIT>
<CTRL>
- <user>sdn</user>
+ <user>admin</user>
<ip1>10.254.1.201</ip1>
<port1>6633</port1>
<ip2>10.254.1.202</ip2>
@@ -62,7 +62,7 @@
</topoConfigName>
#Number of times to iterate each case
- <numIter>35</numIter>
+ <numIter>13</numIter>
<numSwitch>2</numSwitch>
#Number of iterations to ignore initially
<iterIgnore>2</iterIgnore>
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
index 276de24..e2245ba 100644
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
+++ b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.py
@@ -39,9 +39,9 @@
except KeyError:
# Jenkins build number is also used in posting to DB
# If this test is not triggered by jenkins, give
- # it the runNum variable instead, ensuring that
+ # it 0 instead, ensuring that
# the DB post will recognize it as a non-jenkins run
- jenkinsBuildNumber = str(runNum)
+ jenkinsBuildNumber = str(0)
main.log.info( 'Job is not run by jenkins. '+
'Build number set to: ' + jenkinsBuildNumber)
@@ -192,8 +192,8 @@
resultPath = main.params['DB']['switchEventResultPath']
thresholdStr = main.params['TEST']['singleSwThreshold']
thresholdObj = thresholdStr.split(',')
- thresholdMin = float(thresholdObj[0])
- thresholdMax = float(thresholdObj[1])
+ thresholdMin = int(thresholdObj[0])
+ thresholdMax = int(thresholdObj[1])
# Look for 'role-request' messages,
# which replaces the 'vendor' messages previously seen
@@ -266,11 +266,16 @@
main.log.info('Assigning s3 to controller')
main.Mininet1.assignSwController(sw='3',
ip1=nodeIpList[0], port1=defaultSwPort)
-
- time.sleep(10)
-
+
+ jsonStr = []
+ for node in range (0, clusterCount):
+ metricsSwUp = CLIs[node].topologyEventsMetrics()
+ jsonStr.append(metricsSwUp)
+
main.log.info('Stopping all Tshark processes')
main.ONOS1.tsharkStop()
+
+ time.sleep(5)
main.log.info('Copying over tshark files')
os.system('scp ' + ONOSUser + '@' + nodeIpList[0] +
@@ -283,7 +288,8 @@
nodeIpList[0] + ':' + tsharkOfOutput + ' /tmp/')
# Get tcp syn / ack output
- time.sleep(5)
+ time.sleep(1)
+
tcpFile = open(tsharkTcpOutput, 'r')
tempText = tcpFile.readline()
tempText = tempText.split(' ')
@@ -300,7 +306,6 @@
tcpFile.close()
# Get Role reply output
- time.sleep(5)
ofFile = open(tsharkOfOutput, 'r')
lineOfp = ''
while True:
@@ -343,7 +348,13 @@
if len(tempText) > 1:
main.log.info('Object read in from feature reply capture: '+
str(tempText))
- featureTimestamp = float(tempText[1]) * 1000.0
+ if tempText[1] != ' ' and float(tempText[1]) > 1400000000.0:
+ temp = tempText[1]
+ elif tempText[2] != ' ' and float(tempText[2]) > 1400000000.0:
+ temp = tempText[2]
+ else:
+ temp = 0
+ featureTimestamp = float(temp) * 1000.0
else:
main.log.error('Tshark output file for feature reply' +
' returned unexpected results')
@@ -351,16 +362,11 @@
assertion = main.FALSE
featureFile.close()
- # TODO: calculate feature reply, role request times
- # stack measurements correctly and report
-
- #TODO: Refactor in progress
-
for node in range(0, clusterCount):
nodeNum = node+1
- metricsSwUp = CLIs[node].topologyEventsMetrics
- jsonStr = metricsSwUp()
- jsonObj = json.loads(jsonStr)
+ #metricsSwUp = CLIs[node].topologyEventsMetrics
+ #jsonStr = metricsSwUp()
+ jsonObj = json.loads(jsonStr[node])
if jsonObj:
graphTimestamp = jsonObj[graphTimestampKey]['value']
deviceTimestamp = jsonObj[deviceTimestampKey]['value']
@@ -398,8 +404,9 @@
else:
main.log.info("ONOS "+str(nodeNum)+ " end-to-end "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
-
+ "threshold or premature iteration: ")
+ main.log.info(str(endToEnd))
+
if tcpToFeature >= thresholdMin and\
tcpToFeature < thresholdMax and i >= iterIgnore:
tcpToFeatureLatNodeIter[node][i] = tcpToFeature
@@ -408,7 +415,8 @@
else:
main.log.info("ONOS "+str(nodeNum)+ " tcp-to-feature "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
+ "threshold or premature iteration: ")
+ main.log.info(str(tcpToFeature))
if featureToRole >= thresholdMin and\
featureToRole < thresholdMax and i >= iterIgnore:
@@ -418,7 +426,8 @@
else:
main.log.info("ONOS "+str(nodeNum)+ " feature-to-role "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
+ "threshold or premature iteration: ")
+ main.log.info(str(featureToRole))
if roleToOfp >= thresholdMin and\
roleToOfp < thresholdMax and i >= iterIgnore:
@@ -428,7 +437,8 @@
else:
main.log.info("ONOS "+str(nodeNum)+ " role-to-reply "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
+ "threshold or premature iteration: ")
+ main.log.info(str(roleToOfp))
if ofpToDevice >= thresholdMin and\
ofpToDevice < thresholdMax and i >= iterIgnore:
@@ -438,7 +448,8 @@
else:
main.log.info("ONOS "+str(nodeNum)+ " reply-to-device "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
+ "threshold or premature iteration: ")
+ main.log.info(str(ofpToDevice))
if deviceToGraph >= thresholdMin and\
deviceToGraph < thresholdMax and i >= iterIgnore:
@@ -451,13 +462,14 @@
main.log.info("ONOS "+str(nodeNum) +
" device-to-graph measurement "+
"was set to 0 ms because of precision "+
- "uncertainty")
+ "uncertainty. ")
else:
main.log.info("ONOS "+str(nodeNum)+
" device-to-graph "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
-
+ "threshold or premature iteration: ")
+ main.log.info(str(deviceToGraph))
+
# ********************
time.sleep(5)
@@ -489,22 +501,21 @@
main.step('Remove switch from controller')
main.Mininet1.deleteSwController('s3')
firstDevice = deviceList[0]
- main.log.info( "Removing device " +str(firstDevice)+
- " from ONOS" )
- time.sleep( 5 )
-
# We need to get metrics before removing
# device from the store below.
for node in range(0, clusterCount):
metricsSwDown = CLIs[node].topologyEventsMetrics
jsonStr = metricsSwDown()
removeJsonList.append( json.loads(jsonStr) )
-
- #if deviceId:
- main.ONOS1cli.deviceRemove(firstDevice)
main.ONOS1.tsharkStop()
+
+ main.log.info( "Removing device " +str(firstDevice)+
+ " from ONOS" )
+
+ #if deviceId:
+ main.ONOS1cli.deviceRemove(firstDevice)
main.log.info('Copying over tshark files')
os.system('scp ' + ONOSUser + '@' + nodeIpList[0] +
@@ -520,12 +531,28 @@
index = 1
for line in tempLine:
obj = line.split(' ')
-
- if len(obj) > 1:
+
+ # There are at least 3 objects in field (valid
+ # tshark output is lengthy)
+ if len(obj) > 2:
+ # If first index of object is like an epoch time
+ if obj[1] != ' ' and float(obj[1]) > 1400000000.0:
+ temp = obj[1]
+ elif obj[2] != ' 'and float(obj[2]) > 1400000000.0:
+ temp = obj[2]
+ elif obj[3] != ' 'and float(obj[3]) > 1400000000.0:
+ temp = obj[3]
+ else:
+ temp = 0
if index == 1:
- tFinAck = float(obj[1]) * 1000.0
+ tFinAck = float(temp) * 1000.0
+ elif index == 2:
+ continue
elif index == 3:
- tAck = float(obj[1]) * 1000.0
+ tAck = float(temp) * 1000.0
+ else:
+ tFinAck = 0
+ tAck = 0
else:
main.log.error('Tshark output file for OFP' +
' returned unexpected results')
@@ -571,7 +598,8 @@
main.log.info("ONOS " + str(nodeNum) +
" end-to-end disconnection "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
+ "threshold or premature iteration: ")
+ main.log.info(str(endToEndDisc))
if finAckTransaction >= thresholdMin and\
finAckTransaction < thresholdMax and i >= iterIgnore:
@@ -583,8 +611,9 @@
main.log.info("ONOS "+str(nodeNum)+
" fin/ack transaction "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
-
+ "threshold or premature iteration: ")
+ main.log.info(str(finAckTransaction))
+
if ackToDevice >= thresholdMin and\
ackToDevice < thresholdMax and i >= iterIgnore:
ackToDeviceLatNodeIter[node][i] = ackToDevice
@@ -595,8 +624,9 @@
main.log.info("ONOS "+str(nodeNum)+
" ack-to-device "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
-
+ "threshold or premature iteration: ")
+ main.log.info(str(ackToDevice))
+
if deviceToGraph >= thresholdMin and\
deviceToGraph < thresholdMax and i >= iterIgnore:
deviceToGraphDiscLatNodeIter[node][i] = deviceToGraph
@@ -607,7 +637,8 @@
main.log.info("ONOS "+str(nodeNum)+
" device-to-graph disconnect "+
"measurement ignored due to excess in "+
- "threshold or premature iteration")
+ "threshold or premature iteration: ")
+ main.log.info(str(deviceToGraph))
endToEndAvg = 0
ofpToGraphAvg = 0
@@ -648,15 +679,15 @@
roleToOfpList.append(item)
for item in roleReplyToDeviceLatNodeIter[node]:
- if item > 0.0:
- tcpToFeatureList.append(item)
-
+ if item >= 0.0:
+ ofpToDeviceList.append(item)
+
for item in featureToRoleRequestLatNodeIter[node]:
if item > 0.0:
featureToRoleList.append(item)
for item in deviceToGraphLatNodeIter[node]:
- if item > 0.0:
+ if item >= 0.0:
deviceToGraphList.append(item)
# Switch disconnect measurements
@@ -673,9 +704,9 @@
ackToDeviceList.append(item)
for item in deviceToGraphDiscLatNodeIter[node]:
- if item > 0.0:
+ if item >= 0.0:
deviceToGraphDiscList.append(item)
-
+
endToEndAvg = round(numpy.mean(endToEndList), 2)
endToEndStdDev = round(numpy.std(endToEndList), 2)
@@ -837,10 +868,10 @@
downThresholdStr = main.params['TEST']['portDownThreshold']
upThresholdObj = upThresholdStr.split(',')
downThresholdObj = downThresholdStr.split(',')
- upThresholdMin = float(upThresholdObj[0])
- upThresholdMax = float(upThresholdObj[1])
- downThresholdMin = float(downThresholdObj[0])
- downThresholdMax = float(downThresholdObj[1])
+ upThresholdMin = int(upThresholdObj[0])
+ upThresholdMax = int(upThresholdObj[1])
+ downThresholdMin = int(downThresholdObj[0])
+ downThresholdMax = int(downThresholdObj[1])
interfaceConfig = 's1-eth1'
main.log.report('Port enable / disable latency')
@@ -876,6 +907,11 @@
interfaceConfig + ' down')
main.Mininet1.handle.expect('mininet>')
+ jsonStrPtDown = []
+ for node in range (0, clusterCount):
+ metricsPortDown = CLIs[node].topologyEventsMetrics()
+ jsonStrPtDown.append(metricsPortDown)
+
time.sleep(3)
main.ONOS1.tsharkStop()
@@ -906,9 +942,9 @@
for node in range(0, clusterCount):
nodeNum = node+1
- metricsDown = CLIs[node].topologyEventsMetrics
- jsonStrDown = metricsDown()
- jsonObj = json.loads(jsonStrDown)
+ # metricsDown = CLIs[node].topologyEventsMetrics
+ #jsonStrPtDown[node] = metricsDown()
+ jsonObj = json.loads(jsonStrPtDown[node])
if jsonObj:
graphTimestamp = jsonObj[graphTimestampKey]['value']
@@ -979,6 +1015,11 @@
main.Mininet1.handle.sendline('sh ifconfig ' + interfaceConfig + ' up')
main.Mininet1.handle.expect('mininet>')
+ jsonStrPtUp = []
+ for node in range (0, clusterCount):
+ metricsPortUp = CLIs[node].topologyEventsMetrics()
+ jsonStrPtUp.append(metricsPortUp)
+
time.sleep(5)
main.ONOS1.tsharkStop()
@@ -1001,9 +1042,9 @@
for node in range(0, clusterCount):
nodeNum = node+1
- metricsUp = CLIs[node].topologyEventsMetrics
- jsonStrUp = metricsUp()
- jsonObj = json.loads(jsonStrUp)
+ #metricsUp = CLIs[node].topologyEventsMetrics
+ #jsonStrUp = metricsUp()
+ jsonObj = json.loads(jsonStrPtUp[node])
if jsonObj:
graphTimestamp = jsonObj[graphTimestampKey]['value']
@@ -1102,10 +1143,9 @@
portUpDevToLinkList.append(item)
for item in portUpLinkToGraphNodeIter[node]:
- if item > 0.0:
+ if item >= 0.0:
portUpLinkToGraphList.append(item)
-
for item in portDownEndToEndNodeIter[node]:
if item > 0.0:
portDownEndToEndList.append(item)
@@ -1115,11 +1155,11 @@
portDownOfpToDevList.append(item)
for item in portDownDevToLinkNodeIter[node]:
- if item > 0.0:
+ if item >= 0.0:
portDownDevToLinkList.append(item)
for item in portDownLinkToGraphNodeIter[node]:
- if item > 0.0:
+ if item >= 0.0:
portDownLinkToGraphList.append(item)
portUpEndToEndAvg = round(numpy.mean(portUpEndToEndList), 2)
@@ -1149,7 +1189,7 @@
str(round(portDownEndToEndAvg, 2)) + ' ms')
main.log.report(' Port down Ofp-to-device ' +
str(portDownOfpToDevAvg) + ' ms')
- main.log.report(' Port down Device-to-link' +
+ main.log.report(' Port down Device-to-link ' +
str(portDownDevToLinkAvg) + ' ms')
main.log.report(' Port down Link-to-graph' +
str(portDownLinkToGraphAvg) + ' ms')
diff --git a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo
index 6a70f60..570dece 100644
--- a/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo
+++ b/TestON/tests/TopoPerfNextBM/TopoPerfNextBM.topo
@@ -2,7 +2,7 @@
<COMPONENT>
<ONOSbench>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosDriver</type>
@@ -11,7 +11,7 @@
</ONOSbench>
<ONOS1cli>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -20,7 +20,7 @@
</ONOS1cli>
<ONOS2cli>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -29,7 +29,7 @@
</ONOS2cli>
<ONOS3cli>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -38,7 +38,7 @@
</ONOS3cli>
<ONOS4cli>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -47,7 +47,7 @@
</ONOS4cli>
<ONOS5cli>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -56,7 +56,7 @@
</ONOS5cli>
<ONOS6cli>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -65,7 +65,7 @@
</ONOS6cli>
<ONOS7cli>
- <host>10.128.174.10</host>
+ <host>10.254.1.200</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -74,9 +74,9 @@
</ONOS7cli>
<ONOS1>
- <host>10.128.174.1</host>
- <user>sdn</user>
- <password>rocks</password>
+ <host>10.254.1.201</host>
+ <user>admin</user>
+ <password>onos_test</password>
<type>OnosDriver</type>
<connect_order>9</connect_order>
<COMPONENTS> </COMPONENTS>
@@ -84,8 +84,8 @@
<ONOS2>
<host>10.254.1.202</host>
- <user>sdn</user>
- <password>rocks</password>
+ <user>admin</user>
+ <password>onos_test</password>
<type>OnosDriver</type>
<connect_order>10</connect_order>
<COMPONENTS> </COMPONENTS>
@@ -93,8 +93,8 @@
<ONOS3>
<host>10.254.1.203</host>
- <user>sdn</user>
- <password>rocks</password>
+ <user>admin</user>
+ <password>onos_test</password>
<type>OnosDriver</type>
<connect_order>11</connect_order>
<COMPONENTS> </COMPONENTS>
@@ -102,8 +102,8 @@
<ONOS4>
<host>10.254.1.204</host>
- <user>sdn</user>
- <password>rocks</password>
+ <user>admin</user>
+ <password>onos_test</password>
<type>OnosDriver</type>
<connect_order>12</connect_order>
<COMPONENTS> </COMPONENTS>
@@ -111,8 +111,8 @@
<ONOS5>
<host>10.254.1.205</host>
- <user>sdn</user>
- <password>rocks</password>
+ <user>admin</user>
+ <password>onos_test</password>
<type>OnosDriver</type>
<connect_order>13</connect_order>
<COMPONENTS> </COMPONENTS>
@@ -120,8 +120,8 @@
<ONOS6>
<host>10.254.1.206</host>
- <user>sdn</user>
- <password>rocks</password>
+ <user>admin</user>
+ <password>onos_test</password>
<type>OnosDriver</type>
<connect_order>14</connect_order>
<COMPONENTS> </COMPONENTS>
@@ -129,8 +129,8 @@
<ONOS7>
<host>10.254.1.207</host>
- <user>sdn</user>
- <password>rocks</password>
+ <user>admin</user>
+ <password>onos_test</password>
<type>OnosDriver</type>
<connect_order>15</connect_order>
<COMPONENTS> </COMPONENTS>
@@ -144,7 +144,7 @@
<connect_order>16</connect_order>
<COMPONENTS>
<arg1> --custom topo-perf-2sw.py </arg1>
- <arg2> --arp --mac --topo mytopo</arg2>
+ <arg2> --topo mytopo</arg2>
<arg3> </arg3>
<controller> remote </controller>
</COMPONENTS>
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/TopoPerfNextBM/__init__.py
similarity index 100%
rename from TestON/tests/TopoPerfNext/__init__.py
rename to TestON/tests/TopoPerfNextBM/__init__.py
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.params b/TestON/tests/TopoPerfNextBM/backup/TopoPerfNextBM.params
similarity index 77%
rename from TestON/tests/TopoPerfNext/TopoPerfNext.params
rename to TestON/tests/TopoPerfNextBM/backup/TopoPerfNextBM.params
index dc2d519..d681433 100644
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.params
+++ b/TestON/tests/TopoPerfNextBM/backup/TopoPerfNextBM.params
@@ -1,8 +1,9 @@
<PARAMS>
- <testcases>1,2,3,6,2,3,6,2,3,6,2,3</testcases>
+ <testcases>1,2,3,4,2,3,4,2,3,4,2,3</testcases>
<ENV>
<cellName>topo_perf_test</cellName>
+ <cellFeatures>"org.onosproject.metrics,org.onosproject.openflow"</cellFeatures>
</ENV>
<GIT>
@@ -12,7 +13,7 @@
</GIT>
<CTRL>
- <user>admin</user>
+ <user>sdn</user>
<ip1>10.128.174.1</ip1>
<port1>6633</port1>
<ip2>10.128.174.2</ip2>
@@ -35,6 +36,15 @@
<ip>10.128.174.10</ip>
</BENCH>
+ <TSHARK>
+ <ofpPortStatus>OF 1.3 146</ofpPortStatus>
+ <ofpRoleReply>OF 1.3 90 of_role_reply</ofpRoleReply>
+ <featureReply>OF 1.3 98 of_features_reply</featureReply>
+ <roleRequest>OF 1.3 90 of_role_request</roleRequest>
+ <tcpSynAck>TCP 74 6633</tcpSynAck>
+ <finAckSequence>FIN</finAckSequence>
+ </TSHARK>
+
<TEST>
#'on' or 'off' debug mode.
#If on, logging will be more verbose and
@@ -70,10 +80,10 @@
<DB>
<postToDB>on</postToDB>
<portEventResultPath>
- /home/admin/ONLabTest/TestON/tests/TopoPerfNext/portEventResultDb.log
+ /home/admin/ONLabTest/TestON/tests/TopoPerfNextBM/portEventResultDb.log
</portEventResultPath>
<switchEventResultPath>
- /home/admin/ONLabTest/TestON/tests/TopoPerfNext/switchEventResultDb.log
+ /home/admin/ONLabTest/TestON/tests/TopoPerfNextBM/switchEventResultDb.log
</switchEventResultPath>
</DB>
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/TopoPerfNextBM/portEventResultDb.log
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/TopoPerfNextBM/portEventResultDb.log
diff --git a/TestON/tests/TopoPerfNext/__init__.py b/TestON/tests/TopoPerfNextBM/switchEventResultDb.log
similarity index 100%
copy from TestON/tests/TopoPerfNext/__init__.py
copy to TestON/tests/TopoPerfNextBM/switchEventResultDb.log
diff --git a/TestON/tests/flowTP1g/flowTP1g.params b/TestON/tests/flowTP1g/flowTP1g.params
index 0f08c9a..54320a7 100644
--- a/TestON/tests/flowTP1g/flowTP1g.params
+++ b/TestON/tests/flowTP1g/flowTP1g.params
@@ -1,26 +1,27 @@
<PARAMS>
- <testcases>1,2</testcases>
+ <testcases>1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2</testcases>
<isOnBaremetal>False</isOnBaremetal>
- <SCALE>3</SCALE>
+ <SCALE>1,3,3,5,5,7,7</SCALE>
<availableNodes>7</availableNodes>
<ENV>
<cellName>flowTP</cellName>
- <cellApps>null,demo</cellApps>
+ <cellApps>drivers,null,demo</cellApps>
</ENV>
<TEST>
+ <enableFlowRuleStoreBackup>true</enableFlowRuleStoreBackup>
<skipCleanInstall>yes</skipCleanInstall>
- <warmUp>4</warmUp> #number of runs to warm up the system
- <sampleSize>3</sampleSize> #number of runs to take data from
+ <warmUp>5</warmUp> #number of runs to warm up the system
+ <sampleSize>20</sampleSize> #number of runs to take data from
<neighbors>0,a</neighbors> #list of number of neighbors, a = all
- <flows>3500</flows>
+ <flows>122500</flows>
<switches>35</switches>
-
+ <cooldown>10</cooldown>
<testCMD0>flow-tester.py -f</testCMD0> #base command
<testCMD1> -n </testCMD1> #neighbors
@@ -46,24 +47,24 @@
<ip4>10.128.5.54</ip4>
<port4>6633</port4>
- <ip5>10.128.5.65</ip5>
+ <ip5>10.128.5.55</ip5>
<port5>6633</port5>
- <ip6>10.128.5.66</ip6>
+ <ip6>10.128.5.56</ip6>
<port6>6633</port6>
- <ip7>10.128.5.67</ip7>
+ <ip7>10.128.5.57</ip7>
<port7>6633</port7>
</CTRL>
<MN>
- <ip1>10.128.5.55</ip1>
+ <ip1>10.128.5.59</ip1>
</MN>
<BENCH>
<user>admin</user>
- <ip1>10.128.5.55</ip1>
+ <ip1>10.128.5.50</ip1>
</BENCH>
<JSON>
diff --git a/TestON/tests/flowTP1g/flowTP1g.py b/TestON/tests/flowTP1g/flowTP1g.py
index 352341e..cbea039 100644
--- a/TestON/tests/flowTP1g/flowTP1g.py
+++ b/TestON/tests/flowTP1g/flowTP1g.py
@@ -34,9 +34,12 @@
maxNodes = int(main.params[ 'availableNodes' ])
skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cellName = main.params[ 'ENV' ][ 'cellName' ]
-
- #----do i need this line?
- #main.ONOSbench.handle.sendline("export TERM=vt100")
+
+ main.log.info("==========DEBUG VERSION 3===========")
+
+ main.exceptions = [0]*11
+ main.warnings = [0]*11
+ main.errors = [0]*11
# -- INIT SECTION, ONLY RUNS ONCE -- #
if init == False:
@@ -132,7 +135,7 @@
a(ONOSIp[node])
main.log.info("Startup sequence complete")
-
+ main.ONOSbench.onosErrorLog(ONOSIp[1])
def CASE2( self, main ):
#
@@ -145,6 +148,17 @@
import datetime
import traceback
+ global currentNeighbors
+ try:
+ currentNeighbors
+ except:
+ currentNeighbors = "0"
+ else:
+ if currentNeighbors == "r": #reset
+ currentNeighbors = "0"
+ else:
+ currentNeighbors = "a"
+
testCMD = [ 0,0,0,0 ]
warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
@@ -154,30 +168,33 @@
testCMD[1] = main.params[ 'TEST' ][ 'testCMD1' ]
maxNodes = main.params[ 'availableNodes' ]
onBaremetal = main.params['isOnBaremetal']
-
+ cooldown = main.params[ 'TEST' ][ 'cooldown' ]
cellName = main.params[ 'ENV' ][ 'cellName' ]
BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
BENCHUser = main.params[ 'BENCH' ][ 'user' ]
MN1Ip = main.params[ 'MN' ][ 'ip1' ]
maxNodes = int(main.params[ 'availableNodes' ])
homeDir = os.path.expanduser('~')
-
+ flowRuleBackup = str(main.params[ 'TEST' ][ 'enableFlowRuleStoreBackup' ])
+ main.log.info("Flow Rule Backup is set to:" + flowRuleBackup)
+
servers = str(clusterCount)
- for i in range(0, len(neighborList)):
- if neighborList[i] == 'a':
- neighborList[i] = str(clusterCount - 1)
if clusterCount == 1:
neighborList = ['0']
+ currentNeighbors = "r"
+ else:
+ if currentNeighbors == "a":
+ neighborList = [str(clusterCount-1)]
+ currentNeighbors = "r"
+ else:
+ neighborList = ['0']
+
main.log.info("neightborlist: " + str(neighborList))
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
- #initialize log file, remove any previous data
- resultsLog = open("flowTP1gDB","w+")
- resultsLog.close()
-
#write file to change mem limit to 32 gigs (BAREMETAL ONLY!)
if onBaremetal == "true":
filename = "/onos/tools/package/bin/onos-service"
@@ -208,25 +225,33 @@
ipCSV += main.params[ 'CTRL' ][ tempstr ]
if i < int(maxNodes):
ipCSV +=","
-
- filename = "/onos/tools/package/etc/org.onosproject.provider.nil.link.impl.NullLinkProvider.cfg"
- linkConfig = open(homeDir + filename,'w+')
- linkConfig.write("# eventRate = 2000\n")
- linkConfig.write("neighbors = " + ipCSV)
- main.log.info(" NullLinkProvider.cfg: " + ipCSV)
- linkConfig.close()
-
- main.ONOSbench.handle.sendline("""onos $OC1 "cfg setorg.onosproject.provider.nil.NullProviders enabled true" """)
- main.ONOSbench.handle.expect(":~")
+
+
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount 35" """)
main.ONOSbench.handle.expect(":~")
+ time.sleep(3)
main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
main.ONOSbench.handle.expect(":~")
+ time.sleep(3)
main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
main.ONOSbench.handle.expect(":~")
+ time.sleep(3)
main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
main.ONOSbench.handle.expect(":~")
-
+ time.sleep(3)
+ main.log.info("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBackup + """" """)
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBackup + """" """)
+ main.ONOSbench.handle.expect(":~")
+
+ main.ONOSbench.handle.sendline("onos $OC1 summary")
+ main.ONOSbench.handle.expect(":~")
+ check = main.ONOSbench.handle.before
+
+ main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
+ main.ONOSbench.handle.expect(":~")
+ check = main.ONOSbench.handle.before
+ main.log.info("\nStart up check: \n" + check + "\n")
+
#devide flows
flows = int(main.params[ 'TEST' ][ 'flows' ])
main.log.info("Flow Target = " + str(flows))
@@ -243,35 +268,51 @@
data = [[""]*int(servers)]*int(sampleSize)
maxes = [""]*int(sampleSize)
- for test in range(0, (warmUp + sampleSize)):
- flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
- flowCMD += testCMD[0] + " " + str(flows) + " " + testCMD[1]
- flowCMD += " " + str(n) + " " + str(serverEnvVars)
- print("\n")
- main.log.info("COMMAND: " + flowCMD)
- main.log.info("Executing command")
-
- for i in range(0,15):
- main.ONOSbench.handle.sendline(flowCMD)
- time.sleep(1)
- main.ONOSbench.handle.expect(":~")
- rawResult = main.ONOSbench.handle.before
- if " -> " in rawResult:
- break
+ flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
+ flowCMD += testCMD[0] + " " + str(flows) + " " + testCMD[1]
+ flowCMD += " " + str(n) + " " + str(serverEnvVars) + "-j"
+ main.log.info(flowCMD)
+ #time.sleep(60)
+
+ for test in range(0, warmUp + sampleSize):
+ if test < warmUp:
+ main.log.info("Warm up " + str(test + 1) + " of " + str(warmUp))
+ else:
+ main.log.info("====== Test run: " + str(test-warmUp+1) + " ======")
+
+ main.ONOSbench.handle.sendline(flowCMD)
+ main.ONOSbench.handle.expect(":~")
+ rawResult = main.ONOSbench.handle.before
+ main.log.info("Raw results: \n" + rawResult + "\n")
+
+ if "failed" in rawResult:
+ main.log.report("FLOW_TESTER.PY FAILURE")
+ main.log.report( " \n" + rawResult + " \n")
+ break
+
+ ########################################################################################
result = [""]*(clusterCount)
rawResult = rawResult.splitlines()
- print(rawResult)
+
for node in range(1, clusterCount + 1):
for line in rawResult:
- if ONOSIp[node] in line and " -> " in line:
- myLine = line.split(" ")
- for word in myLine:
- if "ms" in word:
- result[node-1] = int(word.replace("ms",""))
- main.log.info("Parsed result: " + str(result[node-1]))
- break
- break
+ #print("line: " + line)
+ if ONOSIp[node] in line and "server" in line:
+ temp = line.split(" ")
+ for word in temp:
+ #print ("word: " + word)
+ if "elapsed" in repr(word):
+ #print("in elapsed ==========")
+ index = temp.index(word) + 1
+ #print ("index: " + str(index))
+ #print ("temp[index]: " + temp[index])
+ myParsed = (temp[index]).replace(",","")
+ myParsed = myParsed.replace("}","")
+ myParsed = int(myParsed)
+ result[node-1] = myParsed
+ main.log.info( ONOSIp[node] + " : " + str(myParsed))
+ break
if test >= warmUp:
for i in result:
@@ -283,40 +324,27 @@
main.log.info("Data collection iteration: " + str(test-warmUp) + " of " + str(sampleSize))
main.log.info("Throughput time: " + str(maxes[test-warmUp]) + "(ms)")
- if test >= warmUp:
data[test-warmUp] = result
# wait for flows = 0
- removedFlows = False
- repeat = 0
- time.sleep(3)
- while ( removedFlows == False and repeat <= 10 ):
- main.ONOSbench.handle.sendline("onos $OC1 summary| cut -d ' ' -f6")
- main.ONOSbench.handle.expect("~")
- before = main.ONOSbench.handle.before
- parseTest = before.splitlines()
- flowsummary = ""
- for line in parseTest:
- if "flow" in str(line):
- flowsummary = line
- break
- currentflow = ""
- for word in flowsummary.split(" "):
- if "flow" in str(word):
- currentflow = str(word)
- currentflow = currentflow.replace(",","")
- currentflow = currentflow.replace("\n","")
- #main.log.info(currentflow)
+ for checkCount in range(0,5):
+ time.sleep(10)
+ main.ONOSbench.handle.sendline("onos $OC1 summary")
+ main.ONOSbench.handle.expect(":~")
+ flowCheck = main.ONOSbench.handle.before
+ if "flows=0," in flowCheck:
+ main.log.info("Flows removed")
+ break
+ else:
+ for line in flowCheck.splitlines():
+ if "flows=" in line:
+ main.log.info("Current Summary: " + line)
+ if checkCount == 2:
+ main.log.info("Flows are stuck, moving on ")
- zeroFlow = "flows=0"
- if zeroFlow in before:
- removedFlows = True
- main.log.info("\t Wait 5 sec of cool down...")
- time.sleep(5)
- time.sleep(5)
- repeat +=1
-
+ time.sleep(5)
+
main.log.info("raw data: " + str(data))
main.log.info("maxes:" + str(maxes))
@@ -361,12 +389,15 @@
main.log.info("Standard deviation of throughput: " + str(stdTP) + " Kflows/second")
resultsLog = open("flowTP1gDB","a")
- resultsLog.write("'" + commit + "',")
- resultsLog.write("'1gig',")
- resultsLog.write((main.params[ 'TEST' ][ 'flows' ]) + ",")
- resultsLog.write(str(clusterCount)+ ",")
- resultsLog.write(str(n) + ",")
- resultsLog.write(str(avgTP) + "," + str(stdTP) + "\n")
+ resultString = ("'" + commit + "',")
+ resultString += ("'1gig',")
+ resultString += ((main.params[ 'TEST' ][ 'flows' ]) + ",")
+ resultString += (str(clusterCount) + ",")
+ resultString += (str(n) + ",")
+ resultString += (str(avgTP) + "," + str(stdTP) + "\n")
+ resultsLog.write(resultString)
resultsLog.close()
-
-
+
+ main.log.report("Result line to file: " + resultString)
+
+ main.ONOSbench.onosErrorLog(ONOSIp[1])
diff --git a/TestON/tests/flowTP1g/flowTP1g.topo b/TestON/tests/flowTP1g/flowTP1g.topo
index 30a1467..763a4d6 100644
--- a/TestON/tests/flowTP1g/flowTP1g.topo
+++ b/TestON/tests/flowTP1g/flowTP1g.topo
@@ -3,7 +3,7 @@
<COMPONENT>
<ONOSbench>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosDriver</type>
@@ -12,7 +12,7 @@
</ONOSbench>
<ONOS1cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -21,7 +21,7 @@
</ONOS1cli>
<ONOS2cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -30,7 +30,7 @@
</ONOS2cli>
<ONOS3cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -39,7 +39,7 @@
</ONOS3cli>
<ONOS4cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -48,7 +48,7 @@
</ONOS4cli>
<ONOS5cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -57,7 +57,7 @@
</ONOS5cli>
<ONOS6cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -66,7 +66,7 @@
</ONOS6cli>
<ONOS7cli>
- <host>10.128.5.55</host>
+ <host>10.128.5.50</host>
<user>admin</user>
<password>onos_test</password>
<type>OnosCliDriver</type>
@@ -112,7 +112,7 @@
<ONOS5>
- <host>10.128.5.65</host>
+ <host>10.128.5.55</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -121,7 +121,7 @@
</ONOS5>
<ONOS6>
- <host>10.128.5.66</host>
+ <host>10.128.5.56</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -130,7 +130,7 @@
</ONOS6>
<ONOS7>
- <host>10.128.5.67</host>
+ <host>10.128.5.57</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosDriver</type>
@@ -141,6 +141,4 @@
</COMPONENT>
</TOPOLOGY>
-
-
-
+